Add queryable store with SQLite implementation

This adds the new "queryable store" API, that allows complex
queries and is backed by SQLite. This new API can be used
for entities annotated with the new QueryableType annotation.
This commit is contained in:
Rene Pfeuffer
2025-04-01 16:18:04 +02:00
parent d5362d634b
commit ada575d871
235 changed files with 10154 additions and 252 deletions

View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.group;
import com.github.legman.ReferenceType;
import com.github.legman.Subscribe;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.store.StoreDeletionNotifier;
@Extension
public class GroupDeletionNotifier implements StoreDeletionNotifier {
private DeletionHandler handler;
@Override
public void registerHandler(DeletionHandler handler) {
this.handler = handler;
}
@Subscribe(referenceType = ReferenceType.STRONG)
public void onDelete(GroupEvent event) {
if (handler != null && event.getEventType() == HandlerEventType.DELETE) {
handler.notifyDeleted(Group.class, event.getItem().getId());
}
}
}

View File

@@ -37,6 +37,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
@@ -47,6 +48,7 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
static final String SCM_ENVIRONMENT_FILE_NAME = "scm-environment.xml";
static final String METADATA_FILE_NAME = "metadata.xml";
static final String STORE_DATA_FILE_NAME = "store-data.tar";
static final String QUERYABLE_STORE_DATA_FILE_NAME = "queryable-store-data.tar";
private final EnvironmentInformationXmlGenerator environmentGenerator;
private final RepositoryMetadataXmlGenerator metadataGenerator;
private final RepositoryServiceFactory serviceFactory;
@@ -55,17 +57,20 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
private final RepositoryExportingCheck repositoryExportingCheck;
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
private final ExportNotificationHandler notificationHandler;
private final AdministrationContext administrationContext;
private final RepositoryQueryableStoreExporter queryableStoreExporter;
@Inject
public FullScmRepositoryExporter(EnvironmentInformationXmlGenerator environmentGenerator,
RepositoryMetadataXmlGenerator metadataGenerator,
RepositoryServiceFactory serviceFactory,
TarArchiveRepositoryStoreExporter storeExporter,
WorkdirProvider workdirProvider,
RepositoryExportingCheck repositoryExportingCheck,
RepositoryImportExportEncryption repositoryImportExportEncryption, ExportNotificationHandler notificationHandler, AdministrationContext administrationContext) {
FullScmRepositoryExporter(EnvironmentInformationXmlGenerator environmentGenerator,
RepositoryMetadataXmlGenerator metadataGenerator,
RepositoryServiceFactory serviceFactory,
TarArchiveRepositoryStoreExporter storeExporter,
WorkdirProvider workdirProvider,
RepositoryExportingCheck repositoryExportingCheck,
RepositoryImportExportEncryption repositoryImportExportEncryption,
ExportNotificationHandler notificationHandler,
AdministrationContext administrationContext,
RepositoryQueryableStoreExporter queryableStoreExporter) {
this.environmentGenerator = environmentGenerator;
this.metadataGenerator = metadataGenerator;
this.serviceFactory = serviceFactory;
@@ -75,6 +80,7 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
this.notificationHandler = notificationHandler;
this.administrationContext = administrationContext;
this.queryableStoreExporter = queryableStoreExporter;
}
public void export(Repository repository, OutputStream outputStream, String password) {
@@ -95,11 +101,12 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
BufferedOutputStream bos = new BufferedOutputStream(outputStream);
OutputStream cos = repositoryImportExportEncryption.optionallyEncrypt(bos, password);
GzipCompressorOutputStream gzos = new GzipCompressorOutputStream(cos);
TarArchiveOutputStream taos = Archives.createTarOutputStream(gzos);
TarArchiveOutputStream taos = Archives.createTarOutputStream(gzos)
) {
writeEnvironmentData(repository, taos);
writeMetadata(repository, taos);
writeStoreData(repository, taos);
writeQueryableStoreData(repository, taos);
writeRepository(service, taos);
taos.finish();
} catch (IOException e) {
@@ -136,20 +143,13 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
}
private void writeRepository(RepositoryService service, TarArchiveOutputStream taos) throws IOException {
File newWorkdir = workdirProvider.createNewWorkdir(service.getRepository().getId());
try {
createAndAddFromTemporaryDirectory(service.getRepository(), taos, createRepositoryEntryName(service), newWorkdir -> {
File repositoryFile = Files.createFile(Paths.get(newWorkdir.getPath(), "repository")).toFile();
try (FileOutputStream repositoryFos = new FileOutputStream(repositoryFile)) {
service.getBundleCommand().bundle(repositoryFos);
}
TarArchiveEntry entry = new TarArchiveEntry(createRepositoryEntryName(service));
entry.setSize(repositoryFile.length());
taos.putArchiveEntry(entry);
Files.copy(repositoryFile.toPath(), taos);
taos.closeArchiveEntry();
} finally {
IOUtil.deleteSilently(newWorkdir);
}
return repositoryFile;
});
}
private String createRepositoryEntryName(RepositoryService service) {
@@ -157,19 +157,46 @@ public class FullScmRepositoryExporter implements FullRepositoryExporter {
}
private void writeStoreData(Repository repository, TarArchiveOutputStream taos) throws IOException {
File newWorkdir = workdirProvider.createNewWorkdir(repository.getId());
try {
createAndAddFromTemporaryDirectory(repository, taos, STORE_DATA_FILE_NAME, newWorkdir -> {
File metadata = Files.createFile(Paths.get(newWorkdir.getPath(), "metadata")).toFile();
try (FileOutputStream metadataFos = new FileOutputStream(metadata)) {
storeExporter.export(repository, metadataFos);
}
TarArchiveEntry entry = new TarArchiveEntry(STORE_DATA_FILE_NAME);
entry.setSize(metadata.length());
taos.putArchiveEntry(entry);
Files.copy(metadata.toPath(), taos);
taos.closeArchiveEntry();
return metadata;
});
}
private void writeQueryableStoreData(Repository repository, TarArchiveOutputStream taos) throws IOException {
createAndAddFromTemporaryDirectory(repository, taos, QUERYABLE_STORE_DATA_FILE_NAME, newWorkdir -> {
Path queryableTarFilePath = Paths.get(newWorkdir.getPath(), QUERYABLE_STORE_DATA_FILE_NAME);
File queryableTarFile = Files.createFile(queryableTarFilePath).toFile();
try (FileOutputStream fos = new FileOutputStream(queryableTarFile);
TarArchiveOutputStream tempTaos = Archives.createTarOutputStream(fos)) {
queryableStoreExporter.addQueryableStoreDataToArchive(repository, newWorkdir, tempTaos);
}
return queryableTarFile;
});
}
private void createAndAddFromTemporaryDirectory(Repository repository, TarArchiveOutputStream taos, String entryName, PackFileProducer packFileProducer) throws IOException {
File newWorkdir = workdirProvider.createNewWorkdir(repository.getId());
try {
File tempFile = packFileProducer.packFile(newWorkdir);
addToTar(entryName, tempFile, taos);
} finally {
IOUtil.deleteSilently(newWorkdir);
}
}
private static void addToTar(String storeDataFileName, File metadata, TarArchiveOutputStream taos) throws IOException {
TarArchiveEntry entry = new TarArchiveEntry(storeDataFileName);
entry.setSize(metadata.length());
taos.putArchiveEntry(entry);
Files.copy(metadata.toPath(), taos);
taos.closeArchiveEntry();
}
private interface PackFileProducer {
File packFile(File newWorkdir) throws IOException;
}
}

View File

@@ -26,6 +26,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.event.ScmEventBus;
import sonia.scm.repository.ClearRepositoryCacheEvent;
import sonia.scm.repository.FullRepositoryImporter;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryImportEvent;
@@ -33,6 +34,7 @@ import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.RepositoryPermissions;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.update.UpdateEngine;
import java.io.BufferedInputStream;
import java.io.IOException;
@@ -53,21 +55,31 @@ public class FullScmRepositoryImporter implements FullRepositoryImporter {
private final RepositoryImportExportEncryption repositoryImportExportEncryption;
private final ScmEventBus eventBus;
private final RepositoryImportLoggerFactory loggerFactory;
private final UpdateEngine updateEngine;
@Inject
public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
MetadataImportStep metadataImportStep,
StoreImportStep storeImportStep,
QueryableStoreImportStep queryableStoreImportStep,
RepositoryImportStep repositoryImportStep,
RepositoryManager repositoryManager,
RepositoryImportExportEncryption repositoryImportExportEncryption,
RepositoryImportLoggerFactory loggerFactory,
ScmEventBus eventBus) {
ScmEventBus eventBus,
UpdateEngine updateEngine) {
this.repositoryManager = repositoryManager;
this.loggerFactory = loggerFactory;
this.repositoryImportExportEncryption = repositoryImportExportEncryption;
this.eventBus = eventBus;
importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
this.updateEngine = updateEngine;
importSteps = new ImportStep[]{
environmentCheckStep,
metadataImportStep,
storeImportStep,
queryableStoreImportStep,
repositoryImportStep
};
}
public Repository importFromStream(Repository repository, InputStream inputStream, String password) {
@@ -122,11 +134,17 @@ public class FullScmRepositoryImporter implements FullRepositoryImporter {
logger.repositoryCreated(state.getRepository());
try {
TarArchiveEntry tarArchiveEntry;
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
while ((tarArchiveEntry = tais.getNextEntry()) != null) {
LOG.trace("Trying to handle tar entry '{}'", tarArchiveEntry.getName());
handle(tais, state, tarArchiveEntry);
}
stream(importSteps).forEach(step -> step.finish(state));
eventBus.post(new ClearRepositoryCacheEvent(createdRepository));
updateEngine.update(repository.getId());
eventBus.post(new ClearRepositoryCacheEvent(createdRepository));
state.getLogger().finished();
return state.getRepository();
} catch (RuntimeException | IOException e) {

View File

@@ -0,0 +1,34 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import java.io.IOException;
import java.io.InputStream;
class NoneClosingTarArchiveInputStream extends TarArchiveInputStream {
NoneClosingTarArchiveInputStream(InputStream is) {
super(is);
}
@Override
public void close() throws IOException {
// Do not close this input stream
}
}

View File

@@ -0,0 +1,85 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import sonia.scm.repository.RepositoryLocationResolver;
import sonia.scm.repository.api.ImportFailedException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
import static sonia.scm.importexport.FullScmRepositoryExporter.QUERYABLE_STORE_DATA_FILE_NAME;
@Slf4j
class QueryableStoreImportStep implements ImportStep {
private final RepositoryQueryableStoreExporter queryableStoreExporter;
private final RepositoryLocationResolver locationResolver;
@Inject
QueryableStoreImportStep(RepositoryQueryableStoreExporter queryableStoreExporter, RepositoryLocationResolver locationResolver) {
this.queryableStoreExporter = queryableStoreExporter;
this.locationResolver = locationResolver;
}
@Override
public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) {
if (entry.getName().equals(QUERYABLE_STORE_DATA_FILE_NAME) && !entry.isDirectory()) {
log.trace("Importing store from tar");
state.getLogger().step("importing queryable stores");
Path repositoryPath = locationResolver
.forClass(Path.class)
.getLocation(state.getRepository().getId());
try {
extractTarToDirectory(inputStream, repositoryPath.toFile());
queryableStoreExporter.importStores(state.getRepository().getId(), repositoryPath.toFile());
return true;
} catch (IOException e) {
throw new ImportFailedException(entity(state.getRepository()).build(), "Failed to extract TAR content", e);
}
}
return false;
}
private void extractTarToDirectory(InputStream inputStream, File outputDir) throws IOException {
try (TarArchiveInputStream tarInput = new NoneClosingTarArchiveInputStream(inputStream)) {
TarArchiveEntry entry;
while ((entry = tarInput.getNextEntry()) != null) {
if (entry.isDirectory()) {
continue;
}
File outputFile = new File(outputDir, entry.getName());
outputFile.getParentFile().mkdirs();
try (OutputStream outputStream = Files.newOutputStream(outputFile.toPath())) {
tarInput.transferTo(outputStream);
}
}
}
}
}

View File

@@ -0,0 +1,81 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import jakarta.inject.Inject;
import lombok.extern.slf4j.Slf4j;
import sonia.scm.repository.RepositoryLocationResolver;
import sonia.scm.repository.xml.PathBasedRepositoryLocationResolver;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* The job of this class is to check for remaining queryable store data from former imports, that have not been
* imported yet. This can happen if a repository was imported, when not all plugins were installed but those plugins
* are installed now. After this is done, the update process has to be run for all repositories.
*/
@Slf4j
public class RemainingQueryableStoreImporter {
private final RepositoryLocationResolver.RepositoryLocationResolverInstance<Path> repositoryLocationResolverInstance;
private final RepositoryQueryableStoreExporter queryableStoreExporter;
@Inject
public RemainingQueryableStoreImporter(PathBasedRepositoryLocationResolver repositoryLocationResolver,
RepositoryQueryableStoreExporter queryableStoreExporter) {
this.repositoryLocationResolverInstance = repositoryLocationResolver.create(Path.class);
this.queryableStoreExporter = queryableStoreExporter;
}
public void onInitializationCompleted() {
log.info("Starting import of remaining queryable store data for all repositories.");
repositoryLocationResolverInstance.forAllLocations((repositoryId, repositoryPath) -> {
File repoDir = repositoryPath.toFile();
File dataDir = new File(repoDir, "queryable-store-data");
if (dataDir.exists() && dataDir.isDirectory()) {
List<File> xmlFiles = getXmlFiles(dataDir);
if (!xmlFiles.isEmpty()) {
log.info("Found {} XML files in repository {} - importing...", xmlFiles.size(), repositoryId);
queryableStoreExporter.importStores(repositoryId, repoDir);
}
}
});
log.info("Finished importing queryable store data.");
}
private List<File> getXmlFiles(File directory) {
try (Stream<Path> files = Files.list(directory.toPath())) {
return files
.map(Path::toFile)
.filter(file -> file.getName().endsWith(".xml"))
.collect(Collectors.toList());
} catch (IOException e) {
log.error("Error reading directory {}: {}", directory.getAbsolutePath(), e.getMessage());
return List.of();
}
}
}

View File

@@ -0,0 +1,146 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import jakarta.inject.Inject;
import jakarta.xml.bind.JAXBContext;
import jakarta.xml.bind.JAXBException;
import jakarta.xml.bind.Marshaller;
import jakarta.xml.bind.Unmarshaller;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlRootElement;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import sonia.scm.repository.Repository;
import sonia.scm.store.QueryableMaintenanceStore;
import sonia.scm.store.QueryableStoreFactory;
import sonia.scm.store.StoreMetaDataProvider;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@Slf4j
public class RepositoryQueryableStoreExporter {
private final StoreMetaDataProvider metaDataProvider;
private final QueryableStoreFactory storeFactory;
@Inject
RepositoryQueryableStoreExporter(StoreMetaDataProvider metaDataProvider,
QueryableStoreFactory storeFactory) {
this.metaDataProvider = metaDataProvider;
this.storeFactory = storeFactory;
}
void addQueryableStoreDataToArchive(Repository repository, File newWorkdir, TarArchiveOutputStream tempTaos) throws IOException {
TarArchiveEntry dirEntry = new TarArchiveEntry("queryable-store-data/");
tempTaos.putArchiveEntry(dirEntry);
tempTaos.closeArchiveEntry();
File dataDir = new File(newWorkdir, "queryable-store-data");
if (!dataDir.mkdirs()) {
throw new RuntimeException("Could not create temp directory: " + dataDir.getAbsolutePath());
}
exportStores(repository.getId(), dataDir);
File[] xmlFiles = dataDir.listFiles();
if (xmlFiles != null) {
for (File xmlFile : xmlFiles) {
TarArchiveEntry fileEntry = new TarArchiveEntry("queryable-store-data/" + xmlFile.getName());
fileEntry.setSize(xmlFile.length());
tempTaos.putArchiveEntry(fileEntry);
Files.copy(xmlFile.toPath(), tempTaos);
tempTaos.closeArchiveEntry();
}
}
tempTaos.finish();
}
void exportStores(String repositoryId, File workdir) {
try {
JAXBContext jaxbContext = JAXBContext.newInstance(StoreExport.class);
Marshaller marshaller = jaxbContext.createMarshaller();
for (Class<?> type : metaDataProvider.getTypesWithParent(Repository.class)) {
Collection<QueryableMaintenanceStore.RawRow> rows = storeFactory.getForMaintenance(type, repositoryId).readRaw();
StoreExport export = new StoreExport(type, rows);
marshaller.marshal(export, new File(workdir, type.getName() + ".xml"));
}
} catch (JAXBException e) {
throw new RuntimeException(e);
}
}
void importStores(String repositoryId, File workdir) {
try {
File dataDir = new File(workdir, "queryable-store-data");
if (!dataDir.exists() || !dataDir.isDirectory()) {
throw new RuntimeException("Directory 'queryable-store-data' not found in workdir: " + workdir.getAbsolutePath());
}
JAXBContext jaxbContext = JAXBContext.newInstance(StoreExport.class);
Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
for (Class<?> type : metaDataProvider.getTypesWithParent(Repository.class)) {
File file = new File(dataDir, type.getName() + ".xml");
if (!file.exists() || file.length() == 0) {
continue;
}
StoreExport export = (StoreExport) unmarshaller.unmarshal(file);
Collection<QueryableMaintenanceStore.RawRow> rows = export.getRows();
if (rows == null) {
continue;
}
storeFactory.getForMaintenance(type, repositoryId).writeRaw(rows);
try {
Files.delete(file.toPath());
log.trace("Deleted imported file: {}", file.getAbsolutePath());
} catch (IOException e) {
log.error("Failed to delete imported file: {} - {}", file.getAbsolutePath(), e.getMessage());
}
}
} catch (JAXBException e) {
throw new RuntimeException(e);
}
}
@Getter
@XmlRootElement
@NoArgsConstructor
@XmlAccessorType(XmlAccessType.FIELD)
private static class StoreExport {
private String type;
private Collection<QueryableMaintenanceStore.RawRow> rows = new ArrayList<>();
StoreExport(Class<?> type, Collection<QueryableMaintenanceStore.RawRow> rows) {
this.type = type.getName();
this.rows = rows != null ? rows : new ArrayList<>();
}
}
}

View File

@@ -21,7 +21,6 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.update.UpdateEngine;
import java.io.InputStream;
@@ -32,12 +31,10 @@ class StoreImportStep implements ImportStep {
private static final Logger LOG = LoggerFactory.getLogger(StoreImportStep.class);
private final TarArchiveRepositoryStoreImporter storeImporter;
private final UpdateEngine updateEngine;
@Inject
StoreImportStep(TarArchiveRepositoryStoreImporter storeImporter, UpdateEngine updateEngine) {
StoreImportStep(TarArchiveRepositoryStoreImporter storeImporter) {
this.storeImporter = storeImporter;
this.updateEngine = updateEngine;
}
@Override
@@ -47,15 +44,11 @@ class StoreImportStep implements ImportStep {
state.getLogger().step("importing stores");
// Inside the repository tar archive stream is another tar archive.
// The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
importStores(state.getRepository(), inputStream, state.getLogger());
Repository repository = state.getRepository();
storeImporter.importFromTarArchive(repository, inputStream, state.getLogger());
state.storeImported();
return true;
}
return false;
}
private void importStores(Repository repository, InputStream inputStream, RepositoryImportLogger logger) {
storeImporter.importFromTarArchive(repository, inputStream, logger);
updateEngine.update(repository.getId());
}
}

View File

@@ -110,16 +110,4 @@ public class TarArchiveRepositoryStoreImporter {
private boolean isConfigStore(String storeType) {
return storeType.equals(StoreType.CONFIG.getValue()) || storeType.equals(StoreType.CONFIG_ENTRY.getValue());
}
static class NoneClosingTarArchiveInputStream extends TarArchiveInputStream {
public NoneClosingTarArchiveInputStream(InputStream is) {
super(is);
}
@Override
public void close() throws IOException {
// Do not close this input stream
}
}
}

View File

@@ -27,6 +27,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.SCMContext;
import sonia.scm.config.LoggingConfiguration;
import sonia.scm.importexport.RemainingQueryableStoreImporter;
import sonia.scm.lifecycle.classloading.ClassLoaderLifeCycle;
import sonia.scm.lifecycle.modules.ApplicationModuleProvider;
import sonia.scm.lifecycle.modules.BootstrapModule;
@@ -172,6 +173,9 @@ public class BootstrapContextListener extends GuiceServletContextListener {
private void processUpdates(PluginLoader pluginLoader, Injector bootstrapInjector) {
Injector updateInjector = bootstrapInjector.createChildInjector(new UpdateStepModule(pluginLoader));
RemainingQueryableStoreImporter importer = updateInjector.getInstance(RemainingQueryableStoreImporter.class);
importer.onInitializationCompleted();
UpdateEngine updateEngine = updateInjector.getInstance(UpdateEngine.class);
updateEngine.update();
}

View File

@@ -16,6 +16,7 @@
package sonia.scm.lifecycle.modules;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.AbstractModule;
import com.google.inject.TypeLiteral;
import com.google.inject.multibindings.Multibinder;
@@ -25,6 +26,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.SCMContext;
import sonia.scm.SCMContextProvider;
import sonia.scm.api.rest.ObjectMapperProvider;
import sonia.scm.cache.CacheManager;
import sonia.scm.cache.GuavaCacheManager;
import sonia.scm.io.DefaultFileSystem;
@@ -52,15 +54,19 @@ import sonia.scm.store.ConfigurationStore;
import sonia.scm.store.ConfigurationStoreDecoratorFactory;
import sonia.scm.store.ConfigurationStoreFactory;
import sonia.scm.store.DataStoreFactory;
import sonia.scm.store.DefaultBlobDirectoryAccess;
import sonia.scm.store.FileBlobStoreFactory;
import sonia.scm.store.FileNamespaceUpdateIterator;
import sonia.scm.store.FileRepositoryUpdateIterator;
import sonia.scm.store.FileStoreUpdateStepUtilFactory;
import sonia.scm.store.JAXBConfigurationEntryStoreFactory;
import sonia.scm.store.JAXBConfigurationStoreFactory;
import sonia.scm.store.JAXBDataStoreFactory;
import sonia.scm.store.JAXBPropertyFileAccess;
import sonia.scm.store.QueryableStoreFactory;
import sonia.scm.store.StoreMetaDataProvider;
import sonia.scm.store.file.DefaultBlobDirectoryAccess;
import sonia.scm.store.file.FileBlobStoreFactory;
import sonia.scm.store.file.FileNamespaceUpdateIterator;
import sonia.scm.store.file.FileRepositoryUpdateIterator;
import sonia.scm.store.file.JAXBConfigurationEntryStoreFactory;
import sonia.scm.store.file.JAXBConfigurationStoreFactory;
import sonia.scm.store.file.JAXBDataStoreFactory;
import sonia.scm.store.file.JAXBPropertyFileAccess;
import sonia.scm.store.sqlite.SQLiteQueryableStoreFactory;
import sonia.scm.store.sqlite.SQLiteStoreMetaDataProvider;
import sonia.scm.update.BlobDirectoryAccess;
import sonia.scm.update.DefaultRepositoryPermissionUpdater;
import sonia.scm.update.NamespaceUpdateIterator;
@@ -116,6 +122,8 @@ public class BootstrapModule extends AbstractModule {
bind(ConfigurationEntryStoreFactory.class, JAXBConfigurationEntryStoreFactory.class);
bind(DataStoreFactory.class, JAXBDataStoreFactory.class);
bind(BlobStoreFactory.class, FileBlobStoreFactory.class);
bind(QueryableStoreFactory.class, SQLiteQueryableStoreFactory.class);
bind(StoreMetaDataProvider.class, SQLiteStoreMetaDataProvider.class);
bind(PluginLoader.class).toInstance(pluginLoader);
bind(V1PropertyDAO.class, XmlV1PropertyDAO.class);
bind(PropertyFileAccess.class, JAXBPropertyFileAccess.class);
@@ -123,8 +131,9 @@ public class BootstrapModule extends AbstractModule {
bind(RepositoryUpdateIterator.class, FileRepositoryUpdateIterator.class);
bind(NamespaceUpdateIterator.class, FileNamespaceUpdateIterator.class);
bind(StoreUpdateStepUtilFactory.class, FileStoreUpdateStepUtilFactory.class);
bind(RepositoryPermissionUpdater.class, DefaultRepositoryPermissionUpdater.class);
bind(ObjectMapper.class).toProvider(ObjectMapperProvider.class);
bind(new TypeLiteral<UpdateStepRepositoryMetadataAccess<Path>>() {}).to(new TypeLiteral<MetadataStore>() {});
bind(RepositoryPermissionUpdater.class, DefaultRepositoryPermissionUpdater.class);
// bind metrics
bind(MeterRegistry.class).toProvider(MeterRegistryProvider.class).asEagerSingleton();

View File

@@ -16,7 +16,6 @@
package sonia.scm.lifecycle.modules;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Provider;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.servlet.RequestScoped;
@@ -31,7 +30,6 @@ import sonia.scm.PushStateDispatcherProvider;
import sonia.scm.RootURL;
import sonia.scm.Undecorated;
import sonia.scm.admin.ScmConfigurationStore;
import sonia.scm.api.rest.ObjectMapperProvider;
import sonia.scm.api.v2.resources.BranchLinkProvider;
import sonia.scm.api.v2.resources.DefaultBranchLinkProvider;
import sonia.scm.api.v2.resources.DefaultRepositoryLinkProvider;
@@ -110,7 +108,7 @@ import sonia.scm.security.LoginAttemptHandler;
import sonia.scm.security.RepositoryPermissionProvider;
import sonia.scm.security.SecuritySystem;
import sonia.scm.store.ConfigurationStoreDecoratorFactory;
import sonia.scm.store.FileStoreExporter;
import sonia.scm.store.file.FileStoreExporter;
import sonia.scm.store.StoreExporter;
import sonia.scm.template.MustacheTemplateEngine;
import sonia.scm.template.TemplateEngine;
@@ -258,7 +256,6 @@ class ScmServletModule extends ServletModule {
bind(TemplateEngine.class).annotatedWith(Default.class).to(
MustacheTemplateEngine.class);
bind(TemplateEngineFactory.class);
bind(ObjectMapper.class).toProvider(ObjectMapperProvider.class);
// bind events

View File

@@ -82,9 +82,13 @@ public class DefaultExtensionProcessor implements ExtensionProcessor {
return collector.getIndexedTypes();
}
@Override
public Iterable<QueryableTypeDescriptor> getQueryableTypes() {
return collector.getQueryableTypes();
}
@Override
public Iterable<ConfigBinding> getConfigBindings() {
return configBindings;
}
}

View File

@@ -50,6 +50,7 @@ public final class ExtensionCollector {
private final Set<ConfigElement> configElements = Sets.newHashSet();
private final Multimap<ExtensionPointElement, Class> extensions = HashMultimap.create();
private final Map<Class, ExtensionPointElement> extensionPointIndex = Maps.newHashMap();
private final Set<QueryableTypeDescriptor> queryableTypes = Sets.newHashSet();
public ExtensionCollector(ClassLoader moduleClassLoader, Set<ScmModule> modules, Set<InstalledPlugin> installedPlugins) {
this.pluginIndex = createPluginIndex(installedPlugins);
@@ -144,6 +145,10 @@ public final class ExtensionCollector {
return indexedTypes;
}
public Iterable<QueryableTypeDescriptor> getQueryableTypes() {
return queryableTypes;
}
private void appendExtension(Class extension) {
boolean found = false;
@@ -221,6 +226,16 @@ public final class ExtensionCollector {
return true;
}
private Collection<? extends QueryableTypeDescriptor> collectQueryableTypes(ClassLoader defaultClassLoader, Iterable<QueryableTypeDescriptor> descriptors) {
Set<QueryableTypeDescriptor> queryableTypes = new HashSet<>();
for (QueryableTypeDescriptor descriptor : descriptors) {
if (isRequirementFulfilled(descriptor)) {
queryableTypes.add(descriptor);
}
}
return queryableTypes;
}
private void collectRootElements(ClassLoader classLoader, ScmModule module) {
for (ExtensionPointElement epe : module.getExtensionPoints()) {
extensionPointIndex.put(epe.getClazz(), epe);
@@ -233,5 +248,6 @@ public final class ExtensionCollector {
webElements.addAll(collectWebElementExtensions(classLoader, module.getWebElements()));
indexedTypes.addAll(collectIndexedTypes(classLoader, module.getIndexedTypes()));
Iterables.addAll(configElements, module.getConfigElements());
queryableTypes.addAll(collectQueryableTypes(classLoader, module.getQueryableTypes()));
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.repository;
import com.github.legman.ReferenceType;
import com.github.legman.Subscribe;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.store.StoreDeletionNotifier;
@Extension
class RepositoryDeletionNotifier implements StoreDeletionNotifier {
private DeletionHandler handler;
@Override
public void registerHandler(DeletionHandler handler) {
this.handler = handler;
}
@Subscribe(referenceType = ReferenceType.STRONG)
public void onDelete(RepositoryEvent event) {
if (handler != null && event.getEventType() == HandlerEventType.DELETE) {
handler.notifyDeleted(Repository.class, event.getItem().getId());
}
}
}

View File

@@ -0,0 +1,51 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.store;
import jakarta.inject.Inject;
import sonia.scm.EagerSingleton;
import sonia.scm.plugin.Extension;
import java.util.Collection;
import java.util.Set;
@Extension
@EagerSingleton
class QueryableStoreDeletionHandler implements StoreDeletionNotifier.DeletionHandler {
private final StoreMetaDataProvider metaDataProvider;
private final QueryableStoreFactory storeFactory;
@Inject
QueryableStoreDeletionHandler(Set<StoreDeletionNotifier> notifiers, StoreMetaDataProvider metaDataProvider, QueryableStoreFactory storeFactory) {
this.metaDataProvider = metaDataProvider;
this.storeFactory = storeFactory;
notifiers.forEach(notifier -> notifier.registerHandler(this));
}
@Override
public void notifyDeleted(StoreDeletionNotifier.ClassWithId... classWithIds) {
Class<?>[] classes = new Class[classWithIds.length];
String[] ids = new String[classWithIds.length];
for (int i = 0; i < classWithIds.length; i++) {
classes[i] = classWithIds[i].clazz();
ids[i] = classWithIds[i].id();
}
Collection<Class<?>> typesWithParent = metaDataProvider.getTypesWithParent(classes);
typesWithParent.forEach(type -> storeFactory.getForMaintenance(type, ids).clear());
}
}

View File

@@ -33,7 +33,7 @@ import sonia.scm.migration.UpdateStep;
import sonia.scm.plugin.Extension;
import sonia.scm.store.ConfigurationEntryStore;
import sonia.scm.store.ConfigurationEntryStoreFactory;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.update.V1Properties;
import sonia.scm.version.Version;

View File

@@ -29,8 +29,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import static sonia.scm.store.StoreConstants.DATA_DIRECTORY_NAME;
import static sonia.scm.store.StoreConstants.VARIABLE_DATA_DIRECTORY_NAME;
import static sonia.scm.store.file.StoreConstants.DATA_DIRECTORY_NAME;
import static sonia.scm.store.file.StoreConstants.VARIABLE_DATA_DIRECTORY_NAME;
@Extension
public class RemoveCombinedIndex implements UpdateStep {

View File

@@ -31,7 +31,7 @@ import sonia.scm.plugin.Extension;
import sonia.scm.security.AnonymousMode;
import sonia.scm.store.ConfigurationStore;
import sonia.scm.store.ConfigurationStoreFactory;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.version.Version;
import java.nio.file.Path;

View File

@@ -23,7 +23,7 @@ import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlRootElement;
import sonia.scm.SCMContextProvider;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import java.io.File;
import java.nio.file.Paths;

View File

@@ -24,7 +24,7 @@ import sonia.scm.migration.UpdateStep;
import sonia.scm.plugin.Extension;
import sonia.scm.repository.xml.PathBasedRepositoryLocationResolver;
import sonia.scm.repository.xml.XmlRepositoryDAO;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.version.Version;
import java.io.IOException;

View File

@@ -29,7 +29,7 @@ import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.xml.XmlRepositoryDAO;
import sonia.scm.store.ConfigurationEntryStore;
import sonia.scm.store.ConfigurationEntryStoreFactory;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.update.CoreUpdateStep;
import sonia.scm.update.V1Properties;
import sonia.scm.version.Version;

View File

@@ -31,7 +31,7 @@ import sonia.scm.plugin.Extension;
import sonia.scm.security.AssignedPermission;
import sonia.scm.store.ConfigurationEntryStore;
import sonia.scm.store.ConfigurationEntryStoreFactory;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.version.Version;
import java.io.File;

View File

@@ -21,7 +21,7 @@ import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
import sonia.scm.migration.UpdateException;
import sonia.scm.store.CopyOnWrite;
import sonia.scm.CopyOnWrite;
import sonia.scm.version.Version;
import sonia.scm.xml.XmlStreams;
import sonia.scm.xml.XmlStreams.AutoCloseableXMLReader;
@@ -41,7 +41,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.stream.Stream;
import static sonia.scm.store.CopyOnWrite.compute;
import static sonia.scm.CopyOnWrite.compute;
abstract class DifferentiateBetweenConfigAndConfigEntryUpdateStep {

View File

@@ -32,7 +32,7 @@ import sonia.scm.plugin.Extension;
import sonia.scm.security.AssignedPermission;
import sonia.scm.store.ConfigurationEntryStore;
import sonia.scm.store.ConfigurationEntryStoreFactory;
import sonia.scm.store.StoreConstants;
import sonia.scm.store.file.StoreConstants;
import sonia.scm.update.V1Properties;
import sonia.scm.user.User;
import sonia.scm.user.xml.XmlUserDAO;

View File

@@ -0,0 +1,40 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.user;
import com.github.legman.ReferenceType;
import com.github.legman.Subscribe;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.store.StoreDeletionNotifier;
@Extension
public class UserDeletionNotifier implements StoreDeletionNotifier {
private DeletionHandler handler;
@Override
public void registerHandler(DeletionHandler handler) {
this.handler = handler;
}
@Subscribe(referenceType = ReferenceType.STRONG)
public void onDelete(UserEvent event) {
if (handler != null && event.getEventType() == HandlerEventType.DELETE) {
handler.notifyDeleted(User.class, event.getItem().getId());
}
}
}

View File

@@ -16,6 +16,7 @@
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -36,6 +37,7 @@ import sonia.scm.web.security.PrivilegedAction;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
@@ -48,6 +50,7 @@ import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -76,6 +79,8 @@ class FullScmRepositoryExporterTest {
private AdministrationContext administrationContext;
@Mock
private RepositoryImportExportEncryption repositoryImportExportEncryption;
@Mock
private RepositoryQueryableStoreExporter queryableStoreExporter;
@InjectMocks
private FullScmRepositoryExporter exporter;
@@ -88,10 +93,23 @@ class FullScmRepositoryExporterTest {
when(metadataGenerator.generate(REPOSITORY)).thenReturn(new byte[0]);
when(repositoryExportingCheck.withExportingLock(any(), any())).thenAnswer(invocation -> invocation.getArgument(1, Supplier.class).get());
when(repositoryImportExportEncryption.optionallyEncrypt(any(), any())).thenAnswer(invocation -> invocation.getArgument(0));
doAnswer(invocation -> {
File directory = invocation.getArgument(1, File.class);
File dummyFile = new File(directory, "dummy.xml");
try (FileWriter writer = new FileWriter(dummyFile)) {
writer.write("<dummy>Dummy content for testing</dummy>");
}
return null;
}).when(queryableStoreExporter).addQueryableStoreDataToArchive(
any(Repository.class),
any(File.class),
any(TarArchiveOutputStream.class)
);
}
@Test
void shouldExportEverythingAsTarArchive(@TempDir Path temp) {
void shouldExportEverythingAsTarArchive(@TempDir Path temp) throws IOException {
BundleCommandBuilder bundleCommandBuilder = mock(BundleCommandBuilder.class);
when(repositoryService.getBundleCommand()).thenReturn(bundleCommandBuilder);
when(repositoryService.getRepository()).thenReturn(REPOSITORY);
@@ -104,6 +122,8 @@ class FullScmRepositoryExporterTest {
verify(metadataGenerator, times(1)).generate(REPOSITORY);
verify(bundleCommandBuilder, times(1)).bundle(any(OutputStream.class));
verify(repositoryExportingCheck).withExportingLock(eq(REPOSITORY), any());
verify(queryableStoreExporter, times(1))
.addQueryableStoreDataToArchive(eq(REPOSITORY), any(File.class), any(TarArchiveOutputStream.class));
workDirsCreated.forEach(wd -> assertThat(wd).doesNotExist());
}

View File

@@ -32,6 +32,7 @@ import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.event.ScmEventBus;
import sonia.scm.repository.ClearRepositoryCacheEvent;
import sonia.scm.repository.ImportRepositoryHookEvent;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryHookEvent;
@@ -62,7 +63,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.lenient;
@@ -114,6 +114,8 @@ class FullScmRepositoryImporterTest {
private StoreImportStep storeImportStep;
@InjectMocks
private RepositoryImportStep repositoryImportStep;
@InjectMocks
private QueryableStoreImportStep queryableStoreImportStep;
@Mock
private RepositoryHookEvent event;
@@ -129,11 +131,13 @@ class FullScmRepositoryImporterTest {
environmentCheckStep,
metadataImportStep,
storeImportStep,
queryableStoreImportStep,
repositoryImportStep,
repositoryManager,
repositoryImportExportEncryption,
loggerFactory,
eventBus);
eventBus,
updateEngine);
}
@BeforeEach
@@ -256,17 +260,30 @@ class FullScmRepositoryImporterTest {
fullImporter.importFromStream(REPOSITORY, stream, null);
assertThat(capturedEvents.getAllValues()).hasSize(2);
assertThat(capturedEvents.getAllValues()).anyMatch(
event ->
event instanceof ImportRepositoryHookEvent &&
((ImportRepositoryHookEvent) event).getRepository().equals(REPOSITORY)
);
assertThat(capturedEvents.getAllValues()).anyMatch(
event ->
event instanceof RepositoryImportEvent &&
((RepositoryImportEvent) event).getItem().equals(REPOSITORY)
);
assertThat(capturedEvents.getAllValues()).hasSize(4);
assertThat(capturedEvents.getAllValues())
.satisfiesExactlyInAnyOrder(
event ->
assertThat(event)
.isInstanceOf(ClearRepositoryCacheEvent.class)
.extracting("repository")
.isEqualTo(REPOSITORY),
event ->
assertThat(event)
.isInstanceOf(ClearRepositoryCacheEvent.class)
.extracting("repository")
.isEqualTo(REPOSITORY),
event ->
assertThat(event)
.isInstanceOf(RepositoryImportEvent.class)
.extracting("item")
.isEqualTo(REPOSITORY),
event ->
assertThat(event)
.isInstanceOf(ImportRepositoryHookEvent.class)
.extracting("repository")
.isEqualTo(REPOSITORY)
);
}
@Test

View File

@@ -0,0 +1,103 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import com.google.common.io.Resources;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryLocationResolver;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Path;
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class QueryableStoreImportStepTest {
private static final String QUERYABLE_STORE_DATA_FILE_NAME = "queryable-store-data.tar";
@Mock
private RepositoryQueryableStoreExporter queryableStoreExporter;
@Mock
private ImportState importState;
@Mock
private RepositoryImportLogger logger;
@Mock
private Repository repository;
@Mock
private RepositoryLocationResolver locationResolver;
@Mock
private RepositoryLocationResolver.RepositoryLocationResolverInstance<Path> forClass;
@InjectMocks
private QueryableStoreImportStep queryableStoreImportStep;
private File tarFile;
@TempDir
private File tempWorkDir;
@BeforeEach
void setUp() {
when(importState.getRepository()).thenReturn(repository);
when(importState.getLogger()).thenReturn(logger);
when(repository.getId()).thenReturn("42");
doNothing().when(logger).step(anyString());
when(locationResolver.forClass(Path.class)).thenReturn(forClass);
when(forClass.getLocation(anyString())).thenReturn(tempWorkDir.toPath());
tarFile = new File(Resources.getResource("sonia/scm/importexport/queryable-store-data.tar").getFile());
}
@Test
void shouldHandleQueryableStoreTarFileCorrectly() throws Exception {
TarArchiveEntry entry = new TarArchiveEntry(tarFile, QUERYABLE_STORE_DATA_FILE_NAME);
entry.setSize(tarFile.length());
doAnswer(
invocation -> {
assertThat(tempWorkDir.listFiles())
.containsExactlyInAnyOrder(
new File(tempWorkDir, "sonia.scm.importexport.SimpleType.xml"),
new File(tempWorkDir, "sonia.scm.importexport.SimpleTypeWithTwoParents.xml")
);
return null;
}
).when(queryableStoreExporter).importStores("42", tempWorkDir);
try (InputStream inputStream = new FileInputStream(tarFile)) {
boolean result = queryableStoreImportStep.handle(entry, importState, inputStream);
assertThat(result).isTrue();
verify(queryableStoreExporter).importStores("42", tempWorkDir);
}
}
}

View File

@@ -0,0 +1,108 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.RepositoryLocationResolver.RepositoryLocationResolverInstance;
import sonia.scm.repository.xml.PathBasedRepositoryLocationResolver;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.util.function.BiConsumer;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
class RemainingQueryableStoreImporterTest {
@TempDir
private File tempDir;
@Mock
private PathBasedRepositoryLocationResolver repositoryLocationResolver;
@Mock
private RepositoryLocationResolverInstance<Path> repositoryLocationResolverInstance;
@Mock
private RepositoryQueryableStoreExporter queryableStoreExporter;
private RemainingQueryableStoreImporter listener;
@BeforeEach
void setUp() throws IOException {
when(repositoryLocationResolver.create(Path.class)).thenReturn(repositoryLocationResolverInstance);
listener = new RemainingQueryableStoreImporter(repositoryLocationResolver, queryableStoreExporter);
File queryableStoreDir = new File(tempDir, "queryable-store-data");
queryableStoreDir.mkdirs();
createXmlFile(new File(queryableStoreDir, "sonia.scm.importexport.SimpleType.xml"));
createXmlFile(new File(queryableStoreDir, "sonia.scm.importexport.SimpleTypeWithTwoParents.xml"));
}
@Test
void shouldImportXmlFilesIfExist() {
Path repoPath = tempDir.toPath();
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
BiConsumer<String, Path> consumer = invocation.getArgument(0);
consumer.accept("test-repo", repoPath);
return null;
}).when(repositoryLocationResolverInstance).forAllLocations(any());
listener.onInitializationCompleted();
verify(queryableStoreExporter).importStores("test-repo", tempDir);
}
@Test
void shouldNotImportIfNoXmlFiles() {
File emptyRepoDir = new File(tempDir, "empty-repo");
emptyRepoDir.mkdirs();
Path emptyRepoPath = emptyRepoDir.toPath();
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
BiConsumer<String, Path> consumer = invocation.getArgument(0);
consumer.accept("empty-repo", emptyRepoPath);
return null;
}).when(repositoryLocationResolverInstance).forAllLocations(any());
listener.onInitializationCompleted();
verify(queryableStoreExporter, never()).importStores(anyString(), any(File.class));
}
private void createXmlFile(File file) throws IOException {
try (FileWriter writer = new FileWriter(file)) {
writer.write("<root><test>data</test></root>");
}
}
}

View File

@@ -0,0 +1,164 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import com.google.common.io.Resources;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.repository.Repository;
import sonia.scm.store.QueryableStoreExtension;
import sonia.scm.store.QueryableStoreFactory;
import sonia.scm.store.StoreMetaDataProvider;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.lenient;
@ExtendWith({QueryableStoreExtension.class, MockitoExtension.class})
@QueryableStoreExtension.QueryableTypes({SimpleType.class, SimpleTypeWithTwoParents.class})
class RepositoryQueryableStoreExporterTest {
@Mock
private StoreMetaDataProvider storeMetaDataProvider;
@BeforeEach
void initMetaDataProvider() {
lenient().when(storeMetaDataProvider.getTypesWithParent(Repository.class)).thenReturn(List.of(SimpleType.class, SimpleTypeWithTwoParents.class));
}
@Nested
class ExportStores {
@Test
void shouldExportSimpleType(QueryableStoreFactory storeFactory, SimpleTypeStoreFactory simpleTypeStoreFactory, @TempDir java.nio.file.Path tempDir) {
simpleTypeStoreFactory.getMutable("23").put("1", new SimpleType("hack"));
simpleTypeStoreFactory.getMutable("42").put("1", new SimpleType("hitchhike"));
simpleTypeStoreFactory.getMutable("42").put("2", new SimpleType("heart of gold"));
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.exportStores("42", tempDir.toFile());
assertThat(tempDir).isNotEmptyDirectory();
}
@Test
void shouldExportTypeWithTwoParents(QueryableStoreFactory storeFactory, SimpleTypeWithTwoParentsStoreFactory simpleTypeStoreFactory, @TempDir java.nio.file.Path tempDir) {
simpleTypeStoreFactory.getMutable("23", "1").put("1", new SimpleTypeWithTwoParents("hack"));
simpleTypeStoreFactory.getMutable("42", "1").put("1", new SimpleTypeWithTwoParents("hitchhike"));
simpleTypeStoreFactory.getMutable("42", "1").put("2", new SimpleTypeWithTwoParents("heart of gold"));
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.exportStores("42", tempDir.toFile());
assertThat(tempDir).isNotEmptyDirectory();
}
}
@Nested
class ImportStores {
private File queryableStoreDir;
@TempDir
private File tempDir;
@BeforeEach
void prepareImportDirectory() throws IOException {
queryableStoreDir = new File(tempDir, "queryable-store-data");
Files.createDirectories(queryableStoreDir.toPath());
}
@Test
void shouldImportSimpleType(QueryableStoreFactory storeFactory, SimpleTypeStoreFactory simpleTypeStoreFactory) throws IOException {
simpleTypeStoreFactory.getMutable("23").put("1", new SimpleType("hack"));
URL url = Resources.getResource("sonia/scm/importexport/SimpleType.xml");
Files.createFile(queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleType.xml"));
Files.writeString(queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleType.xml"), Resources.toString(url, StandardCharsets.UTF_8));
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.importStores("42", tempDir);
assertThat(simpleTypeStoreFactory.getMutable("42").getAll()).hasSize(2);
}
@Test
void shouldImportTypeWithTwoParents(QueryableStoreFactory storeFactory, SimpleTypeWithTwoParentsStoreFactory simpleTypeStoreFactory) throws IOException {
simpleTypeStoreFactory.getMutable("23", "1").put("1", new SimpleTypeWithTwoParents("hack"));
URL url = Resources.getResource("sonia/scm/importexport/SimpleTypeWithTwoParents.xml");
Files.writeString(queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleTypeWithTwoParents.xml"), Resources.toString(url, StandardCharsets.UTF_8));
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.importStores("42", tempDir);
assertThat(simpleTypeStoreFactory.getMutable("42", "1").getAll()).hasSize(2);
}
@Test
void shouldNotImportWhenFileDoesNotExist(QueryableStoreFactory storeFactory, SimpleTypeStoreFactory simpleTypeStoreFactory) {
simpleTypeStoreFactory.getMutable("23").put("1", new SimpleType("hack"));
File nonExistentFile = queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleType.xml").toFile();
assertThat(nonExistentFile).doesNotExist();
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.importStores("42", tempDir);
assertThat(simpleTypeStoreFactory.getMutable("42").getAll()).isEmpty();
}
@Test
void shouldThrowExceptionForMalformedXML(QueryableStoreFactory storeFactory) throws IOException {
Files.writeString(queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleType.xml"), "<malformed><xml></broken>");
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
assertThrows(RuntimeException.class, () -> exporter.importStores("42", tempDir));
}
@Test
void shouldNotImportFromEmptyFile(QueryableStoreFactory storeFactory, SimpleTypeStoreFactory simpleTypeStoreFactory) throws IOException {
simpleTypeStoreFactory.getMutable("42").put("1", new SimpleType("existing data"));
Files.createFile(queryableStoreDir.toPath().resolve("sonia.scm.importexport.SimpleType.xml"));
RepositoryQueryableStoreExporter exporter = new RepositoryQueryableStoreExporter(storeMetaDataProvider, storeFactory);
exporter.importStores("42", tempDir);
SimpleType simpleType = simpleTypeStoreFactory.getMutable("42").get("1");
assertThat(simpleType)
.extracting("someField")
.isEqualTo("existing data");
}
}
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import sonia.scm.repository.Repository;
import sonia.scm.store.QueryableType;
@Data
@NoArgsConstructor
@AllArgsConstructor
@QueryableType(Repository.class)
class SimpleType {
private String someField;
}

View File

@@ -0,0 +1,31 @@
/*
* Copyright (c) 2020 - present Cloudogu GmbH
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Affero General Public License as published by the Free
* Software Foundation, version 3.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
* details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see https://www.gnu.org/licenses/.
*/
package sonia.scm.importexport;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import sonia.scm.repository.Repository;
import sonia.scm.store.QueryableType;
@Data
@NoArgsConstructor
@AllArgsConstructor
@QueryableType({Repository.class, SimpleType.class})
class SimpleTypeWithTwoParents {
private String someField;
}

View File

@@ -27,8 +27,9 @@ import org.mockito.MockitoAnnotations;
import sonia.scm.AbstractTestBase;
import sonia.scm.auditlog.Auditor;
import sonia.scm.plugin.PluginLoader;
import sonia.scm.store.JAXBConfigurationEntryStoreFactory;
import sonia.scm.store.StoreCacheConfigProvider;
import sonia.scm.store.file.JAXBConfigurationEntryStoreFactory;
import sonia.scm.store.file.StoreCacheConfigProvider;
import sonia.scm.store.file.StoreCacheFactory;
import sonia.scm.util.ClassLoaders;
import sonia.scm.util.MockUtil;
@@ -60,7 +61,7 @@ public class DefaultSecuritySystemTest extends AbstractTestBase
public void createSecuritySystem()
{
jaxbConfigurationEntryStoreFactory =
spy(new JAXBConfigurationEntryStoreFactory(contextProvider , repositoryLocationResolver, new UUIDKeyGenerator(), null, new StoreCacheConfigProvider(false)) {});
spy(new JAXBConfigurationEntryStoreFactory(contextProvider , repositoryLocationResolver, new UUIDKeyGenerator(), null, new StoreCacheFactory(new StoreCacheConfigProvider(false))) {});
pluginLoader = mock(PluginLoader.class);
when(pluginLoader.getUberClassLoader()).thenReturn(ClassLoaders.getContextClassLoader(DefaultSecuritySystem.class));

View File

@@ -26,8 +26,9 @@ import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import sonia.scm.SCMContextProvider;
import sonia.scm.store.ConfigurationStoreFactory;
import sonia.scm.store.JAXBConfigurationStoreFactory;
import sonia.scm.store.StoreCacheConfigProvider;
import sonia.scm.store.file.JAXBConfigurationStoreFactory;
import sonia.scm.store.file.StoreCacheConfigProvider;
import sonia.scm.store.file.StoreCacheFactory;
import java.nio.file.Path;
import java.util.Optional;
@@ -47,7 +48,7 @@ class DefaultMigrationStrategyDAOTest {
@BeforeEach
void initStore(@TempDir Path tempDir) {
when(contextProvider.getBaseDirectory()).thenReturn(tempDir.toFile());
storeFactory = new JAXBConfigurationStoreFactory(contextProvider, null, null, emptySet(), new StoreCacheConfigProvider(false));
storeFactory = new JAXBConfigurationStoreFactory(contextProvider, null, null, emptySet(), new StoreCacheFactory(new StoreCacheConfigProvider(false)));
}
@Test

View File

@@ -25,8 +25,9 @@ import org.junit.Rule;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import sonia.scm.NotFoundException;
import sonia.scm.store.JAXBConfigurationStoreFactory;
import sonia.scm.store.StoreCacheConfigProvider;
import sonia.scm.store.file.JAXBConfigurationStoreFactory;
import sonia.scm.store.file.StoreCacheConfigProvider;
import sonia.scm.store.file.StoreCacheFactory;
import sonia.scm.user.xml.XmlUserDAO;
import static java.util.Collections.emptySet;
@@ -153,6 +154,6 @@ public class DefaultUserManagerTest extends UserManagerTestBase {
}
private XmlUserDAO createXmlUserDAO() {
return new XmlUserDAO(new JAXBConfigurationStoreFactory(contextProvider, locationResolver, null, emptySet(), new StoreCacheConfigProvider(false)));
return new XmlUserDAO(new JAXBConfigurationStoreFactory(contextProvider, locationResolver, null, emptySet(), new StoreCacheFactory(new StoreCacheConfigProvider(false))));
}
}

View File

@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Copyright (c) 2020 - present Cloudogu GmbH
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU Affero General Public License as published by the Free
Software Foundation, version 3.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see https://www.gnu.org/licenses/.
-->
<storeExport>
<type>sonia.scm.importexport.SimpleType</type>
<rows>
<parentIds>42</parentIds>
<id>1</id>
<value>{"someField":"hitchhike"}</value>
</rows>
<rows>
<parentIds>42</parentIds>
<id>2</id>
<value>{"someField":"heart of gold"}</value>
</rows>
</storeExport>

View File

@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<!--
Copyright (c) 2020 - present Cloudogu GmbH
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU Affero General Public License as published by the Free
Software Foundation, version 3.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see https://www.gnu.org/licenses/.
-->
<storeExport>
<type>sonia.scm.importexport.SimpleTypeWithTwoParents</type>
<rows>
<parentIds>42</parentIds>
<parentIds>1</parentIds>
<id>1</id>
<value>{"someField":"hitchhike"}</value>
</rows>
<rows>
<parentIds>42</parentIds>
<parentIds>1</parentIds>
<id>2</id>
<value>{"someField":"heart of gold"}</value>
</rows>
</storeExport>