diff --git a/gradle/changelog/change_import_export_order.yaml b/gradle/changelog/change_import_export_order.yaml new file mode 100644 index 0000000000..91e893cbb8 --- /dev/null +++ b/gradle/changelog/change_import_export_order.yaml @@ -0,0 +1,2 @@ +- type: changed + description: Change the order of files inside the repository archive ([#1538](https://github.com/scm-manager/scm-manager/pull/1538)) diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java new file mode 100644 index 0000000000..6f70bc373c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import sonia.scm.ContextEntry; +import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; + +import javax.inject.Inject; +import javax.xml.bind.JAXB; +import java.io.InputStream; + +import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME; + +class EnvironmentCheckStep implements ImportStep { + + @SuppressWarnings("java:S115") // we like this name here + private static final int _1_MB = 1024*1024; + + private final ScmEnvironmentCompatibilityChecker compatibilityChecker; + + @Inject + EnvironmentCheckStep(ScmEnvironmentCompatibilityChecker compatibilityChecker) { + this.compatibilityChecker = compatibilityChecker; + } + + @Override + public boolean handle(TarArchiveEntry environmentEntry, ImportState state, InputStream inputStream) { + if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory()) { + if (environmentEntry.getSize() > _1_MB) { + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. SCM-Manager environment description file 'scm-environment.xml' too big." + ); + } + boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(inputStream), ScmEnvironment.class)); + if (!validEnvironment) { + throw new IncompatibleEnvironmentForImportException(); + } + state.environmentChecked(); + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + if (!state.isEnvironmentChecked()) { + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml'." + ); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java index a891a59ee8..7bf9b41776 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java @@ -88,8 +88,8 @@ public class FullScmRepositoryExporter { ) { writeEnvironmentData(taos); writeMetadata(repository, taos); - writeRepository(service, taos); writeStoreData(repository, taos); + writeRepository(service, taos); taos.finish(); } catch (IOException e) { throw new ExportFailedException( diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java index bd57051f98..b804cf1430 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java @@ -24,56 +24,39 @@ package sonia.scm.importexport; -import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import sonia.scm.ContextEntry; -import sonia.scm.importexport.RepositoryMetadataXmlGenerator.RepositoryMetadata; import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryManager; -import sonia.scm.repository.RepositoryPermission; import sonia.scm.repository.api.ImportFailedException; -import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; -import sonia.scm.repository.api.RepositoryService; -import sonia.scm.repository.api.RepositoryServiceFactory; -import sonia.scm.update.UpdateEngine; import javax.inject.Inject; -import javax.xml.bind.JAXB; import java.io.BufferedInputStream; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME; -import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME; -import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME; +import static java.util.Arrays.stream; public class FullScmRepositoryImporter { - @SuppressWarnings("java:S115") // we like this name here - private static final int _1_MB = 1000000; + private static final Logger LOG = LoggerFactory.getLogger(FullScmRepositoryImporter.class); - private final RepositoryServiceFactory serviceFactory; + private final ImportStep[] importSteps; private final RepositoryManager repositoryManager; - private final ScmEnvironmentCompatibilityChecker compatibilityChecker; - private final TarArchiveRepositoryStoreImporter storeImporter; - private final UpdateEngine updateEngine; @Inject - public FullScmRepositoryImporter(RepositoryServiceFactory serviceFactory, - RepositoryManager repositoryManager, - ScmEnvironmentCompatibilityChecker compatibilityChecker, - TarArchiveRepositoryStoreImporter storeImporter, - UpdateEngine updateEngine) { - this.serviceFactory = serviceFactory; + public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep, + MetadataImportStep metadataImportStep, + StoreImportStep storeImportStep, + RepositoryImportStep repositoryImportStep, + RepositoryManager repositoryManager + ) { this.repositoryManager = repositoryManager; - this.compatibilityChecker = compatibilityChecker; - this.storeImporter = storeImporter; - this.updateEngine = updateEngine; + importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep}; } public Repository importFromStream(Repository repository, InputStream inputStream) { @@ -84,12 +67,7 @@ public class FullScmRepositoryImporter { GzipCompressorInputStream gcis = new GzipCompressorInputStream(bif); TarArchiveInputStream tais = new TarArchiveInputStream(gcis) ) { - checkScmEnvironment(repository, tais); - Collection importedPermissions = processRepositoryMetadata(tais); - Repository createdRepository = importRepositoryFromFile(repository, tais); - importStoresForCreatedRepository(createdRepository, tais); - importRepositoryPermissions(createdRepository, importedPermissions); - return createdRepository; + return run(repository, tais); } } else { throw new ImportFailedException( @@ -106,92 +84,35 @@ public class FullScmRepositoryImporter { } } - private void importRepositoryPermissions(Repository repository, Collection importedPermissions) { - Collection existingPermissions = repository.getPermissions(); - RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger(); - Collection permissions = permissionMerger.merge(existingPermissions, importedPermissions); - repository.setPermissions(permissions); - repositoryManager.modify(repository); - } - - private void importStoresForCreatedRepository(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry metadataEntry = tais.getNextEntry(); - if (metadataEntry.getName().equals(STORE_DATA_FILE_NAME) && !metadataEntry.isDirectory()) { - // Inside the repository tar archive stream is another tar archive. - // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter - storeImporter.importFromTarArchive(repository, tais); - updateEngine.update(repository.getId()); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing metadata file 'scm-metadata.tar' in tar." - ); - } - } - - private Repository importRepositoryFromFile(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry repositoryEntry = tais.getNextEntry(); - if (!repositoryEntry.isDirectory()) { - return repositoryManager.create(repository, repo -> { - try (RepositoryService service = serviceFactory.create(repo)) { - service.getUnbundleCommand().unbundle(new NoneClosingInputStream(tais)); - } catch (IOException e) { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Repository import failed. Could not import repository from file.", - e - ); - } - }); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing repository dump file." - ); - } - } - - private void checkScmEnvironment(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry environmentEntry = tais.getNextEntry(); - if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory() && environmentEntry.getSize() < _1_MB) { - boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(tais), ScmEnvironment.class)); - if (!validEnvironment) { - throw new IncompatibleEnvironmentForImportException(); + private Repository run(Repository repository, TarArchiveInputStream tais) throws IOException { + ImportState state = new ImportState(repositoryManager.create(repository)); + try { + TarArchiveEntry tarArchiveEntry; + while ((tarArchiveEntry = tais.getNextTarEntry()) != null) { + LOG.trace("Trying to handle tar entry '{}'", tarArchiveEntry.getName()); + handle(tais, state, tarArchiveEntry); } - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml' or file too big." - ); - } - } - - private Collection processRepositoryMetadata(TarArchiveInputStream tais) throws IOException { - ArchiveEntry metadataEntry = tais.getNextEntry(); - if (metadataEntry.getName().equals(METADATA_FILE_NAME)) { - RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(tais), RepositoryMetadata.class); - if (metadata != null && metadata.getPermissions() != null) { - return new HashSet<>(metadata.getPermissions()); + stream(importSteps).forEach(step -> step.finish(state)); + return state.getRepository(); + } finally { + stream(importSteps) + .forEach(step -> step.cleanup(state)); + if (!state.success()) { + // Delete the repository if any error occurs during the import + repositoryManager.delete(state.getRepository()); } - return Collections.emptySet(); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.noContext(), - String.format("Invalid import format. Missing SCM-Manager metadata description file %s.", METADATA_FILE_NAME) - ); } } - @SuppressWarnings("java:S4929") // we only want to override close here - static class NoneClosingInputStream extends FilterInputStream { - - NoneClosingInputStream(InputStream delegate) { - super(delegate); - } - - @Override - public void close() { - // Avoid closing stream because JAXB tries to close the stream + private void handle(TarArchiveInputStream tais, ImportState state, TarArchiveEntry currentEntry) { + for (ImportStep step : importSteps) { + if (step.handle(currentEntry, state, tais)) { + return; + } } + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. Unknown file in tar: " + currentEntry.getName() + ); } } diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java b/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java new file mode 100644 index 0000000000..9e61702844 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java @@ -0,0 +1,98 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryPermission; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +class ImportState { + + private Repository repository; + + private boolean environmentChecked; + private boolean storeImported; + private boolean repositoryImported; + + private Collection repositoryPermissions; + + private Path temporaryRepositoryBundle; + + ImportState(Repository repository) { + this.repository = repository; + } + + public void setRepository(Repository repository) { + this.repository = repository; + } + + public Repository getRepository() { + return repository; + } + + public void environmentChecked() { + environmentChecked = true; + } + + public boolean isEnvironmentChecked() { + return environmentChecked; + } + + void setPermissions(Collection repositoryPermissions) { + this.repositoryPermissions = repositoryPermissions; + } + + Collection getRepositoryPermissions() { + return Collections.unmodifiableCollection(repositoryPermissions); + } + + public boolean success() { + return environmentChecked && repositoryImported; + } + + public void storeImported() { + this.storeImported = true; + } + + public boolean isStoreImported() { + return storeImported; + } + + public void setTemporaryRepositoryBundle(Path path) { + this.temporaryRepositoryBundle = path; + } + + public Optional getTemporaryRepositoryBundle() { + return Optional.ofNullable(temporaryRepositoryBundle); + } + + public void repositoryImported() { + this.repositoryImported = true; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java new file mode 100644 index 0000000000..8bb7a9d9b3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java @@ -0,0 +1,39 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; + +import java.io.InputStream; + +interface ImportStep { + boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream); + + default void finish(ImportState state) { + } + + default void cleanup(ImportState state) { + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java new file mode 100644 index 0000000000..3ef642a420 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java @@ -0,0 +1,82 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryManager; +import sonia.scm.repository.RepositoryPermission; + +import javax.inject.Inject; +import javax.xml.bind.JAXB; +import java.io.InputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; + +import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME; + +class MetadataImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(MetadataImportStep.class); + + private final RepositoryManager repositoryManager; + + @Inject + MetadataImportStep(RepositoryManager repositoryManager) { + this.repositoryManager = repositoryManager; + } + + @Override + public boolean handle(TarArchiveEntry metadataEntry, ImportState state, InputStream inputStream) { + if (metadataEntry.getName().equals(METADATA_FILE_NAME)) { + LOG.trace("Importing metadata from tar"); + RepositoryMetadataXmlGenerator.RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(inputStream), RepositoryMetadataXmlGenerator.RepositoryMetadata.class); + if (metadata != null && metadata.getPermissions() != null) { + state.setPermissions(new HashSet<>(metadata.getPermissions())); + } else { + state.setPermissions(Collections.emptySet()); + } + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + LOG.trace("Saving permissions for imported repository"); + importRepositoryPermissions(state.getRepository(), state.getRepositoryPermissions()); + } + + private void importRepositoryPermissions(Repository repository, Collection importedPermissions) { + Collection existingPermissions = repository.getPermissions(); + RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger(); + Collection permissions = permissionMerger.merge(existingPermissions, importedPermissions); + repository.setPermissions(permissions); + repositoryManager.modify(repository); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java b/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java new file mode 100644 index 0000000000..927222c67e --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java @@ -0,0 +1,42 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import java.io.FilterInputStream; +import java.io.InputStream; + +@SuppressWarnings("java:S4929") + // we only want to override close here +class NoneClosingInputStream extends FilterInputStream { + + NoneClosingInputStream(InputStream delegate) { + super(delegate); + } + + @Override + public void close() { + // Avoid closing stream because JAXB tries to close the stream + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java new file mode 100644 index 0000000000..cb0976f773 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java @@ -0,0 +1,129 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.ContextEntry; +import sonia.scm.repository.Repository; +import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.RepositoryService; +import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.repository.work.WorkdirProvider; +import sonia.scm.util.IOUtil; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static sonia.scm.ContextEntry.ContextBuilder.entity; + +class RepositoryImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(RepositoryImportStep.class); + + private final RepositoryServiceFactory serviceFactory; + private final WorkdirProvider workdirProvider; + + @Inject + RepositoryImportStep(RepositoryServiceFactory serviceFactory, WorkdirProvider workdirProvider) { + this.serviceFactory = serviceFactory; + this.workdirProvider = workdirProvider; + } + + @Override + public boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream) { + if (!currentEntry.isDirectory()) { + if (state.isStoreImported()) { + LOG.trace("Importing directly from tar stream (entry '{}')", currentEntry.getName()); + unbundleRepository(state, inputStream); + } else { + LOG.debug("Temporally storing tar entry '{}' in work dir", currentEntry.getName()); + Path path = saveRepositoryDataFromTarArchiveEntry(state.getRepository(), inputStream); + state.setTemporaryRepositoryBundle(path); + } + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + state.getTemporaryRepositoryBundle() + .ifPresent(path -> importFromTemporaryPath(state, path)); + } + + @Override + public void cleanup(ImportState state) { + state.getTemporaryRepositoryBundle() + .ifPresent(path -> IOUtil.deleteSilently(path.getParent().toFile())); + } + + private void importFromTemporaryPath(ImportState state, Path path) { + LOG.debug("Importing repository from temporary location in work dir"); + try { + unbundleRepository(state, Files.newInputStream(path)); + } catch (IOException e) { + throw new ImportFailedException( + entity(state.getRepository()).build(), + "Repository import failed. Could not import repository from temporary file.", + e + ); + } + } + + private void unbundleRepository(ImportState state, InputStream is) { + try (RepositoryService service = serviceFactory.create(state.getRepository())) { + service.getUnbundleCommand().unbundle(new NoneClosingInputStream(is)); + state.repositoryImported(); + } catch (IOException e) { + throw new ImportFailedException( + entity(state.getRepository()).build(), + "Repository import failed. Could not import repository from file.", + e + ); + } + } + + private Path saveRepositoryDataFromTarArchiveEntry(Repository repository, InputStream tais) { + // The order of files inside the repository archives was changed. + // Due to ensure backwards compatible with existing repository archives we save the repository + // and read it again after the stores were imported. + Path repositoryPath = createSavedRepositoryLocation(repository); + try { + Files.copy(tais, repositoryPath); + } catch (IOException e) { + throw new ImportFailedException(ContextEntry.ContextBuilder.noContext(), "Could not temporarilly store repository bundle", e); + } + return repositoryPath; + } + + private Path createSavedRepositoryLocation(Repository repository) { + return workdirProvider.createNewWorkdir(repository.getId()).toPath().resolve("repository"); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java new file mode 100644 index 0000000000..aa53449000 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java @@ -0,0 +1,68 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.Repository; +import sonia.scm.update.UpdateEngine; + +import javax.inject.Inject; +import java.io.InputStream; + +import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME; + +class StoreImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(StoreImportStep.class); + + private final TarArchiveRepositoryStoreImporter storeImporter; + private final UpdateEngine updateEngine; + + @Inject + StoreImportStep(TarArchiveRepositoryStoreImporter storeImporter, UpdateEngine updateEngine) { + this.storeImporter = storeImporter; + this.updateEngine = updateEngine; + } + + @Override + public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) { + if (entry.getName().equals(STORE_DATA_FILE_NAME) && !entry.isDirectory()) { + LOG.trace("Importing store from tar"); + // Inside the repository tar archive stream is another tar archive. + // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter + importStores(state.getRepository(), inputStream); + state.storeImported(); + return true; + } + return false; + } + + private void importStores(Repository repository, InputStream inputStream) { + storeImporter.importFromTarArchive(repository, inputStream); + updateEngine.update(repository.getId()); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java index 86910418f5..0f3b586a43 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java @@ -48,7 +48,7 @@ public class TarArchiveRepositoryStoreImporter { } public void importFromTarArchive(Repository repository, InputStream inputStream) { - try (TarArchiveInputStream tais = new TarArchiveInputStream(inputStream)) { + try (TarArchiveInputStream tais = new NoneClosingTarArchiveInputStream(inputStream)) { ArchiveEntry entry = tais.getNextEntry(); while (entry != null) { String[] entryPathParts = entry.getName().split(File.separator); @@ -101,7 +101,18 @@ public class TarArchiveRepositoryStoreImporter { return entryPathParts.length == 3; } } - // We only support config and data stores yet return false; } + + static class NoneClosingTarArchiveInputStream extends TarArchiveInputStream { + + public NoneClosingTarArchiveInputStream(InputStream is) { + super(is); + } + + @Override + public void close() throws IOException { + // Do not close this input stream + } + } } diff --git a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java index a3d7f2cded..3d227da3c0 100644 --- a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java +++ b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java @@ -24,7 +24,6 @@ package sonia.scm.importexport; -import com.google.common.io.Files; import com.google.common.io.Resources; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -44,15 +43,15 @@ import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; import sonia.scm.repository.api.UnbundleCommandBuilder; +import sonia.scm.repository.work.WorkdirProvider; import sonia.scm.update.UpdateEngine; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; -import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -60,6 +59,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -83,10 +84,25 @@ class FullScmRepositoryImporterTest { private TarArchiveRepositoryStoreImporter storeImporter; @Mock private UpdateEngine updateEngine; + @Mock + private WorkdirProvider workdirProvider; @InjectMocks + private EnvironmentCheckStep environmentCheckStep; + @InjectMocks + private MetadataImportStep metadataImportStep; + @InjectMocks + private StoreImportStep storeImportStep; + @InjectMocks + private RepositoryImportStep repositoryImportStep; + private FullScmRepositoryImporter fullImporter; + @BeforeEach + void initTestObject() { + fullImporter = new FullScmRepositoryImporter(environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep, repositoryManager); + } + @BeforeEach void initRepositoryService() { lenient().when(serviceFactory.create(REPOSITORY)).thenReturn(service); @@ -95,9 +111,9 @@ class FullScmRepositoryImporterTest { @Test void shouldNotImportRepositoryIfFileNotExists(@TempDir Path temp) throws IOException { - File emptyFile = new File(temp.resolve("empty").toString()); - Files.touch(emptyFile); - FileInputStream inputStream = new FileInputStream(emptyFile); + Path emptyFile = temp.resolve("empty"); + Files.createFile(emptyFile); + FileInputStream inputStream = new FileInputStream(emptyFile.toFile()); assertThrows( ImportFailedException.class, () -> fullImporter.importFromStream(REPOSITORY, inputStream) @@ -119,16 +135,15 @@ class FullScmRepositoryImporterTest { class WithValidEnvironment { @BeforeEach - void setUpEnvironment() { + void setUpEnvironment(@TempDir Path temp) { + lenient().when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile()); + when(compatibilityChecker.check(any())).thenReturn(true); - when(repositoryManager.create(eq(REPOSITORY), any())).thenAnswer(invocation -> { - invocation.getArgument(1, Consumer.class).accept(REPOSITORY); - return REPOSITORY; - }); + when(repositoryManager.create(eq(REPOSITORY))).thenReturn(REPOSITORY); } @Test - void shouldImportScmRepositoryArchive() throws IOException { + void shouldImportScmRepositoryArchiveWithWorkDir() throws IOException { InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); Repository repository = fullImporter.importFromStream(REPOSITORY, stream); @@ -138,7 +153,18 @@ class FullScmRepositoryImporterTest { verify(repositoryManager).modify(REPOSITORY); Collection updatedPermissions = REPOSITORY.getPermissions(); assertThat(updatedPermissions).hasSize(2); - verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(FullScmRepositoryImporter.NoneClosingInputStream.class))); + verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class))); + verify(workdirProvider, times(1)).createNewWorkdir(REPOSITORY.getId()); + } + + @Test + void shouldNotExistWorkDirAfterRepositoryImportIsFinished(@TempDir Path temp) throws IOException { + when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile()); + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); + fullImporter.importFromStream(REPOSITORY, stream); + + boolean workDirExists = Files.exists(temp); + assertThat(workDirExists).isFalse(); } @Test @@ -149,5 +175,17 @@ class FullScmRepositoryImporterTest { verify(updateEngine).update(REPOSITORY.getId()); } + + @Test + void shouldImportRepositoryDirectlyWithoutCopyInWorkDir() throws IOException { + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz").openStream(); + Repository repository = fullImporter.importFromStream(REPOSITORY, stream); + + assertThat(repository).isEqualTo(REPOSITORY); + verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class)); + verify(repositoryManager).modify(REPOSITORY); + verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class))); + verify(workdirProvider, never()).createNewWorkdir(REPOSITORY.getId()); + } } } diff --git a/scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz b/scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz new file mode 100644 index 0000000000..22664ff534 Binary files /dev/null and b/scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz differ