From 5ea28a84fc6fa0ac7d35701c4041e07c2d12d606 Mon Sep 17 00:00:00 2001 From: Eduard Heimbuch Date: Mon, 15 Feb 2021 15:43:26 +0100 Subject: [PATCH] Change file order inside repository archive (#1538) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change repository archive order to export/import repository stores before the actual repository. This is done due to import stores before importing the actual repository and firing hooks that may trigger unnecessary computations otherwise. Co-authored-by: René Pfeuffer --- .../changelog/change_import_export_order.yaml | 2 + .../importexport/EnvironmentCheckStep.java | 78 +++++++++ .../FullScmRepositoryExporter.java | 2 +- .../FullScmRepositoryImporter.java | 155 +++++------------- .../sonia/scm/importexport/ImportState.java | 98 +++++++++++ .../sonia/scm/importexport/ImportStep.java | 39 +++++ .../scm/importexport/MetadataImportStep.java | 82 +++++++++ .../importexport/NoneClosingInputStream.java | 42 +++++ .../importexport/RepositoryImportStep.java | 129 +++++++++++++++ .../scm/importexport/StoreImportStep.java | 68 ++++++++ .../TarArchiveRepositoryStoreImporter.java | 15 +- .../FullScmRepositoryImporterTest.java | 64 ++++++-- ...scm-import-stores-before-repository.tar.gz | Bin 0 -> 1646 bytes 13 files changed, 641 insertions(+), 133 deletions(-) create mode 100644 gradle/changelog/change_import_export_order.yaml create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java create mode 100644 scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java create mode 100644 scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz diff --git a/gradle/changelog/change_import_export_order.yaml b/gradle/changelog/change_import_export_order.yaml new file mode 100644 index 0000000000..91e893cbb8 --- /dev/null +++ b/gradle/changelog/change_import_export_order.yaml @@ -0,0 +1,2 @@ +- type: changed + description: Change the order of files inside the repository archive ([#1538](https://github.com/scm-manager/scm-manager/pull/1538)) diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java new file mode 100644 index 0000000000..6f70bc373c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import sonia.scm.ContextEntry; +import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; + +import javax.inject.Inject; +import javax.xml.bind.JAXB; +import java.io.InputStream; + +import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME; + +class EnvironmentCheckStep implements ImportStep { + + @SuppressWarnings("java:S115") // we like this name here + private static final int _1_MB = 1024*1024; + + private final ScmEnvironmentCompatibilityChecker compatibilityChecker; + + @Inject + EnvironmentCheckStep(ScmEnvironmentCompatibilityChecker compatibilityChecker) { + this.compatibilityChecker = compatibilityChecker; + } + + @Override + public boolean handle(TarArchiveEntry environmentEntry, ImportState state, InputStream inputStream) { + if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory()) { + if (environmentEntry.getSize() > _1_MB) { + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. SCM-Manager environment description file 'scm-environment.xml' too big." + ); + } + boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(inputStream), ScmEnvironment.class)); + if (!validEnvironment) { + throw new IncompatibleEnvironmentForImportException(); + } + state.environmentChecked(); + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + if (!state.isEnvironmentChecked()) { + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml'." + ); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java index a891a59ee8..7bf9b41776 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java @@ -88,8 +88,8 @@ public class FullScmRepositoryExporter { ) { writeEnvironmentData(taos); writeMetadata(repository, taos); - writeRepository(service, taos); writeStoreData(repository, taos); + writeRepository(service, taos); taos.finish(); } catch (IOException e) { throw new ExportFailedException( diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java index bd57051f98..b804cf1430 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java @@ -24,56 +24,39 @@ package sonia.scm.importexport; -import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import sonia.scm.ContextEntry; -import sonia.scm.importexport.RepositoryMetadataXmlGenerator.RepositoryMetadata; import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryManager; -import sonia.scm.repository.RepositoryPermission; import sonia.scm.repository.api.ImportFailedException; -import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; -import sonia.scm.repository.api.RepositoryService; -import sonia.scm.repository.api.RepositoryServiceFactory; -import sonia.scm.update.UpdateEngine; import javax.inject.Inject; -import javax.xml.bind.JAXB; import java.io.BufferedInputStream; -import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME; -import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME; -import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME; +import static java.util.Arrays.stream; public class FullScmRepositoryImporter { - @SuppressWarnings("java:S115") // we like this name here - private static final int _1_MB = 1000000; + private static final Logger LOG = LoggerFactory.getLogger(FullScmRepositoryImporter.class); - private final RepositoryServiceFactory serviceFactory; + private final ImportStep[] importSteps; private final RepositoryManager repositoryManager; - private final ScmEnvironmentCompatibilityChecker compatibilityChecker; - private final TarArchiveRepositoryStoreImporter storeImporter; - private final UpdateEngine updateEngine; @Inject - public FullScmRepositoryImporter(RepositoryServiceFactory serviceFactory, - RepositoryManager repositoryManager, - ScmEnvironmentCompatibilityChecker compatibilityChecker, - TarArchiveRepositoryStoreImporter storeImporter, - UpdateEngine updateEngine) { - this.serviceFactory = serviceFactory; + public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep, + MetadataImportStep metadataImportStep, + StoreImportStep storeImportStep, + RepositoryImportStep repositoryImportStep, + RepositoryManager repositoryManager + ) { this.repositoryManager = repositoryManager; - this.compatibilityChecker = compatibilityChecker; - this.storeImporter = storeImporter; - this.updateEngine = updateEngine; + importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep}; } public Repository importFromStream(Repository repository, InputStream inputStream) { @@ -84,12 +67,7 @@ public class FullScmRepositoryImporter { GzipCompressorInputStream gcis = new GzipCompressorInputStream(bif); TarArchiveInputStream tais = new TarArchiveInputStream(gcis) ) { - checkScmEnvironment(repository, tais); - Collection importedPermissions = processRepositoryMetadata(tais); - Repository createdRepository = importRepositoryFromFile(repository, tais); - importStoresForCreatedRepository(createdRepository, tais); - importRepositoryPermissions(createdRepository, importedPermissions); - return createdRepository; + return run(repository, tais); } } else { throw new ImportFailedException( @@ -106,92 +84,35 @@ public class FullScmRepositoryImporter { } } - private void importRepositoryPermissions(Repository repository, Collection importedPermissions) { - Collection existingPermissions = repository.getPermissions(); - RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger(); - Collection permissions = permissionMerger.merge(existingPermissions, importedPermissions); - repository.setPermissions(permissions); - repositoryManager.modify(repository); - } - - private void importStoresForCreatedRepository(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry metadataEntry = tais.getNextEntry(); - if (metadataEntry.getName().equals(STORE_DATA_FILE_NAME) && !metadataEntry.isDirectory()) { - // Inside the repository tar archive stream is another tar archive. - // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter - storeImporter.importFromTarArchive(repository, tais); - updateEngine.update(repository.getId()); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing metadata file 'scm-metadata.tar' in tar." - ); - } - } - - private Repository importRepositoryFromFile(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry repositoryEntry = tais.getNextEntry(); - if (!repositoryEntry.isDirectory()) { - return repositoryManager.create(repository, repo -> { - try (RepositoryService service = serviceFactory.create(repo)) { - service.getUnbundleCommand().unbundle(new NoneClosingInputStream(tais)); - } catch (IOException e) { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Repository import failed. Could not import repository from file.", - e - ); - } - }); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing repository dump file." - ); - } - } - - private void checkScmEnvironment(Repository repository, TarArchiveInputStream tais) throws IOException { - ArchiveEntry environmentEntry = tais.getNextEntry(); - if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory() && environmentEntry.getSize() < _1_MB) { - boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(tais), ScmEnvironment.class)); - if (!validEnvironment) { - throw new IncompatibleEnvironmentForImportException(); + private Repository run(Repository repository, TarArchiveInputStream tais) throws IOException { + ImportState state = new ImportState(repositoryManager.create(repository)); + try { + TarArchiveEntry tarArchiveEntry; + while ((tarArchiveEntry = tais.getNextTarEntry()) != null) { + LOG.trace("Trying to handle tar entry '{}'", tarArchiveEntry.getName()); + handle(tais, state, tarArchiveEntry); } - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.entity(repository).build(), - "Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml' or file too big." - ); - } - } - - private Collection processRepositoryMetadata(TarArchiveInputStream tais) throws IOException { - ArchiveEntry metadataEntry = tais.getNextEntry(); - if (metadataEntry.getName().equals(METADATA_FILE_NAME)) { - RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(tais), RepositoryMetadata.class); - if (metadata != null && metadata.getPermissions() != null) { - return new HashSet<>(metadata.getPermissions()); + stream(importSteps).forEach(step -> step.finish(state)); + return state.getRepository(); + } finally { + stream(importSteps) + .forEach(step -> step.cleanup(state)); + if (!state.success()) { + // Delete the repository if any error occurs during the import + repositoryManager.delete(state.getRepository()); } - return Collections.emptySet(); - } else { - throw new ImportFailedException( - ContextEntry.ContextBuilder.noContext(), - String.format("Invalid import format. Missing SCM-Manager metadata description file %s.", METADATA_FILE_NAME) - ); } } - @SuppressWarnings("java:S4929") // we only want to override close here - static class NoneClosingInputStream extends FilterInputStream { - - NoneClosingInputStream(InputStream delegate) { - super(delegate); - } - - @Override - public void close() { - // Avoid closing stream because JAXB tries to close the stream + private void handle(TarArchiveInputStream tais, ImportState state, TarArchiveEntry currentEntry) { + for (ImportStep step : importSteps) { + if (step.handle(currentEntry, state, tais)) { + return; + } } + throw new ImportFailedException( + ContextEntry.ContextBuilder.entity(state.getRepository()).build(), + "Invalid import format. Unknown file in tar: " + currentEntry.getName() + ); } } diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java b/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java new file mode 100644 index 0000000000..9e61702844 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java @@ -0,0 +1,98 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryPermission; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Optional; + +class ImportState { + + private Repository repository; + + private boolean environmentChecked; + private boolean storeImported; + private boolean repositoryImported; + + private Collection repositoryPermissions; + + private Path temporaryRepositoryBundle; + + ImportState(Repository repository) { + this.repository = repository; + } + + public void setRepository(Repository repository) { + this.repository = repository; + } + + public Repository getRepository() { + return repository; + } + + public void environmentChecked() { + environmentChecked = true; + } + + public boolean isEnvironmentChecked() { + return environmentChecked; + } + + void setPermissions(Collection repositoryPermissions) { + this.repositoryPermissions = repositoryPermissions; + } + + Collection getRepositoryPermissions() { + return Collections.unmodifiableCollection(repositoryPermissions); + } + + public boolean success() { + return environmentChecked && repositoryImported; + } + + public void storeImported() { + this.storeImported = true; + } + + public boolean isStoreImported() { + return storeImported; + } + + public void setTemporaryRepositoryBundle(Path path) { + this.temporaryRepositoryBundle = path; + } + + public Optional getTemporaryRepositoryBundle() { + return Optional.ofNullable(temporaryRepositoryBundle); + } + + public void repositoryImported() { + this.repositoryImported = true; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java new file mode 100644 index 0000000000..8bb7a9d9b3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java @@ -0,0 +1,39 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; + +import java.io.InputStream; + +interface ImportStep { + boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream); + + default void finish(ImportState state) { + } + + default void cleanup(ImportState state) { + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java new file mode 100644 index 0000000000..3ef642a420 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java @@ -0,0 +1,82 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryManager; +import sonia.scm.repository.RepositoryPermission; + +import javax.inject.Inject; +import javax.xml.bind.JAXB; +import java.io.InputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; + +import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME; + +class MetadataImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(MetadataImportStep.class); + + private final RepositoryManager repositoryManager; + + @Inject + MetadataImportStep(RepositoryManager repositoryManager) { + this.repositoryManager = repositoryManager; + } + + @Override + public boolean handle(TarArchiveEntry metadataEntry, ImportState state, InputStream inputStream) { + if (metadataEntry.getName().equals(METADATA_FILE_NAME)) { + LOG.trace("Importing metadata from tar"); + RepositoryMetadataXmlGenerator.RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(inputStream), RepositoryMetadataXmlGenerator.RepositoryMetadata.class); + if (metadata != null && metadata.getPermissions() != null) { + state.setPermissions(new HashSet<>(metadata.getPermissions())); + } else { + state.setPermissions(Collections.emptySet()); + } + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + LOG.trace("Saving permissions for imported repository"); + importRepositoryPermissions(state.getRepository(), state.getRepositoryPermissions()); + } + + private void importRepositoryPermissions(Repository repository, Collection importedPermissions) { + Collection existingPermissions = repository.getPermissions(); + RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger(); + Collection permissions = permissionMerger.merge(existingPermissions, importedPermissions); + repository.setPermissions(permissions); + repositoryManager.modify(repository); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java b/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java new file mode 100644 index 0000000000..927222c67e --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java @@ -0,0 +1,42 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import java.io.FilterInputStream; +import java.io.InputStream; + +@SuppressWarnings("java:S4929") + // we only want to override close here +class NoneClosingInputStream extends FilterInputStream { + + NoneClosingInputStream(InputStream delegate) { + super(delegate); + } + + @Override + public void close() { + // Avoid closing stream because JAXB tries to close the stream + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java new file mode 100644 index 0000000000..cb0976f773 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java @@ -0,0 +1,129 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.ContextEntry; +import sonia.scm.repository.Repository; +import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.RepositoryService; +import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.repository.work.WorkdirProvider; +import sonia.scm.util.IOUtil; + +import javax.inject.Inject; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import static sonia.scm.ContextEntry.ContextBuilder.entity; + +class RepositoryImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(RepositoryImportStep.class); + + private final RepositoryServiceFactory serviceFactory; + private final WorkdirProvider workdirProvider; + + @Inject + RepositoryImportStep(RepositoryServiceFactory serviceFactory, WorkdirProvider workdirProvider) { + this.serviceFactory = serviceFactory; + this.workdirProvider = workdirProvider; + } + + @Override + public boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream) { + if (!currentEntry.isDirectory()) { + if (state.isStoreImported()) { + LOG.trace("Importing directly from tar stream (entry '{}')", currentEntry.getName()); + unbundleRepository(state, inputStream); + } else { + LOG.debug("Temporally storing tar entry '{}' in work dir", currentEntry.getName()); + Path path = saveRepositoryDataFromTarArchiveEntry(state.getRepository(), inputStream); + state.setTemporaryRepositoryBundle(path); + } + return true; + } + return false; + } + + @Override + public void finish(ImportState state) { + state.getTemporaryRepositoryBundle() + .ifPresent(path -> importFromTemporaryPath(state, path)); + } + + @Override + public void cleanup(ImportState state) { + state.getTemporaryRepositoryBundle() + .ifPresent(path -> IOUtil.deleteSilently(path.getParent().toFile())); + } + + private void importFromTemporaryPath(ImportState state, Path path) { + LOG.debug("Importing repository from temporary location in work dir"); + try { + unbundleRepository(state, Files.newInputStream(path)); + } catch (IOException e) { + throw new ImportFailedException( + entity(state.getRepository()).build(), + "Repository import failed. Could not import repository from temporary file.", + e + ); + } + } + + private void unbundleRepository(ImportState state, InputStream is) { + try (RepositoryService service = serviceFactory.create(state.getRepository())) { + service.getUnbundleCommand().unbundle(new NoneClosingInputStream(is)); + state.repositoryImported(); + } catch (IOException e) { + throw new ImportFailedException( + entity(state.getRepository()).build(), + "Repository import failed. Could not import repository from file.", + e + ); + } + } + + private Path saveRepositoryDataFromTarArchiveEntry(Repository repository, InputStream tais) { + // The order of files inside the repository archives was changed. + // Due to ensure backwards compatible with existing repository archives we save the repository + // and read it again after the stores were imported. + Path repositoryPath = createSavedRepositoryLocation(repository); + try { + Files.copy(tais, repositoryPath); + } catch (IOException e) { + throw new ImportFailedException(ContextEntry.ContextBuilder.noContext(), "Could not temporarilly store repository bundle", e); + } + return repositoryPath; + } + + private Path createSavedRepositoryLocation(Repository repository) { + return workdirProvider.createNewWorkdir(repository.getId()).toPath().resolve("repository"); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java b/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java new file mode 100644 index 0000000000..aa53449000 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java @@ -0,0 +1,68 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.importexport; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.Repository; +import sonia.scm.update.UpdateEngine; + +import javax.inject.Inject; +import java.io.InputStream; + +import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME; + +class StoreImportStep implements ImportStep { + + private static final Logger LOG = LoggerFactory.getLogger(StoreImportStep.class); + + private final TarArchiveRepositoryStoreImporter storeImporter; + private final UpdateEngine updateEngine; + + @Inject + StoreImportStep(TarArchiveRepositoryStoreImporter storeImporter, UpdateEngine updateEngine) { + this.storeImporter = storeImporter; + this.updateEngine = updateEngine; + } + + @Override + public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) { + if (entry.getName().equals(STORE_DATA_FILE_NAME) && !entry.isDirectory()) { + LOG.trace("Importing store from tar"); + // Inside the repository tar archive stream is another tar archive. + // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter + importStores(state.getRepository(), inputStream); + state.storeImported(); + return true; + } + return false; + } + + private void importStores(Repository repository, InputStream inputStream) { + storeImporter.importFromTarArchive(repository, inputStream); + updateEngine.update(repository.getId()); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java index 86910418f5..0f3b586a43 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java @@ -48,7 +48,7 @@ public class TarArchiveRepositoryStoreImporter { } public void importFromTarArchive(Repository repository, InputStream inputStream) { - try (TarArchiveInputStream tais = new TarArchiveInputStream(inputStream)) { + try (TarArchiveInputStream tais = new NoneClosingTarArchiveInputStream(inputStream)) { ArchiveEntry entry = tais.getNextEntry(); while (entry != null) { String[] entryPathParts = entry.getName().split(File.separator); @@ -101,7 +101,18 @@ public class TarArchiveRepositoryStoreImporter { return entryPathParts.length == 3; } } - // We only support config and data stores yet return false; } + + static class NoneClosingTarArchiveInputStream extends TarArchiveInputStream { + + public NoneClosingTarArchiveInputStream(InputStream is) { + super(is); + } + + @Override + public void close() throws IOException { + // Do not close this input stream + } + } } diff --git a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java index a3d7f2cded..3d227da3c0 100644 --- a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java +++ b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java @@ -24,7 +24,6 @@ package sonia.scm.importexport; -import com.google.common.io.Files; import com.google.common.io.Resources; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -44,15 +43,15 @@ import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; import sonia.scm.repository.api.UnbundleCommandBuilder; +import sonia.scm.repository.work.WorkdirProvider; import sonia.scm.update.UpdateEngine; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Collection; -import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -60,6 +59,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -83,10 +84,25 @@ class FullScmRepositoryImporterTest { private TarArchiveRepositoryStoreImporter storeImporter; @Mock private UpdateEngine updateEngine; + @Mock + private WorkdirProvider workdirProvider; @InjectMocks + private EnvironmentCheckStep environmentCheckStep; + @InjectMocks + private MetadataImportStep metadataImportStep; + @InjectMocks + private StoreImportStep storeImportStep; + @InjectMocks + private RepositoryImportStep repositoryImportStep; + private FullScmRepositoryImporter fullImporter; + @BeforeEach + void initTestObject() { + fullImporter = new FullScmRepositoryImporter(environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep, repositoryManager); + } + @BeforeEach void initRepositoryService() { lenient().when(serviceFactory.create(REPOSITORY)).thenReturn(service); @@ -95,9 +111,9 @@ class FullScmRepositoryImporterTest { @Test void shouldNotImportRepositoryIfFileNotExists(@TempDir Path temp) throws IOException { - File emptyFile = new File(temp.resolve("empty").toString()); - Files.touch(emptyFile); - FileInputStream inputStream = new FileInputStream(emptyFile); + Path emptyFile = temp.resolve("empty"); + Files.createFile(emptyFile); + FileInputStream inputStream = new FileInputStream(emptyFile.toFile()); assertThrows( ImportFailedException.class, () -> fullImporter.importFromStream(REPOSITORY, inputStream) @@ -119,16 +135,15 @@ class FullScmRepositoryImporterTest { class WithValidEnvironment { @BeforeEach - void setUpEnvironment() { + void setUpEnvironment(@TempDir Path temp) { + lenient().when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile()); + when(compatibilityChecker.check(any())).thenReturn(true); - when(repositoryManager.create(eq(REPOSITORY), any())).thenAnswer(invocation -> { - invocation.getArgument(1, Consumer.class).accept(REPOSITORY); - return REPOSITORY; - }); + when(repositoryManager.create(eq(REPOSITORY))).thenReturn(REPOSITORY); } @Test - void shouldImportScmRepositoryArchive() throws IOException { + void shouldImportScmRepositoryArchiveWithWorkDir() throws IOException { InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); Repository repository = fullImporter.importFromStream(REPOSITORY, stream); @@ -138,7 +153,18 @@ class FullScmRepositoryImporterTest { verify(repositoryManager).modify(REPOSITORY); Collection updatedPermissions = REPOSITORY.getPermissions(); assertThat(updatedPermissions).hasSize(2); - verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(FullScmRepositoryImporter.NoneClosingInputStream.class))); + verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class))); + verify(workdirProvider, times(1)).createNewWorkdir(REPOSITORY.getId()); + } + + @Test + void shouldNotExistWorkDirAfterRepositoryImportIsFinished(@TempDir Path temp) throws IOException { + when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile()); + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); + fullImporter.importFromStream(REPOSITORY, stream); + + boolean workDirExists = Files.exists(temp); + assertThat(workDirExists).isFalse(); } @Test @@ -149,5 +175,17 @@ class FullScmRepositoryImporterTest { verify(updateEngine).update(REPOSITORY.getId()); } + + @Test + void shouldImportRepositoryDirectlyWithoutCopyInWorkDir() throws IOException { + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz").openStream(); + Repository repository = fullImporter.importFromStream(REPOSITORY, stream); + + assertThat(repository).isEqualTo(REPOSITORY); + verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class)); + verify(repositoryManager).modify(REPOSITORY); + verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class))); + verify(workdirProvider, never()).createNewWorkdir(REPOSITORY.getId()); + } } } diff --git a/scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz b/scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..22664ff5340dad5bfef3bdfa94aacfb5de437cf7 GIT binary patch literal 1646 zcmV-!29fz6iwFP!00000|LvLyP!m@e$0Ht9MC&MO6@i#4bi`zjWD|}hh>$20UDz% zjzXBx{ms1W+i&0IdvD+S{@=?^NL;}tlu>exN~s`}6i2IwV01Si1cH1Xj|mzvKSHno zWe0<>ut(iJdOoPjWd>#rI zDn=Ww(h#9W2SS(waXAoMG}k*&ADm>8q59ufWyy3l_RL3n2R4@01U)_+Uz-1^_Sa5_R&ANHgg z!wrIfi4~nvV6;7IBa|!DDhQX z8o8R%iw6pro^>P)LZgtA`XQMn4XH*IsSbR2NhrZ0NCE}S&5v&A6gZRdm|WGUA_!r? z{JGPEQNU2|6fINdDDdb(_4Zsp4U*C&V75U4^l{K5bP@kNecdenxja4{|CYco|D-;= zByuUBi$5Pjty|MUSNad}xn}c^fG|kUza@BX{Wq?dhk7i*A|t4&8ceC%_A4;GT0U;G zMuE;ddV9z;G&D3cI!w0XJ+=Y-|NO5@_D}Br5CM1;#>h3PVd@YPJE|( zPwQzPUB*9*n8iN`!!RBHmO!Wf?o6{JK*##4gDQL(fm0-)#_)FyA8ZH`a1jYX@DKvw z5)dEfBU~KjhvO0q#1H|OfDs-n7DF7lSo>@~x`=-m2F>38!w?K1bo^U_f2aUlcd+UO z*L~@jvX&R}=k}Sa-I`|a^$1%o(zy-ReV>i)$SV2Ui?e@m zSgZ4%TWOOW6W2?IWn2$ml<1PX|E;}YG07vVQkpN}{h1pWG`+?PdNLZG*gKchn{w!3Ikw-)Jk;b!XHVDICN@y1H(@C zX5);acf6b~p9^oeeJN8^RkM;h@rG>y6qiDWQi`_S@l9Qhty*Yltlu=Wi zRCURLn~r-2Gz5%3%CBE@q^ZquO^(aV;JA!~q9NsX*Qi>VIkN*=Hu(JJA!QU4`d+CS z9f(x*8+~u#N{rFhcj>LYX(vLEK0SePrjVsj$w(48yx*QfoOIrRN;-%qPhD;HMGe{awEl;)s}6|yX=pt8Vx zWXU#JSw_yt8yCOi#@QU6u>9cbvXE#_a&q~EpIjVegFQwC#oY^Sn3$En&o{epz>>)$ z>}oG$-fqi_vODMIKB=#4c+-(n{)etY!@>DGkL|k{zq7u|ZQq!f7F(AO+fU7ktFO#i z2(>pP#W`x8AYd*Wp;{>OSdPf2rMV zrA_7XK#&Us4b((ZcOoyA%)t`;hPI+H4=M)D-EVy+G8Mu~(w*$B7#A9c$UL>FqR`^A s>|$Wj&ul+7rw_v+vESLJ<$~T``!qB(G&D3cG_NB60&$_-umCgw0J@bhz5oCK literal 0 HcmV?d00001