Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2026-01-21 23:12:11 +01:00
Code Issues Packages Projects Releases Activity

Change file order inside repository archive (#1538)

Browse Source 
Change repository archive order to export/import repository stores before the actual repository. This is done due to import stores before importing the actual repository and firing hooks that may trigger unnecessary computations otherwise.

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>
This commit is contained in:
Eduard Heimbuch
2021-02-15 15:43:26 +01:00
committed by GitHub
parent 1a2dabeb66
commit 5ea28a84fc
13 changed files with 641 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
gradle/changelog/change_import_export_order.yaml Normal file
View File

@@ -0,0 +1,2 @@
- type: changed
description: Change the order of files inside the repository archive ([#1538](https://github.com/scm-manager/scm-manager/pull/1538))

78
scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java Normal file
View File

@@ -0,0 +1,78 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import sonia.scm.ContextEntry;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.IncompatibleEnvironmentForImportException;
import javax.inject.Inject;
import javax.xml.bind.JAXB;
import java.io.InputStream;
import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME;
class EnvironmentCheckStep implements ImportStep {
@SuppressWarnings("java:S115") // we like this name here
private static final int _1_MB = 1024*1024;
private final ScmEnvironmentCompatibilityChecker compatibilityChecker;
@Inject
EnvironmentCheckStep(ScmEnvironmentCompatibilityChecker compatibilityChecker) {
this.compatibilityChecker = compatibilityChecker;
}
@Override
public boolean handle(TarArchiveEntry environmentEntry, ImportState state, InputStream inputStream) {
if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory()) {
if (environmentEntry.getSize() > _1_MB) {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(state.getRepository()).build(),
"Invalid import format. SCM-Manager environment description file 'scm-environment.xml' too big."
);
}
boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(inputStream), ScmEnvironment.class));
if (!validEnvironment) {
throw new IncompatibleEnvironmentForImportException();
}
state.environmentChecked();
return true;
}
return false;
}
@Override
public void finish(ImportState state) {
if (!state.isEnvironmentChecked()) {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(state.getRepository()).build(),
"Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml'."
);
}
}
}

2
scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryExporter.java
View File

@@ -88,8 +88,8 @@ public class FullScmRepositoryExporter {
) {
writeEnvironmentData(taos);
writeMetadata(repository, taos);
writeRepository(service, taos);
writeStoreData(repository, taos);
writeRepository(service, taos);
taos.finish();
} catch (IOException e) {
throw new ExportFailedException(

155
scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java
View File

@@ -24,56 +24,39 @@
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.importexport.RepositoryMetadataXmlGenerator.RepositoryMetadata;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.IncompatibleEnvironmentForImportException;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.update.UpdateEngine;
import javax.inject.Inject;
import javax.xml.bind.JAXB;
import java.io.BufferedInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME;
import static sonia.scm.importexport.FullScmRepositoryExporter.SCM_ENVIRONMENT_FILE_NAME;
import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME;
import static java.util.Arrays.stream;
public class FullScmRepositoryImporter {
@SuppressWarnings("java:S115") // we like this name here
private static final int _1_MB = 1000000;
private static final Logger LOG = LoggerFactory.getLogger(FullScmRepositoryImporter.class);
private final RepositoryServiceFactory serviceFactory;
private final ImportStep[] importSteps;
private final RepositoryManager repositoryManager;
private final ScmEnvironmentCompatibilityChecker compatibilityChecker;
private final TarArchiveRepositoryStoreImporter storeImporter;
private final UpdateEngine updateEngine;
@Inject
public FullScmRepositoryImporter(RepositoryServiceFactory serviceFactory,
RepositoryManager repositoryManager,
ScmEnvironmentCompatibilityChecker compatibilityChecker,
TarArchiveRepositoryStoreImporter storeImporter,
UpdateEngine updateEngine) {
this.serviceFactory = serviceFactory;
public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
MetadataImportStep metadataImportStep,
StoreImportStep storeImportStep,
RepositoryImportStep repositoryImportStep,
RepositoryManager repositoryManager
) {
this.repositoryManager = repositoryManager;
this.compatibilityChecker = compatibilityChecker;
this.storeImporter = storeImporter;
this.updateEngine = updateEngine;
importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
}
public Repository importFromStream(Repository repository, InputStream inputStream) {
@@ -84,12 +67,7 @@ public class FullScmRepositoryImporter {
GzipCompressorInputStream gcis = new GzipCompressorInputStream(bif);
TarArchiveInputStream tais = new TarArchiveInputStream(gcis)
) {
checkScmEnvironment(repository, tais);
Collection<RepositoryPermission> importedPermissions = processRepositoryMetadata(tais);
Repository createdRepository = importRepositoryFromFile(repository, tais);
importStoresForCreatedRepository(createdRepository, tais);
importRepositoryPermissions(createdRepository, importedPermissions);
return createdRepository;
return run(repository, tais);
}
} else {
throw new ImportFailedException(
@@ -106,92 +84,35 @@ public class FullScmRepositoryImporter {
}
}
private void importRepositoryPermissions(Repository repository, Collection<RepositoryPermission> importedPermissions) {
Collection<RepositoryPermission> existingPermissions = repository.getPermissions();
RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger();
Collection<RepositoryPermission> permissions = permissionMerger.merge(existingPermissions, importedPermissions);
repository.setPermissions(permissions);
repositoryManager.modify(repository);
}
private void importStoresForCreatedRepository(Repository repository, TarArchiveInputStream tais) throws IOException {
ArchiveEntry metadataEntry = tais.getNextEntry();
if (metadataEntry.getName().equals(STORE_DATA_FILE_NAME) && !metadataEntry.isDirectory()) {
// Inside the repository tar archive stream is another tar archive.
// The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
storeImporter.importFromTarArchive(repository, tais);
updateEngine.update(repository.getId());
} else {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(repository).build(),
"Invalid import format. Missing metadata file 'scm-metadata.tar' in tar."
);
}
}
private Repository importRepositoryFromFile(Repository repository, TarArchiveInputStream tais) throws IOException {
ArchiveEntry repositoryEntry = tais.getNextEntry();
if (!repositoryEntry.isDirectory()) {
return repositoryManager.create(repository, repo -> {
try (RepositoryService service = serviceFactory.create(repo)) {
service.getUnbundleCommand().unbundle(new NoneClosingInputStream(tais));
} catch (IOException e) {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(repository).build(),
"Repository import failed. Could not import repository from file.",
e
);
}
});
} else {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(repository).build(),
"Invalid import format. Missing repository dump file."
);
}
}
private void checkScmEnvironment(Repository repository, TarArchiveInputStream tais) throws IOException {
ArchiveEntry environmentEntry = tais.getNextEntry();
if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory() && environmentEntry.getSize() < _1_MB) {
boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(tais), ScmEnvironment.class));
if (!validEnvironment) {
throw new IncompatibleEnvironmentForImportException();
private Repository run(Repository repository, TarArchiveInputStream tais) throws IOException {
ImportState state = new ImportState(repositoryManager.create(repository));
try {
TarArchiveEntry tarArchiveEntry;
while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
LOG.trace("Trying to handle tar entry '{}'", tarArchiveEntry.getName());
handle(tais, state, tarArchiveEntry);
}
} else {
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(repository).build(),
"Invalid import format. Missing SCM-Manager environment description file 'scm-environment.xml' or file too big."
);
}
}
private Collection<RepositoryPermission> processRepositoryMetadata(TarArchiveInputStream tais) throws IOException {
ArchiveEntry metadataEntry = tais.getNextEntry();
if (metadataEntry.getName().equals(METADATA_FILE_NAME)) {
RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(tais), RepositoryMetadata.class);
if (metadata != null && metadata.getPermissions() != null) {
return new HashSet<>(metadata.getPermissions());
stream(importSteps).forEach(step -> step.finish(state));
return state.getRepository();
} finally {
stream(importSteps)
.forEach(step -> step.cleanup(state));
if (!state.success()) {
// Delete the repository if any error occurs during the import
repositoryManager.delete(state.getRepository());
}
return Collections.emptySet();
} else {
throw new ImportFailedException(
ContextEntry.ContextBuilder.noContext(),
String.format("Invalid import format. Missing SCM-Manager metadata description file %s.", METADATA_FILE_NAME)
);
}
}
@SuppressWarnings("java:S4929") // we only want to override close here
static class NoneClosingInputStream extends FilterInputStream {
NoneClosingInputStream(InputStream delegate) {
super(delegate);
}
@Override
public void close() {
// Avoid closing stream because JAXB tries to close the stream
private void handle(TarArchiveInputStream tais, ImportState state, TarArchiveEntry currentEntry) {
for (ImportStep step : importSteps) {
if (step.handle(currentEntry, state, tais)) {
return;
}
}
throw new ImportFailedException(
ContextEntry.ContextBuilder.entity(state.getRepository()).build(),
"Invalid import format. Unknown file in tar: " + currentEntry.getName()
);
}
}

98
scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java Normal file
View File

@@ -0,0 +1,98 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryPermission;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
class ImportState {
private Repository repository;
private boolean environmentChecked;
private boolean storeImported;
private boolean repositoryImported;
private Collection<RepositoryPermission> repositoryPermissions;
private Path temporaryRepositoryBundle;
ImportState(Repository repository) {
this.repository = repository;
}
public void setRepository(Repository repository) {
this.repository = repository;
}
public Repository getRepository() {
return repository;
}
public void environmentChecked() {
environmentChecked = true;
}
public boolean isEnvironmentChecked() {
return environmentChecked;
}
void setPermissions(Collection<RepositoryPermission> repositoryPermissions) {
this.repositoryPermissions = repositoryPermissions;
}
Collection<RepositoryPermission> getRepositoryPermissions() {
return Collections.unmodifiableCollection(repositoryPermissions);
}
public boolean success() {
return environmentChecked && repositoryImported;
}
public void storeImported() {
this.storeImported = true;
}
public boolean isStoreImported() {
return storeImported;
}
public void setTemporaryRepositoryBundle(Path path) {
this.temporaryRepositoryBundle = path;
}
public Optional<Path> getTemporaryRepositoryBundle() {
return Optional.ofNullable(temporaryRepositoryBundle);
}
public void repositoryImported() {
this.repositoryImported = true;
}
}

39
scm-webapp/src/main/java/sonia/scm/importexport/ImportStep.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import java.io.InputStream;
interface ImportStep {
boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream);
default void finish(ImportState state) {
}
default void cleanup(ImportState state) {
}
}

82
scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java Normal file
View File

@@ -0,0 +1,82 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.repository.RepositoryPermission;
import javax.inject.Inject;
import javax.xml.bind.JAXB;
import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import static sonia.scm.importexport.FullScmRepositoryExporter.METADATA_FILE_NAME;
class MetadataImportStep implements ImportStep {
private static final Logger LOG = LoggerFactory.getLogger(MetadataImportStep.class);
private final RepositoryManager repositoryManager;
@Inject
MetadataImportStep(RepositoryManager repositoryManager) {
this.repositoryManager = repositoryManager;
}
@Override
public boolean handle(TarArchiveEntry metadataEntry, ImportState state, InputStream inputStream) {
if (metadataEntry.getName().equals(METADATA_FILE_NAME)) {
LOG.trace("Importing metadata from tar");
RepositoryMetadataXmlGenerator.RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(inputStream), RepositoryMetadataXmlGenerator.RepositoryMetadata.class);
if (metadata != null && metadata.getPermissions() != null) {
state.setPermissions(new HashSet<>(metadata.getPermissions()));
} else {
state.setPermissions(Collections.emptySet());
}
return true;
}
return false;
}
@Override
public void finish(ImportState state) {
LOG.trace("Saving permissions for imported repository");
importRepositoryPermissions(state.getRepository(), state.getRepositoryPermissions());
}
private void importRepositoryPermissions(Repository repository, Collection<RepositoryPermission> importedPermissions) {
Collection<RepositoryPermission> existingPermissions = repository.getPermissions();
RepositoryImportPermissionMerger permissionMerger = new RepositoryImportPermissionMerger();
Collection<RepositoryPermission> permissions = permissionMerger.merge(existingPermissions, importedPermissions);
repository.setPermissions(permissions);
repositoryManager.modify(repository);
}
}

42
scm-webapp/src/main/java/sonia/scm/importexport/NoneClosingInputStream.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import java.io.FilterInputStream;
import java.io.InputStream;
@SuppressWarnings("java:S4929")
// we only want to override close here
class NoneClosingInputStream extends FilterInputStream {
NoneClosingInputStream(InputStream delegate) {
super(delegate);
}
@Override
public void close() {
// Avoid closing stream because JAXB tries to close the stream
}
}

129
scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java Normal file
View File

@@ -0,0 +1,129 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.ContextEntry;
import sonia.scm.repository.Repository;
import sonia.scm.repository.api.ImportFailedException;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.repository.work.WorkdirProvider;
import sonia.scm.util.IOUtil;
import javax.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import static sonia.scm.ContextEntry.ContextBuilder.entity;
class RepositoryImportStep implements ImportStep {
private static final Logger LOG = LoggerFactory.getLogger(RepositoryImportStep.class);
private final RepositoryServiceFactory serviceFactory;
private final WorkdirProvider workdirProvider;
@Inject
RepositoryImportStep(RepositoryServiceFactory serviceFactory, WorkdirProvider workdirProvider) {
this.serviceFactory = serviceFactory;
this.workdirProvider = workdirProvider;
}
@Override
public boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream) {
if (!currentEntry.isDirectory()) {
if (state.isStoreImported()) {
LOG.trace("Importing directly from tar stream (entry '{}')", currentEntry.getName());
unbundleRepository(state, inputStream);
} else {
LOG.debug("Temporally storing tar entry '{}' in work dir", currentEntry.getName());
Path path = saveRepositoryDataFromTarArchiveEntry(state.getRepository(), inputStream);
state.setTemporaryRepositoryBundle(path);
}
return true;
}
return false;
}
@Override
public void finish(ImportState state) {
state.getTemporaryRepositoryBundle()
.ifPresent(path -> importFromTemporaryPath(state, path));
}
@Override
public void cleanup(ImportState state) {
state.getTemporaryRepositoryBundle()
.ifPresent(path -> IOUtil.deleteSilently(path.getParent().toFile()));
}
private void importFromTemporaryPath(ImportState state, Path path) {
LOG.debug("Importing repository from temporary location in work dir");
try {
unbundleRepository(state, Files.newInputStream(path));
} catch (IOException e) {
throw new ImportFailedException(
entity(state.getRepository()).build(),
"Repository import failed. Could not import repository from temporary file.",
e
);
}
}
private void unbundleRepository(ImportState state, InputStream is) {
try (RepositoryService service = serviceFactory.create(state.getRepository())) {
service.getUnbundleCommand().unbundle(new NoneClosingInputStream(is));
state.repositoryImported();
} catch (IOException e) {
throw new ImportFailedException(
entity(state.getRepository()).build(),
"Repository import failed. Could not import repository from file.",
e
);
}
}
private Path saveRepositoryDataFromTarArchiveEntry(Repository repository, InputStream tais) {
// The order of files inside the repository archives was changed.
// Due to ensure backwards compatible with existing repository archives we save the repository
// and read it again after the stores were imported.
Path repositoryPath = createSavedRepositoryLocation(repository);
try {
Files.copy(tais, repositoryPath);
} catch (IOException e) {
throw new ImportFailedException(ContextEntry.ContextBuilder.noContext(), "Could not temporarilly store repository bundle", e);
}
return repositoryPath;
}
private Path createSavedRepositoryLocation(Repository repository) {
return workdirProvider.createNewWorkdir(repository.getId()).toPath().resolve("repository");
}
}

68
scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java Normal file
View File

@@ -0,0 +1,68 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.importexport;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.repository.Repository;
import sonia.scm.update.UpdateEngine;
import javax.inject.Inject;
import java.io.InputStream;
import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_NAME;
class StoreImportStep implements ImportStep {
private static final Logger LOG = LoggerFactory.getLogger(StoreImportStep.class);
private final TarArchiveRepositoryStoreImporter storeImporter;
private final UpdateEngine updateEngine;
@Inject
StoreImportStep(TarArchiveRepositoryStoreImporter storeImporter, UpdateEngine updateEngine) {
this.storeImporter = storeImporter;
this.updateEngine = updateEngine;
}
@Override
public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) {
if (entry.getName().equals(STORE_DATA_FILE_NAME) && !entry.isDirectory()) {
LOG.trace("Importing store from tar");
// Inside the repository tar archive stream is another tar archive.
// The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
importStores(state.getRepository(), inputStream);
state.storeImported();
return true;
}
return false;
}
private void importStores(Repository repository, InputStream inputStream) {
storeImporter.importFromTarArchive(repository, inputStream);
updateEngine.update(repository.getId());
}
}

15
scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java
View File

@@ -48,7 +48,7 @@ public class TarArchiveRepositoryStoreImporter {
}
public void importFromTarArchive(Repository repository, InputStream inputStream) {
try (TarArchiveInputStream tais = new TarArchiveInputStream(inputStream)) {
try (TarArchiveInputStream tais = new NoneClosingTarArchiveInputStream(inputStream)) {
ArchiveEntry entry = tais.getNextEntry();
while (entry != null) {
String[] entryPathParts = entry.getName().split(File.separator);
@@ -101,7 +101,18 @@ public class TarArchiveRepositoryStoreImporter {
return entryPathParts.length == 3;
}
}
// We only support config and data stores yet
return false;
}
static class NoneClosingTarArchiveInputStream extends TarArchiveInputStream {
public NoneClosingTarArchiveInputStream(InputStream is) {
super(is);
}
@Override
public void close() throws IOException {
// Do not close this input stream
}
}
}

64
scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java
View File

@@ -24,7 +24,6 @@
package sonia.scm.importexport;
import com.google.common.io.Files;
import com.google.common.io.Resources;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
@@ -44,15 +43,15 @@ import sonia.scm.repository.api.IncompatibleEnvironmentForImportException;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.repository.api.UnbundleCommandBuilder;
import sonia.scm.repository.work.WorkdirProvider;
import sonia.scm.update.UpdateEngine;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.function.Consumer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -60,6 +59,8 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -83,10 +84,25 @@ class FullScmRepositoryImporterTest {
private TarArchiveRepositoryStoreImporter storeImporter;
@Mock
private UpdateEngine updateEngine;
@Mock
private WorkdirProvider workdirProvider;
@InjectMocks
private EnvironmentCheckStep environmentCheckStep;
@InjectMocks
private MetadataImportStep metadataImportStep;
@InjectMocks
private StoreImportStep storeImportStep;
@InjectMocks
private RepositoryImportStep repositoryImportStep;
private FullScmRepositoryImporter fullImporter;
@BeforeEach
void initTestObject() {
fullImporter = new FullScmRepositoryImporter(environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep, repositoryManager);
}
@BeforeEach
void initRepositoryService() {
lenient().when(serviceFactory.create(REPOSITORY)).thenReturn(service);
@@ -95,9 +111,9 @@ class FullScmRepositoryImporterTest {
@Test
void shouldNotImportRepositoryIfFileNotExists(@TempDir Path temp) throws IOException {
File emptyFile = new File(temp.resolve("empty").toString());
Files.touch(emptyFile);
FileInputStream inputStream = new FileInputStream(emptyFile);
Path emptyFile = temp.resolve("empty");
Files.createFile(emptyFile);
FileInputStream inputStream = new FileInputStream(emptyFile.toFile());
assertThrows(
ImportFailedException.class,
() -> fullImporter.importFromStream(REPOSITORY, inputStream)
@@ -119,16 +135,15 @@ class FullScmRepositoryImporterTest {
class WithValidEnvironment {
@BeforeEach
void setUpEnvironment() {
void setUpEnvironment(@TempDir Path temp) {
lenient().when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile());
when(compatibilityChecker.check(any())).thenReturn(true);
when(repositoryManager.create(eq(REPOSITORY), any())).thenAnswer(invocation -> {
invocation.getArgument(1, Consumer.class).accept(REPOSITORY);
return REPOSITORY;
});
when(repositoryManager.create(eq(REPOSITORY))).thenReturn(REPOSITORY);
}
@Test
void shouldImportScmRepositoryArchive() throws IOException {
void shouldImportScmRepositoryArchiveWithWorkDir() throws IOException {
InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream();
Repository repository = fullImporter.importFromStream(REPOSITORY, stream);
@@ -138,7 +153,18 @@ class FullScmRepositoryImporterTest {
verify(repositoryManager).modify(REPOSITORY);
Collection<RepositoryPermission> updatedPermissions = REPOSITORY.getPermissions();
assertThat(updatedPermissions).hasSize(2);
verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(FullScmRepositoryImporter.NoneClosingInputStream.class)));
verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class)));
verify(workdirProvider, times(1)).createNewWorkdir(REPOSITORY.getId());
}
@Test
void shouldNotExistWorkDirAfterRepositoryImportIsFinished(@TempDir Path temp) throws IOException {
when(workdirProvider.createNewWorkdir(REPOSITORY.getId())).thenReturn(temp.toFile());
InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream();
fullImporter.importFromStream(REPOSITORY, stream);
boolean workDirExists = Files.exists(temp);
assertThat(workDirExists).isFalse();
}
@Test
@@ -149,5 +175,17 @@ class FullScmRepositoryImporterTest {
verify(updateEngine).update(REPOSITORY.getId());
}
@Test
void shouldImportRepositoryDirectlyWithoutCopyInWorkDir() throws IOException {
InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz").openStream();
Repository repository = fullImporter.importFromStream(REPOSITORY, stream);
assertThat(repository).isEqualTo(REPOSITORY);
verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class));
verify(repositoryManager).modify(REPOSITORY);
verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class)));
verify(workdirProvider, never()).createNewWorkdir(REPOSITORY.getId());
}
}
}

BIN
scm-webapp/src/test/resources/sonia/scm/repository/import/scm-import-stores-before-repository.tar.gz Normal file
View File

Binary file not shown.