From e0d2630a083f616904a3862e2124ac2ba34e6a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Pfeuffer?= Date: Wed, 10 Feb 2021 08:12:48 +0100 Subject: [PATCH] Feature repository specific data migration (#1526) This adds a new migration mechanism for repository data. Instead of using UpdateSteps for all data migrations, repository data shall from now on be implemented with RepositoryUpdateSteps. The general logic stays the same. Executed updates are stored with the repository. Doing this, we can now execute updates on imported repositories without touching other data. This way we can import repositories even though they were exported with older versions of SCM-Manager or a plugin. --- docs/de/user/repo/index.md | 4 +- docs/de/user/repo/settings.md | 5 +- docs/en/user/repo/index.md | 4 +- docs/en/user/repo/settings.md | 5 +- .../repository_specific_migration.yaml | 2 + .../migration/RepositoryUpdateContext.java | 46 ++++ .../scm/migration/RepositoryUpdateStep.java | 52 +++++ .../java/sonia/scm/migration/UpdateStep.java | 83 +------ .../sonia/scm/migration/UpdateStepTarget.java | 47 ++++ .../sonia/scm/migration/package-info.java | 96 ++++++++ ...mpatibleEnvironmentForImportException.java | 48 ++++ .../scm/update/RepositoryUpdateIterator.java | 33 +++ ...nMemoryConfigurationEntryStoreFactory.java | 7 +- scm-ui/ui-webapp/public/locales/de/repos.json | 4 +- scm-ui/ui-webapp/public/locales/en/repos.json | 4 +- .../FullScmRepositoryImporter.java | 14 +- .../ScmEnvironmentCompatibilityChecker.java | 16 +- .../lifecycle/modules/UpdateStepModule.java | 8 +- .../java/sonia/scm/update/UpdateEngine.java | 206 ++++++++++++++++-- .../main/resources/locales/de/plugins.json | 4 + .../main/resources/locales/en/plugins.json | 4 + .../FullScmRepositoryImporterTest.java | 70 ++++-- ...cmEnvironmentCompatibilityCheckerTest.java | 18 +- .../sonia/scm/update/UpdateEngineTest.java | 109 ++++++++- 24 files changed, 743 insertions(+), 146 deletions(-) create mode 100644 gradle/changelog/repository_specific_migration.yaml create mode 100644 scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateContext.java create mode 100644 scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateStep.java create mode 100644 scm-core/src/main/java/sonia/scm/migration/UpdateStepTarget.java create mode 100644 scm-core/src/main/java/sonia/scm/migration/package-info.java create mode 100644 scm-core/src/main/java/sonia/scm/repository/api/IncompatibleEnvironmentForImportException.java diff --git a/docs/de/user/repo/index.md b/docs/de/user/repo/index.md index 368a7880ff..d047509152 100644 --- a/docs/de/user/repo/index.md +++ b/docs/de/user/repo/index.md @@ -45,7 +45,9 @@ Wechseln Sie über den Schalter oben rechts auf die Importseite und füllen Sie Das gewählte Repository wird zum SCM-Manager hinzugefügt und sämtliche Repository Daten inklusive aller Branches und Tags werden importiert. Zusätzlich zum normalen Repository Import gibt es die Möglichkeit ein Repository Archiv mit Metadaten zu importieren. -Dieses Repository Archiv muss von einem anderen SCM-Manager exportiert worden sein und wird beim Importieren auf Kompatibilität der Daten überprüft. +Dieses Repository Archiv muss von einem anderen SCM-Manager exportiert worden sein und wird vor dem Import auf +Kompatibilität der Daten überprüft (der SCM-Manager und alle installierten Plugins müssen mindestens die Version des +exportierenden Systems haben). ![Repository importieren](assets/import-repository.png) diff --git a/docs/de/user/repo/settings.md b/docs/de/user/repo/settings.md index 81724c6ada..6af7344034 100644 --- a/docs/de/user/repo/settings.md +++ b/docs/de/user/repo/settings.md @@ -23,7 +23,10 @@ Das Ausgabeformat des Repository kann über die angebotenen Optionen verändert * `Standard`: Werden keine Optionen ausgewählt, wird das Repository im Standard Format exportiert. Git und Mercurial werden dabei als `Tar Archiv` exportiert und Subversion nutzt das `Dump` Format. * `Komprimieren`: Das Ausgabeformat wird zusätzlich mit `GZip` komprimiert, um die Dateigröße zu verringern. -* `Mit Metadaten`: Statt dem Standard-Format wird ein Repository Archiv exportiert, welches außer dem Repository noch weitere Metadaten enthält. +* `Mit Metadaten`: Statt dem Standard-Format wird ein Repository Archiv exportiert, welches außer dem Repository noch + weitere Metadaten enthält. Für diesen Export sollte sichergestellt werden, dass alle installierten Plugins aktuell sind. + Ein Import eines so exportierten Repositories ist nur in einem SCM-Manager mit derselben oder einer neueren Version + möglich. Dieses gilt ebenso für alle installierten Plugins. ![Repository-Settings-General-Git](assets/repository-settings-general-git.png) diff --git a/docs/en/user/repo/index.md b/docs/en/user/repo/index.md index a0734243d6..be634c9b64 100644 --- a/docs/en/user/repo/index.md +++ b/docs/en/user/repo/index.md @@ -43,7 +43,9 @@ Just use the Switcher on top right to navigate to the import page and fill the i Your repository will be added to SCM-Manager and all repository data including all branches and tags will be imported. In addition to the normal repository import, there is the possibility to import a repository archive with metadata. -This repository archive must have been exported from another SCM manager and is checked for data compatibility during import. +This repository archive must have been exported from another SCM-Manager and is checked for data compatibility before +import (the SCM-Manager and all its installed plugins have to have at least the versions of the system the export has +been created on). ![Import Repository](assets/import-repository.png) diff --git a/docs/en/user/repo/settings.md b/docs/en/user/repo/settings.md index cc2a8504bd..10c6b874a8 100644 --- a/docs/en/user/repo/settings.md +++ b/docs/en/user/repo/settings.md @@ -21,7 +21,10 @@ The output format of the repository can be changed via the offered options: * `Standard`: If no options are selected, the repository will be exported in the standard format. Git and Mercurial are exported as `Tar archive` and Subversion uses the `Dump` format. * `Compress`: The output format is additionally compressed with `GZip` to reduce the file size. -* `With metadata`: Instead of the standard format a repository archive is exported, which contains additional metadata besides the repository. +* `With metadata`: Instead of the standard format, a repository archive is exported, which contains additional metadata + besides the repository. When you use this, please make sure all installed plugins are up to date. An import of + such an export is possible only in an SCM-Manager with the same or a newer version. The same is valid for all + installed plugins. ![Repository-Settings-General-Git](assets/repository-settings-general-git.png) diff --git a/gradle/changelog/repository_specific_migration.yaml b/gradle/changelog/repository_specific_migration.yaml new file mode 100644 index 0000000000..13fa7b8711 --- /dev/null +++ b/gradle/changelog/repository_specific_migration.yaml @@ -0,0 +1,2 @@ +- type: added + description: Repository data can be migrated independently to enable the import of dumps from older versions ([#1526](https://github.com/scm-manager/scm-manager/pull/1526)) diff --git a/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateContext.java b/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateContext.java new file mode 100644 index 0000000000..e257f15440 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateContext.java @@ -0,0 +1,46 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.migration; + +/** + * Data for the repository, whose data that should be migrated. + * + * @since 2.14.0 + */ +public final class RepositoryUpdateContext { + + private final String repositoryId; + + public RepositoryUpdateContext(String repositoryId) { + this.repositoryId = repositoryId; + } + + /** + * The id of the repository, whose data should be migrated. + */ + public String getRepositoryId() { + return repositoryId; + } +} diff --git a/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateStep.java b/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateStep.java new file mode 100644 index 0000000000..2b586547bf --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/migration/RepositoryUpdateStep.java @@ -0,0 +1,52 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.migration; + +import sonia.scm.plugin.ExtensionPoint; + +/** + * This is the main interface for "repository specific" data migration/update. Using this interface, SCM-Manager + * provides the possibility to change data structures between versions for a given type of data. This class should be + * used only for repository specific data (eg. the store is created with a call of forRepository in a store + * factory. To migrate global data, use a {@link UpdateStep}. + *

For information about {@link #getAffectedDataType()} and {@link #getTargetVersion()}, see the package + * documentation.

+ * + * @see sonia.scm.migration + * + * @since 2.14.0 + */ +@ExtensionPoint +public interface RepositoryUpdateStep extends UpdateStepTarget { + /** + * Implement this to update the data to the new version for a specific repository. If any {@link Exception} is thrown, + * SCM-Manager will not start up. + * + * @param repositoryUpdateContext A context providing specifics about the repository, whose data should be migrated + * (eg. it id). + */ + @SuppressWarnings("java:S112") // we suppress this one, because an implementation should feel free to throw any exception it deems necessary + void doUpdate(RepositoryUpdateContext repositoryUpdateContext) throws Exception; +} diff --git a/scm-core/src/main/java/sonia/scm/migration/UpdateStep.java b/scm-core/src/main/java/sonia/scm/migration/UpdateStep.java index 79ad1c6d19..5a599ed33b 100644 --- a/scm-core/src/main/java/sonia/scm/migration/UpdateStep.java +++ b/scm-core/src/main/java/sonia/scm/migration/UpdateStep.java @@ -25,85 +25,24 @@ package sonia.scm.migration; import sonia.scm.plugin.ExtensionPoint; -import sonia.scm.version.Version; /** - * This is the main interface for data migration/update. Using this interface, SCM-Manager provides the possibility to - * change data structures between versions for a given type of data. - *

The data type can be an arbitrary string, but it is considered a best practice to use a qualified name, for - * example - *

- *

- *

The version is unrelated to other versions and therefore can be chosen freely, so that a data type can be updated - * without in various ways independent of other data types or the official version of the plugin or the core. - * A coordination between different data types and their versions is only necessary, when update steps of different data - * types rely on each other. If a update step of data type A has to run before another step for data type - * B, the version number of the second step has to be greater in regards to {@link Version#compareTo(Version)}. - *

- *

The algorithm looks something like this:
- * Whenever the SCM-Manager starts, - *

- *

- *

Mind that an implementation of this class has to be annotated with {@link sonia.scm.plugin.Extension}, so that the - * step will be found.

+ * This is the main interface for "global" data migration/update. Using this interface, SCM-Manager provides the + * possibility to change data structures between versions for a given type of data. This class should be used only + * for global data, that is not repository specific (eg. the store is created without a call of + * forRepository in a store factory. To migrate repository specific data, use a + * {@link RepositoryUpdateStep}. + *

For information about {@link #getAffectedDataType()} and {@link #getTargetVersion()}, see the package + * documentation.

+ * + * @see sonia.scm.migration */ @ExtensionPoint -public interface UpdateStep { +public interface UpdateStep extends UpdateStepTarget { /** * Implement this to update the data to the new version. If any {@link Exception} is thrown, SCM-Manager will not * start up. */ + @SuppressWarnings("java:S112") // we suppress this one, because an implementation should feel free to throw any exception it deems necessary void doUpdate() throws Exception; - - /** - * Declares the new version of the data type given by {@link #getAffectedDataType()}. A update step will only be - * executed, when this version is bigger than the last recorded version for its data type according to - * {@link Version#compareTo(Version)} - */ - Version getTargetVersion(); - - /** - * Declares the data type this update step will take care of. This should be a qualified name, like - * com.example.myPlugin.configuration. - */ - String getAffectedDataType(); } diff --git a/scm-core/src/main/java/sonia/scm/migration/UpdateStepTarget.java b/scm-core/src/main/java/sonia/scm/migration/UpdateStepTarget.java new file mode 100644 index 0000000000..38f5d64c09 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/migration/UpdateStepTarget.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.migration; + +import sonia.scm.version.Version; + +/** + * Base class for update steps, telling the target version and the affected data type. + * + * @since 2.14.0 + */ +public interface UpdateStepTarget { + /** + * Declares the new version of the data type given by {@link #getAffectedDataType()}. A update step will only be + * executed, when this version is bigger than the last recorded version for its data type according to + * {@link Version#compareTo(Version)} + */ + Version getTargetVersion(); + + /** + * Declares the data type this update step will take care of. This should be a qualified name, like + * com.example.myPlugin.configuration. + */ + String getAffectedDataType(); +} diff --git a/scm-core/src/main/java/sonia/scm/migration/package-info.java b/scm-core/src/main/java/sonia/scm/migration/package-info.java new file mode 100644 index 0000000000..0717946b6d --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/migration/package-info.java @@ -0,0 +1,96 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * This package holds classes for data migrations between different versions of SCM-Manager or its plugins. The central + * classes for this are {@link sonia.scm.migration.UpdateStep} and {@link sonia.scm.migration.RepositoryUpdateStep}. + * Implementations of these classes tell, what data they migrate + * ({@link sonia.scm.migration.UpdateStepTarget#getAffectedDataType()}) and to what version this data type is migrated + * ({@link sonia.scm.migration.UpdateStepTarget#getTargetVersion()}). + *

The data type provided by {@link sonia.scm.migration.UpdateStepTarget#getAffectedDataType()} can be an arbitrary + * string, but it is considered a best practice to use a qualified name, for example + *

+ *

+ *

The version provided by {@link sonia.scm.migration.UpdateStepTarget#getTargetVersion()} is unrelated to other + * versions and therefore can be chosen freely, so that a data type can be updated without in various ways independent + * of other data types or the official version of the plugin or the core. + * A coordination between different data types and their versions is only necessary, when update steps of different data + * types rely on each other. If a update step of data type A has to run before another step for data type + * B, the version number of the second step has to be greater in regards to + * {@link sonia.scm.version.Version#compareTo(Version)}. + *

+ *

The algorithm looks something like this:
+ * Whenever the SCM-Manager starts, + *

+ *

+ *

Mind that an implementation of this class has to be annotated with {@link sonia.scm.plugin.Extension}, so that the + * step will be found.

+ */ +package sonia.scm.migration; + +import sonia.scm.version.Version; diff --git a/scm-core/src/main/java/sonia/scm/repository/api/IncompatibleEnvironmentForImportException.java b/scm-core/src/main/java/sonia/scm/repository/api/IncompatibleEnvironmentForImportException.java new file mode 100644 index 0000000000..36692f3cf1 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/repository/api/IncompatibleEnvironmentForImportException.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.repository.api; + +import sonia.scm.ExceptionWithContext; + +import static sonia.scm.ContextEntry.ContextBuilder.noContext; + +/** + * This exception is thrown if the repository import fails. + * + * @since 2.14.0 + */ +public class IncompatibleEnvironmentForImportException extends ExceptionWithContext { + + private static final String CODE = "5GSO9ZkzX1"; + + public IncompatibleEnvironmentForImportException() { + super(noContext(), "Incompatible SCM-Manager environment. Cannot import file. See previous logs for more detail."); + } + + @Override + public String getCode() { + return CODE; + } +} diff --git a/scm-core/src/main/java/sonia/scm/update/RepositoryUpdateIterator.java b/scm-core/src/main/java/sonia/scm/update/RepositoryUpdateIterator.java index 7ac218bcd7..f230aa0b65 100644 --- a/scm-core/src/main/java/sonia/scm/update/RepositoryUpdateIterator.java +++ b/scm-core/src/main/java/sonia/scm/update/RepositoryUpdateIterator.java @@ -24,6 +24,8 @@ package sonia.scm.update; +import sonia.scm.migration.UpdateException; + import java.util.function.Consumer; /** @@ -39,4 +41,35 @@ public interface RepositoryUpdateIterator { * @since 2.13.0 */ void forEachRepository(Consumer repositoryIdConsumer); + + /** + * Equivalent to {@link #forEachRepository(Consumer)} with the difference, that you can throw exceptions in the given + * update code, that will then be wrapped in a {@link UpdateException}. + * + * @since 2.14.0 + */ + default void updateEachRepository(Updater updater) { + forEachRepository( + repositoryId -> { + try { + updater.update(repositoryId); + } catch (Exception e) { + throw new UpdateException("failed to update repository with id " + repositoryId, e); + } + } + ); + } + + /** + * Simple callback with the id of an existing repository with the possibility to throw exceptions. + * + * @since 2.14.0 + */ + interface Updater { + /** + * Implements the update logic for a single repository, denoted by its id. + */ + @SuppressWarnings("java:S112") // We explicitly want to allow arbitrary exceptions here + void update(String repositoryId) throws Exception; + } } diff --git a/scm-test/src/main/java/sonia/scm/store/InMemoryConfigurationEntryStoreFactory.java b/scm-test/src/main/java/sonia/scm/store/InMemoryConfigurationEntryStoreFactory.java index 0c463de951..51b31aca67 100644 --- a/scm-test/src/main/java/sonia/scm/store/InMemoryConfigurationEntryStoreFactory.java +++ b/scm-test/src/main/java/sonia/scm/store/InMemoryConfigurationEntryStoreFactory.java @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.store; import java.util.HashMap; @@ -38,10 +38,13 @@ public class InMemoryConfigurationEntryStoreFactory implements ConfigurationEntr @Override public ConfigurationEntryStore getStore(TypedStoreParameters storeParameters) { String name = storeParameters.getName(); + if (storeParameters.getRepositoryId() != null) { + name = name + "-" + storeParameters.getRepositoryId(); + } return get(name); } public InMemoryConfigurationEntryStore get(String name) { - return stores.computeIfAbsent(name, x -> new InMemoryConfigurationEntryStore()); + return stores.computeIfAbsent(name, x -> new InMemoryConfigurationEntryStore()); } } diff --git a/scm-ui/ui-webapp/public/locales/de/repos.json b/scm-ui/ui-webapp/public/locales/de/repos.json index 99c9e22f48..1f3a87eeb1 100644 --- a/scm-ui/ui-webapp/public/locales/de/repos.json +++ b/scm-ui/ui-webapp/public/locales/de/repos.json @@ -78,7 +78,7 @@ }, "fullImport": { "title": "SCM-Manager Repository Archiv", - "helpText": "Wählen Sie das Repository Archiv aus. Das Archiv muss von einer SCM-Manager Instanz exportiert worden sein." + "helpText": "Wählen Sie das Repository Archiv aus. Das Archiv muss von einer SCM-Manager Instanz exportiert worden sein. Diese Instanz und alle installierten Plugins müssen dieselbe oder eine neuere Version haben." }, "pending": { "subtitle": "Repository wird importiert...", @@ -260,7 +260,7 @@ }, "fullExport": { "label": "Mit Metadaten (Experimentell)", - "helpText": "Zusätzlich zum Repository Dump werden Metadaten zum Repository und zur SCM-Instanz exportiert. Gespeicherte Passwörter funktionieren nicht bei einem Import in andere SCM-Manager Instanzen. Dieses Feature ist noch experimentell. Es sollte (noch) nicht für Backups genutzt werden!" + "helpText": "Zusätzlich zum Repository Dump werden Metadaten zum Repository und zur SCM-Instanz exportiert. Installierte Plugins sollten nach Möglichkeit in der neuesten Version installiert sein. Gespeicherte Passwörter funktionieren nicht bei einem Import in andere SCM-Manager Instanzen. Dieses Feature ist noch experimentell. Es sollte (noch) nicht für Backups genutzt werden!" }, "exportButton": "Repository exportieren", "exportStarted": "Der Repository Export wurde gestartet. Abhängig von der Größe des Repository kann dies einige Momente dauern." diff --git a/scm-ui/ui-webapp/public/locales/en/repos.json b/scm-ui/ui-webapp/public/locales/en/repos.json index a6b68dec05..5621997ad1 100644 --- a/scm-ui/ui-webapp/public/locales/en/repos.json +++ b/scm-ui/ui-webapp/public/locales/en/repos.json @@ -78,7 +78,7 @@ }, "fullImport": { "title": "SCM-Manager Repository Archive", - "helpText": "Select the archive file which should be imported. The archive must have been exported from an SCM-Manager instance." + "helpText": "Select the archive file which should be imported. The archive must have been exported from an SCM-Manager instance. This instance and all installed plugins have to be of the same or a newer version." }, "pending": { "subtitle": "Importing Repository...", @@ -260,7 +260,7 @@ }, "fullExport": { "label": "With metadata (Experimental)", - "helpText": "In addition to the repository dump, metadata about the repository and SCM instance is exported. Stored passwords will not work if this is imported in other instances of SCM-Manager. This feature is still experimental. Do not use this as a backup mechanism (yet)!" + "helpText": "In addition to the repository dump, metadata about the repository and SCM instance is exported. If possible, ensure that installed plugins are up to date. However, stored passwords will not work if this is imported in other instances of SCM-Manager. This feature is still experimental. Do not use this as a backup mechanism (yet)!" }, "exportButton": "Export Repository", "exportStarted": "The repository export was started. Depending on the repository size this may take a while." diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java index 3de8adef38..71882ab676 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java @@ -33,8 +33,10 @@ import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryManager; import sonia.scm.repository.RepositoryPermission; import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.update.UpdateEngine; import javax.inject.Inject; import javax.xml.bind.JAXB; @@ -51,22 +53,26 @@ import static sonia.scm.importexport.FullScmRepositoryExporter.STORE_DATA_FILE_N public class FullScmRepositoryImporter { + @SuppressWarnings("java:S115") // we like this name here private static final int _1_MB = 1000000; private final RepositoryServiceFactory serviceFactory; private final RepositoryManager repositoryManager; private final ScmEnvironmentCompatibilityChecker compatibilityChecker; private final TarArchiveRepositoryStoreImporter storeImporter; + private final UpdateEngine updateEngine; @Inject public FullScmRepositoryImporter(RepositoryServiceFactory serviceFactory, RepositoryManager repositoryManager, ScmEnvironmentCompatibilityChecker compatibilityChecker, - TarArchiveRepositoryStoreImporter storeImporter) { + TarArchiveRepositoryStoreImporter storeImporter, + UpdateEngine updateEngine) { this.serviceFactory = serviceFactory; this.repositoryManager = repositoryManager; this.compatibilityChecker = compatibilityChecker; this.storeImporter = storeImporter; + this.updateEngine = updateEngine; } public Repository importFromStream(Repository repository, InputStream inputStream) { @@ -113,6 +119,7 @@ public class FullScmRepositoryImporter { // Inside the repository tar archive stream is another tar archive. // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter storeImporter.importFromTarArchive(repository, tais); + updateEngine.update(repository.getId()); } else { throw new ImportFailedException( ContextEntry.ContextBuilder.entity(repository).build(), @@ -148,10 +155,7 @@ public class FullScmRepositoryImporter { if (environmentEntry.getName().equals(SCM_ENVIRONMENT_FILE_NAME) && !environmentEntry.isDirectory() && environmentEntry.getSize() < _1_MB) { boolean validEnvironment = compatibilityChecker.check(JAXB.unmarshal(new NoneClosingInputStream(tais), ScmEnvironment.class)); if (!validEnvironment) { - throw new ImportFailedException( - ContextEntry.ContextBuilder.noContext(), - "Incompatible SCM-Manager environment. Could not import file." - ); + throw new IncompatibleEnvironmentForImportException(); } } else { throw new ImportFailedException( diff --git a/scm-webapp/src/main/java/sonia/scm/importexport/ScmEnvironmentCompatibilityChecker.java b/scm-webapp/src/main/java/sonia/scm/importexport/ScmEnvironmentCompatibilityChecker.java index b9bf237fe4..f780a243fd 100644 --- a/scm-webapp/src/main/java/sonia/scm/importexport/ScmEnvironmentCompatibilityChecker.java +++ b/scm-webapp/src/main/java/sonia/scm/importexport/ScmEnvironmentCompatibilityChecker.java @@ -29,9 +29,9 @@ import org.slf4j.LoggerFactory; import sonia.scm.SCMContextProvider; import sonia.scm.plugin.PluginInformation; import sonia.scm.plugin.PluginManager; +import sonia.scm.version.Version; import javax.inject.Inject; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -54,10 +54,10 @@ public class ScmEnvironmentCompatibilityChecker { } private boolean isCoreVersionCompatible(String currentCoreVersion, String coreVersionFromImport) { - boolean compatible = currentCoreVersion.equals(coreVersionFromImport); + boolean compatible = Version.parse(currentCoreVersion).isNewerOrEqual(coreVersionFromImport); if (!compatible) { LOG.info( - "SCM-Manager version is not compatible with dump. Dump can only be imported with SCM-Manager version: {}; you are running version {}", + "SCM-Manager version is not compatible with dump. Dump can only be imported with SCM-Manager version {} or newer; you are running version {}.", coreVersionFromImport, currentCoreVersion ); @@ -73,9 +73,9 @@ public class ScmEnvironmentCompatibilityChecker { for (EnvironmentPluginDescriptor plugin : environment.getPlugins().getPlugin()) { Optional matchingInstalledPlugin = findMatchingInstalledPlugin(currentlyInstalledPlugins, plugin); - if (isPluginIncompatible(plugin, matchingInstalledPlugin)) { + if (matchingInstalledPlugin.isPresent() && isPluginIncompatible(plugin, matchingInstalledPlugin.get())) { LOG.info( - "The installed plugin \"{}\" with version \"{}\" doesn't match the plugin data version \"{}\" from the SCM-Manager environment the dump was created with.", + "The installed plugin \"{}\" with version \"{}\" is older than the plugin data version \"{}\" from the SCM-Manager environment the dump was created with. Please update the plugin.", matchingInstalledPlugin.get().getName(), matchingInstalledPlugin.get().getVersion(), plugin.getVersion() @@ -86,8 +86,8 @@ public class ScmEnvironmentCompatibilityChecker { return true; } - private boolean isPluginIncompatible(EnvironmentPluginDescriptor plugin, Optional matchingInstalledPlugin) { - return matchingInstalledPlugin.isPresent() && isPluginVersionIncompatible(plugin.getVersion(), matchingInstalledPlugin.get().getVersion()); + private boolean isPluginIncompatible(EnvironmentPluginDescriptor plugin, PluginInformation matchingInstalledPlugin) { + return isPluginVersionIncompatible(plugin.getVersion(), matchingInstalledPlugin.getVersion()); } private Optional findMatchingInstalledPlugin(List currentlyInstalledPlugins, EnvironmentPluginDescriptor plugin) { @@ -98,6 +98,6 @@ public class ScmEnvironmentCompatibilityChecker { } private boolean isPluginVersionIncompatible(String previousPluginVersion, String installedPluginVersion) { - return !installedPluginVersion.equals(previousPluginVersion); + return Version.parse(installedPluginVersion).isOlder(previousPluginVersion); } } diff --git a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/UpdateStepModule.java b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/UpdateStepModule.java index c0891c494e..9a5415ea03 100644 --- a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/UpdateStepModule.java +++ b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/UpdateStepModule.java @@ -21,11 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.lifecycle.modules; import com.google.inject.AbstractModule; import com.google.inject.multibindings.Multibinder; +import sonia.scm.migration.RepositoryUpdateStep; import sonia.scm.migration.UpdateStep; import sonia.scm.plugin.PluginLoader; @@ -40,9 +41,14 @@ public class UpdateStepModule extends AbstractModule { @Override protected void configure() { Multibinder updateStepBinder = Multibinder.newSetBinder(binder(), UpdateStep.class); + Multibinder repositoryUdateStepBinder = Multibinder.newSetBinder(binder(), RepositoryUpdateStep.class); pluginLoader .getExtensionProcessor() .byExtensionPoint(UpdateStep.class) .forEach(stepClass -> updateStepBinder.addBinding().to(stepClass)); + pluginLoader + .getExtensionProcessor() + .byExtensionPoint(RepositoryUpdateStep.class) + .forEach(stepClass -> repositoryUdateStepBinder.addBinding().to(stepClass)); } } diff --git a/scm-webapp/src/main/java/sonia/scm/update/UpdateEngine.java b/scm-webapp/src/main/java/sonia/scm/update/UpdateEngine.java index 4e783d15e9..c59f7c92da 100644 --- a/scm-webapp/src/main/java/sonia/scm/update/UpdateEngine.java +++ b/scm-webapp/src/main/java/sonia/scm/update/UpdateEngine.java @@ -21,22 +21,28 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.update; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sonia.scm.migration.RepositoryUpdateContext; +import sonia.scm.migration.RepositoryUpdateStep; import sonia.scm.migration.UpdateException; import sonia.scm.migration.UpdateStep; +import sonia.scm.migration.UpdateStepTarget; import sonia.scm.store.ConfigurationEntryStore; import sonia.scm.store.ConfigurationEntryStoreFactory; +import sonia.scm.version.Version; import javax.inject.Inject; import java.util.Comparator; import java.util.List; import java.util.Set; +import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import static java.util.stream.Stream.concat; public class UpdateEngine { @@ -44,56 +50,84 @@ public class UpdateEngine { private static final String STORE_NAME = "executedUpdates"; - private final List steps; + private final List steps; private final ConfigurationEntryStore store; + private final ConfigurationEntryStoreFactory storeFactory; + private final RepositoryUpdateIterator repositoryUpdateIterator; @Inject - public UpdateEngine(Set steps, ConfigurationEntryStoreFactory storeFactory) { - this.steps = sortSteps(steps); + public UpdateEngine( + Set globalSteps, + Set repositorySteps, + ConfigurationEntryStoreFactory storeFactory, + RepositoryUpdateIterator repositoryUpdateIterator + ) { + this.storeFactory = storeFactory; + this.repositoryUpdateIterator = repositoryUpdateIterator; this.store = storeFactory.withType(UpdateVersionInfo.class).withName(STORE_NAME).build(); + this.steps = sortSteps(globalSteps, repositorySteps); } - private List sortSteps(Set steps) { + private List sortSteps(Set globalSteps, Set repositorySteps) { LOG.trace("sorting available update steps:"); - List sortedSteps = steps.stream() + List sortedSteps = + concat( + globalSteps.stream().filter(this::notRunYet).map(GlobalUpdateStepWrapper::new), + repositorySteps.stream().map(RepositoryUpdateStepWrapper::new)) .sorted( Comparator - .comparing(UpdateStep::getTargetVersion) - .thenComparing(this::isCoreUpdateStep) - .reversed()) + .comparing(UpdateStepWrapper::getTargetVersion) + .thenComparing(UpdateStepWrapper::isGlobalUpdateStep) + .thenComparing(UpdateEngine::isCoreUpdateStep) + .reversed() + ) .collect(toList()); sortedSteps.forEach(step -> LOG.trace("{} for version {}", step.getAffectedDataType(), step.getTargetVersion())); return sortedSteps; } - private boolean isCoreUpdateStep(UpdateStep updateStep) { - return updateStep instanceof CoreUpdateStep; + private static boolean isCoreUpdateStep(UpdateStepWrapper updateStep) { + return updateStep.isCoreUpdate(); } public void update() { - steps - .stream() - .filter(this::notRunYet) - .forEach(this::execute); + steps.forEach(this::execute); } - private void execute(UpdateStep updateStep) { + public void update(String repositoryId) { + steps.forEach(step -> execute(step, repositoryId)); + } + + private void execute(UpdateStepWrapper updateStep) { try { - LOG.info("running update step for type {} and version {} (class {})", - updateStep.getAffectedDataType(), - updateStep.getTargetVersion(), - updateStep.getClass().getName() - ); updateStep.doUpdate(); } catch (Exception e) { throw new UpdateException( - String.format( + format( "could not execute update for type %s to version %s in %s", updateStep.getAffectedDataType(), updateStep.getTargetVersion(), updateStep.getClass()), e); } + } + + private void execute(UpdateStepWrapper updateStep, String repositoryId) { + try { + updateStep.doUpdate(repositoryId); + } catch (Exception e) { + throw new UpdateException( + format( + "could not execute update for type %s to version %s in %s for repository id %s", + updateStep.getAffectedDataType(), + updateStep.getTargetVersion(), + updateStep.getClass(), + repositoryId), + e); + } + } + + private void storeNewVersion(ConfigurationEntryStore store, UpdateStepTarget updateStep) { UpdateVersionInfo newVersionInfo = new UpdateVersionInfo(updateStep.getTargetVersion().getParsedVersion()); store.put(updateStep.getAffectedDataType(), newVersionInfo); } @@ -103,6 +137,10 @@ public class UpdateEngine { updateStep.getAffectedDataType(), updateStep.getTargetVersion() ); + return notRunYet(this.store, updateStep); + } + + private boolean notRunYet(ConfigurationEntryStore store, UpdateStepTarget updateStep) { UpdateVersionInfo updateVersionInfo = store.get(updateStep.getAffectedDataType()); if (updateVersionInfo == null) { LOG.trace("no updates for type {} run yet; step will be executed", updateStep.getAffectedDataType()); @@ -116,4 +154,128 @@ public class UpdateEngine { ); return result; } + + private abstract static class UpdateStepWrapper implements UpdateStepTarget { + + private final UpdateStepTarget delegate; + + protected UpdateStepWrapper(UpdateStepTarget delegate) { + this.delegate = delegate; + } + + @Override + public Version getTargetVersion() { + return delegate.getTargetVersion(); + } + + @Override + public String getAffectedDataType() { + return delegate.getAffectedDataType(); + } + + abstract boolean isGlobalUpdateStep(); + + abstract boolean isCoreUpdate(); + + @SuppressWarnings("java:S112") // we explicitly want to allow all kinds of exceptions here + abstract void doUpdate() throws Exception; + + @SuppressWarnings("java:S112") // we explicitly want to allow all kinds of exceptions here + abstract void doUpdate(String repositoryId) throws Exception; + } + + private class GlobalUpdateStepWrapper extends UpdateStepWrapper { + private final UpdateStep delegate; + private final boolean coreUpdate; + + protected GlobalUpdateStepWrapper(UpdateStep delegate) { + super(delegate); + this.delegate = delegate; + this.coreUpdate = delegate instanceof CoreUpdateStep; + } + + @Override + public Version getTargetVersion() { + return delegate.getTargetVersion(); + } + + @Override + public String getAffectedDataType() { + return delegate.getAffectedDataType(); + } + + public boolean isCoreUpdate() { + return coreUpdate; + } + + public boolean isGlobalUpdateStep() { + return true; + } + + void doUpdate() throws Exception { + LOG.info("running update step for type {} and version {} (class {})", + delegate.getAffectedDataType(), + delegate.getTargetVersion(), + delegate.getClass().getName() + ); + delegate.doUpdate(); + storeNewVersion(store, delegate); + } + + void doUpdate(String repositoryId) { + // nothing to do for repositories here + } + } + + private class RepositoryUpdateStepWrapper extends UpdateStepWrapper { + + private final RepositoryUpdateStep delegate; + + public RepositoryUpdateStepWrapper(RepositoryUpdateStep delegate) { + super(delegate); + this.delegate = delegate; + } + + @Override + public boolean isGlobalUpdateStep() { + return false; + } + + @Override + boolean isCoreUpdate() { + return false; + } + + @Override + void doUpdate() { + repositoryUpdateIterator.updateEachRepository(this::doUpdate); + } + + @Override + void doUpdate(String repositoryId) throws Exception { + if (notRunYet(repositoryId)) { + LOG.info("running update step for type {} and version {} (class {}) for repository id {}", + delegate.getAffectedDataType(), + delegate.getTargetVersion(), + delegate.getClass().getName(), + repositoryId + ); + delegate.doUpdate(new RepositoryUpdateContext(repositoryId)); + storeNewVersion(storeForRepository(repositoryId), delegate); + } + } + + private boolean notRunYet(String repositoryId) { + LOG.trace("checking whether to run update step for type {} and version {} on repository id {}", + delegate.getAffectedDataType(), + delegate.getTargetVersion(), + repositoryId + ); + return UpdateEngine.this.notRunYet(storeForRepository(repositoryId), delegate); + } + + private ConfigurationEntryStore storeForRepository(String repositoryId) { + return storeFactory.withType(UpdateVersionInfo.class).withName(STORE_NAME).forRepository(repositoryId).build(); + } + } } diff --git a/scm-webapp/src/main/resources/locales/de/plugins.json b/scm-webapp/src/main/resources/locales/de/plugins.json index 450a7fcf67..afeb58802c 100644 --- a/scm-webapp/src/main/resources/locales/de/plugins.json +++ b/scm-webapp/src/main/resources/locales/de/plugins.json @@ -334,6 +334,10 @@ "8YR7aawFW1": { "displayName": "Aktuelles Passwort falsch", "description": "Das aktuelle Passwort ist falsch. Bitte versuchen Sie es noch einmal." + }, + "5GSO9ZkzX1": { + "displayName": "Inkompatible Umgebung", + "description": "Die Version dieses SCM-Managers oder eines der installierten Plugins ist zu alt für den Import des Dumps. Bitte installieren Sie die neuesten Versionen. Nähere Informationen finden sich im Log." } }, "namespaceStrategies": { diff --git a/scm-webapp/src/main/resources/locales/en/plugins.json b/scm-webapp/src/main/resources/locales/en/plugins.json index 80ea11ac90..d8c23385b3 100644 --- a/scm-webapp/src/main/resources/locales/en/plugins.json +++ b/scm-webapp/src/main/resources/locales/en/plugins.json @@ -331,6 +331,10 @@ "displayName": "Repository is being exported", "description": "The repository is being exported and therefore must not be modified." }, + "5GSO9ZkzX1": { + "displayName": "Incompatible environment", + "description": "The version of this SCM-Manager or one of its plugins is too old for the import of the given dump. Please update to the latest versions. See the log for more information." + }, "8YR7aawFW1": { "displayName": "Wrong current password", "description": "The current password is wrong. Please try again." diff --git a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java index cadf9597ca..a3d7f2cded 100644 --- a/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java +++ b/scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java @@ -27,6 +27,7 @@ package sonia.scm.importexport; import com.google.common.io.Files; import com.google.common.io.Resources; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -39,9 +40,11 @@ import sonia.scm.repository.RepositoryManager; import sonia.scm.repository.RepositoryPermission; import sonia.scm.repository.RepositoryTestData; import sonia.scm.repository.api.ImportFailedException; +import sonia.scm.repository.api.IncompatibleEnvironmentForImportException; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; import sonia.scm.repository.api.UnbundleCommandBuilder; +import sonia.scm.update.UpdateEngine; import java.io.File; import java.io.FileInputStream; @@ -49,16 +52,19 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.Collection; +import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) +@SuppressWarnings("UnstableApiUsage") class FullScmRepositoryImporterTest { private static final Repository REPOSITORY = RepositoryTestData.createHeartOfGold("svn"); @@ -75,6 +81,8 @@ class FullScmRepositoryImporterTest { private ScmEnvironmentCompatibilityChecker compatibilityChecker; @Mock private TarArchiveRepositoryStoreImporter storeImporter; + @Mock + private UpdateEngine updateEngine; @InjectMocks private FullScmRepositoryImporter fullImporter; @@ -89,29 +97,57 @@ class FullScmRepositoryImporterTest { void shouldNotImportRepositoryIfFileNotExists(@TempDir Path temp) throws IOException { File emptyFile = new File(temp.resolve("empty").toString()); Files.touch(emptyFile); - assertThrows(ImportFailedException.class, () -> fullImporter.importFromStream(REPOSITORY, new FileInputStream(emptyFile))); - } - - @Test - void shouldFailIfScmEnvironmentIsIncompatible() { - when(compatibilityChecker.check(any())).thenReturn(false); - + FileInputStream inputStream = new FileInputStream(emptyFile); assertThrows( ImportFailedException.class, - () -> fullImporter.importFromStream(REPOSITORY, Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream()) + () -> fullImporter.importFromStream(REPOSITORY, inputStream) ); } @Test - void shouldImportScmRepositoryArchive() throws IOException { - when(compatibilityChecker.check(any())).thenReturn(true); - when(repositoryManager.create(eq(REPOSITORY), any())).thenReturn(REPOSITORY); + void shouldFailIfScmEnvironmentIsIncompatible() throws IOException { + when(compatibilityChecker.check(any())).thenReturn(false); - Repository repository = fullImporter.importFromStream(REPOSITORY, Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream()); - assertThat(repository).isEqualTo(REPOSITORY); - verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class)); - verify(repositoryManager).modify(REPOSITORY); - Collection updatedPermissions = REPOSITORY.getPermissions(); - assertThat(updatedPermissions).hasSize(2); + InputStream importStream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); + assertThrows( + IncompatibleEnvironmentForImportException.class, + () -> fullImporter.importFromStream(REPOSITORY, importStream) + ); + } + + @Nested + class WithValidEnvironment { + + @BeforeEach + void setUpEnvironment() { + when(compatibilityChecker.check(any())).thenReturn(true); + when(repositoryManager.create(eq(REPOSITORY), any())).thenAnswer(invocation -> { + invocation.getArgument(1, Consumer.class).accept(REPOSITORY); + return REPOSITORY; + }); + } + + @Test + void shouldImportScmRepositoryArchive() throws IOException { + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); + + Repository repository = fullImporter.importFromStream(REPOSITORY, stream); + + assertThat(repository).isEqualTo(REPOSITORY); + verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class)); + verify(repositoryManager).modify(REPOSITORY); + Collection updatedPermissions = REPOSITORY.getPermissions(); + assertThat(updatedPermissions).hasSize(2); + verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(FullScmRepositoryImporter.NoneClosingInputStream.class))); + } + + @Test + void shouldTriggerUpdateForImportedRepository() throws IOException { + InputStream stream = Resources.getResource("sonia/scm/repository/import/scm-import.tar.gz").openStream(); + + fullImporter.importFromStream(REPOSITORY, stream); + + verify(updateEngine).update(REPOSITORY.getId()); + } } } diff --git a/scm-webapp/src/test/java/sonia/scm/importexport/ScmEnvironmentCompatibilityCheckerTest.java b/scm-webapp/src/test/java/sonia/scm/importexport/ScmEnvironmentCompatibilityCheckerTest.java index 8afaa3f6b2..22a2a8b67d 100644 --- a/scm-webapp/src/test/java/sonia/scm/importexport/ScmEnvironmentCompatibilityCheckerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/importexport/ScmEnvironmentCompatibilityCheckerTest.java @@ -35,6 +35,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.SCMContextProvider; import sonia.scm.plugin.InstalledPlugin; import sonia.scm.plugin.PluginManager; +import sonia.scm.version.Version; import java.util.Collections; import java.util.List; @@ -61,7 +62,7 @@ class ScmEnvironmentCompatibilityCheckerTest { } @Test - void shouldReturnTrueIfEnvironmentIsCompatible() { + void shouldReturnTrueIfEnvironmentIsSame() { when(scmContextProvider.getVersion()).thenReturn("2.0.0"); ImmutableList plugins = ImmutableList.of( new EnvironmentPluginDescriptor("scm-first-plugin", "1.0.0"), @@ -74,6 +75,20 @@ class ScmEnvironmentCompatibilityCheckerTest { assertThat(compatible).isTrue(); } + @Test + void shouldReturnTrueIfEnvironmentIsNewer() { + when(scmContextProvider.getVersion()).thenReturn("2.1.0"); + ImmutableList plugins = ImmutableList.of( + new EnvironmentPluginDescriptor("scm-first-plugin", "0.9.0"), + new EnvironmentPluginDescriptor("scm-second-plugin", "1.0.1") + ); + ScmEnvironment env = createScmEnvironment("2.0.0", "linux", "64", plugins); + + boolean compatible = checker.check(env); + + assertThat(compatible).isTrue(); + } + @Test void shouldReturnFalseIfCoreVersionIncompatible() { when(scmContextProvider.getVersion()).thenReturn("2.0.0"); @@ -124,5 +139,4 @@ class ScmEnvironmentCompatibilityCheckerTest { scmEnvironment.setPlugins(environmentPluginsDescriptor); return scmEnvironment; } - } diff --git a/scm-webapp/src/test/java/sonia/scm/update/UpdateEngineTest.java b/scm-webapp/src/test/java/sonia/scm/update/UpdateEngineTest.java index 7e7179198e..831c774aec 100644 --- a/scm-webapp/src/test/java/sonia/scm/update/UpdateEngineTest.java +++ b/scm-webapp/src/test/java/sonia/scm/update/UpdateEngineTest.java @@ -24,7 +24,10 @@ package sonia.scm.update; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import sonia.scm.migration.RepositoryUpdateContext; +import sonia.scm.migration.RepositoryUpdateStep; import sonia.scm.migration.UpdateStep; import sonia.scm.store.ConfigurationEntryStoreFactory; import sonia.scm.store.InMemoryConfigurationEntryStoreFactory; @@ -33,16 +36,35 @@ import sonia.scm.version.Version; import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; +import java.util.function.Consumer; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import static sonia.scm.version.Version.parse; class UpdateEngineTest { ConfigurationEntryStoreFactory storeFactory = new InMemoryConfigurationEntryStoreFactory(); + RepositoryUpdateIterator repositoryUpdateIterator = mock(RepositoryUpdateIterator.class, CALLS_REAL_METHODS); List processedUpdates = new ArrayList<>(); + @BeforeEach + void mockRepositories() { + doAnswer(invocation -> { + Consumer consumer = invocation.getArgument(0, Consumer.class); + consumer.accept("42"); + consumer.accept("1337"); + return null; + }).when(repositoryUpdateIterator).forEachRepository(any()); + } + @Test void shouldProcessStepsInCorrectOrder() { LinkedHashSet updateSteps = new LinkedHashSet<>(); @@ -51,13 +73,45 @@ class UpdateEngineTest { updateSteps.add(new FixedVersionUpdateStep("test", "1.2.0")); updateSteps.add(new FixedVersionUpdateStep("test", "1.1.0")); - UpdateEngine updateEngine = new UpdateEngine(updateSteps, storeFactory); + UpdateEngine updateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); updateEngine.update(); assertThat(processedUpdates) .containsExactly("test:1.1.0", "test:1.1.1", "test:1.2.0"); } + @Test + void shouldProcessStepsInCorrectOrderWithRepositoryUpdates() { + LinkedHashSet updateSteps = new LinkedHashSet<>(); + LinkedHashSet repositoryUpdateSteps = new LinkedHashSet<>(); + + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.1.1")); + updateSteps.add(new FixedVersionUpdateStep("test", "1.2.0")); + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.1.0")); + + UpdateEngine updateEngine = new UpdateEngine(updateSteps, repositoryUpdateSteps, storeFactory, repositoryUpdateIterator); + updateEngine.update(); + + assertThat(processedUpdates) + .containsExactly("test:1.1.0-42", "test:1.1.0-1337", "test:1.1.1-42", "test:1.1.1-1337", "test:1.2.0"); + } + + @Test + void shouldProcessStepsForSingleRepository() { + LinkedHashSet updateSteps = new LinkedHashSet<>(); + LinkedHashSet repositoryUpdateSteps = new LinkedHashSet<>(); + + updateSteps.add(new FixedVersionUpdateStep("test", "1.2.0")); + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.1.1")); + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.1.0")); + + UpdateEngine updateEngine = new UpdateEngine(updateSteps, repositoryUpdateSteps, storeFactory, repositoryUpdateIterator); + updateEngine.update("1337"); + + assertThat(processedUpdates) + .containsExactly("test:1.1.0-1337", "test:1.1.1-1337"); + } + @Test void shouldProcessCoreStepsBeforeOther() { LinkedHashSet updateSteps = new LinkedHashSet<>(); @@ -65,7 +119,7 @@ class UpdateEngineTest { updateSteps.add(new FixedVersionUpdateStep("test", "1.2.0")); updateSteps.add(new CoreFixedVersionUpdateStep("core", "1.2.0")); - UpdateEngine updateEngine = new UpdateEngine(updateSteps, storeFactory); + UpdateEngine updateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); updateEngine.update(); assertThat(processedUpdates) @@ -73,41 +127,73 @@ class UpdateEngineTest { } @Test - void shouldRunStepsOnlyOnce() { + void shouldProcessGlobalStepsBeforeRepository() { + Set updateSteps = singleton(new FixedVersionUpdateStep("test", "1.2.0")); + Set repositoryUpdateSteps = singleton(new FixedVersionUpdateStep("test", "1.2.0")); + + UpdateEngine updateEngine = new UpdateEngine(updateSteps, repositoryUpdateSteps, storeFactory, repositoryUpdateIterator); + updateEngine.update(); + + assertThat(processedUpdates) + .containsExactly("test:1.2.0", "test:1.2.0-42", "test:1.2.0-1337"); + } + + @Test + void shouldRunGlobalStepsOnlyOnce() { LinkedHashSet updateSteps = new LinkedHashSet<>(); updateSteps.add(new FixedVersionUpdateStep("test", "1.1.1")); - UpdateEngine updateEngine = new UpdateEngine(updateSteps, storeFactory); - updateEngine.update(); + UpdateEngine firstUpdateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); + firstUpdateEngine.update(); processedUpdates.clear(); - updateEngine.update(); + UpdateEngine secondUpdateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); + secondUpdateEngine.update(); assertThat(processedUpdates).isEmpty(); } + @Test + void shouldRunRepositoryStepsOnlyOnce() { + LinkedHashSet repositoryUpdateSteps = new LinkedHashSet<>(); + + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.1.1")); + + UpdateEngine firstUpdateEngine = new UpdateEngine(emptySet(), repositoryUpdateSteps, storeFactory, repositoryUpdateIterator); + firstUpdateEngine.update(); + + processedUpdates.clear(); + + repositoryUpdateSteps.add(new FixedVersionUpdateStep("test", "1.2.0")); + UpdateEngine secondUpdateEngine = new UpdateEngine(emptySet(), repositoryUpdateSteps, storeFactory, repositoryUpdateIterator); + secondUpdateEngine.update(); + + assertThat(processedUpdates) + .containsExactly("test:1.2.0-42", "test:1.2.0-1337"); + } + @Test void shouldRunStepsForDifferentTypesIndependently() { LinkedHashSet updateSteps = new LinkedHashSet<>(); updateSteps.add(new FixedVersionUpdateStep("test", "1.1.1")); - UpdateEngine updateEngine = new UpdateEngine(updateSteps, storeFactory); + UpdateEngine updateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); updateEngine.update(); processedUpdates.clear(); updateSteps.add(new FixedVersionUpdateStep("other", "1.1.1")); - updateEngine = new UpdateEngine(updateSteps, storeFactory); + updateEngine = new UpdateEngine(updateSteps, emptySet(), storeFactory, repositoryUpdateIterator); updateEngine.update(); assertThat(processedUpdates).containsExactly("other:1.1.1"); } - class FixedVersionUpdateStep implements UpdateStep { + class FixedVersionUpdateStep implements UpdateStep, RepositoryUpdateStep { private final String type; private final String version; @@ -130,6 +216,11 @@ class UpdateEngineTest { public void doUpdate() { processedUpdates.add(type + ":" + version); } + + @Override + public void doUpdate(RepositoryUpdateContext repositoryUpdateContext) throws Exception { + processedUpdates.add(type + ":" + version + "-" + repositoryUpdateContext.getRepositoryId()); + } } class CoreFixedVersionUpdateStep extends FixedVersionUpdateStep implements CoreUpdateStep {