Simple approach for import logs

scm-webapp/src/main/java/sonia/scm/importexport/EnvironmentCheckStep.java

@@ -60,6 +60,7 @@ class EnvironmentCheckStep implements ImportStep {
       if (!validEnvironment) {
         throw new IncompatibleEnvironmentForImportException();
       }
+      state.getLogger().step("checked environment");
       state.environmentChecked();
       return true;
     }

scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java

@@ -42,6 +42,7 @@ import java.io.IOException;
 import java.io.InputStream;
 
 import static java.util.Arrays.stream;
+import static sonia.scm.importexport.RepositoryImportLog.ImportType.FULL;
 import static sonia.scm.util.Archives.createTarInputStream;
 import static sonia.scm.ContextEntry.ContextBuilder.noContext;
 
@@ -53,6 +54,7 @@ public class FullScmRepositoryImporter {
   private final RepositoryManager repositoryManager;
   private final RepositoryImportExportEncryption repositoryImportExportEncryption;
   private final ScmEventBus eventBus;
+  private final RepositoryImportLoggerFactory loggerFactory;
 
   @Inject
   public FullScmRepositoryImporter(EnvironmentCheckStep environmentCheckStep,
@@ -61,15 +63,18 @@ public class FullScmRepositoryImporter {
                                    RepositoryImportStep repositoryImportStep,
                                    RepositoryManager repositoryManager,
                                    RepositoryImportExportEncryption repositoryImportExportEncryption,
+                                   RepositoryImportLoggerFactory loggerFactory,
                                    ScmEventBus eventBus
   ) {
     this.repositoryManager = repositoryManager;
+    this.loggerFactory = loggerFactory;
     this.repositoryImportExportEncryption = repositoryImportExportEncryption;
     importSteps = new ImportStep[]{environmentCheckStep, metadataImportStep, storeImportStep, repositoryImportStep};
     this.eventBus = eventBus;
   }
 
   public Repository importFromStream(Repository repository, InputStream inputStream, String password) {
+    RepositoryImportLogger logger = startLogger(repository);
     try {
       if (inputStream.available() > 0) {
         try (
@@ -78,7 +83,7 @@ public class FullScmRepositoryImporter {
           GzipCompressorInputStream gcis = new GzipCompressorInputStream(cif);
           TarArchiveInputStream tais = createTarInputStream(gcis)
         ) {
-          return run(repository, tais);
+          return run(repository, tais, logger);
         }
       } else {
         throw new ImportFailedException(
@@ -103,8 +108,15 @@ public class FullScmRepositoryImporter {
     }
   }
 
-  private Repository run(Repository repository, TarArchiveInputStream tais) throws IOException {
-    ImportState state = new ImportState(repositoryManager.create(repository));
+  private RepositoryImportLogger startLogger(Repository repository) {
+    RepositoryImportLogger logger = loggerFactory.createLogger();
+    logger.start(FULL, repository);
+    return logger;
+  }
+
+  private Repository run(Repository repository, TarArchiveInputStream tais, RepositoryImportLogger logger) throws IOException {
+    ImportState state = new ImportState(repositoryManager.create(repository), logger);
+    logger.repositoryCreated(state.getRepository());
     try {
       TarArchiveEntry tarArchiveEntry;
       while ((tarArchiveEntry = tais.getNextTarEntry()) != null) {
@@ -112,7 +124,11 @@ public class FullScmRepositoryImporter {
         handle(tais, state, tarArchiveEntry);
       }
       stream(importSteps).forEach(step -> step.finish(state));
+      logger.finished();
       return state.getRepository();
+    } catch (RuntimeException | IOException e) {
+      logger.failed(e);
+      throw e;
     } finally {
       stream(importSteps).forEach(step -> step.cleanup(state));
       if (state.success()) {
@@ -127,6 +143,7 @@ public class FullScmRepositoryImporter {
   }
 
   private void handle(TarArchiveInputStream tais, ImportState state, TarArchiveEntry currentEntry) {
+    state.getLogger().step("inspecting file " + currentEntry.getName());
     for (ImportStep step : importSteps) {
       if (step.handle(currentEntry, state, tais)) {
         return;

scm-webapp/src/main/java/sonia/scm/importexport/ImportState.java

@@ -36,6 +36,8 @@ import java.util.Optional;
 
 class ImportState {
 
+  private final RepositoryImportLogger logger;
+
   private Repository repository;
 
   private boolean environmentChecked;
@@ -48,11 +50,8 @@ class ImportState {
 
   private final List<Object> pendingEvents = new ArrayList<>();
 
-  ImportState(Repository repository) {
-    this.repository = repository;
-  }
-
-  public void setRepository(Repository repository) {
+  ImportState(Repository repository, RepositoryImportLogger logger) {
+    this.logger = logger;
     this.repository = repository;
   }
 
@@ -103,6 +102,9 @@ class ImportState {
   public void addPendingEvent(Object event) {
     this.pendingEvents.add(event);
   }
+  RepositoryImportLogger getLogger() {
+    return logger;
+  }
 
   public Collection<Object> getPendingEvents() {
     return Collections.unmodifiableCollection(pendingEvents);

scm-webapp/src/main/java/sonia/scm/importexport/MetadataImportStep.java

@@ -58,6 +58,7 @@ class MetadataImportStep implements ImportStep {
       RepositoryMetadataXmlGenerator.RepositoryMetadata metadata = JAXB.unmarshal(new NoneClosingInputStream(inputStream), RepositoryMetadataXmlGenerator.RepositoryMetadata.class);
       if (metadata != null && metadata.getPermissions() != null) {
         state.setPermissions(new HashSet<>(metadata.getPermissions()));
+        state.getLogger().step("reading repository metadata with permissions");
       } else {
         state.setPermissions(Collections.emptySet());
       }
@@ -69,6 +70,7 @@ class MetadataImportStep implements ImportStep {
   @Override
   public void finish(ImportState state) {
     LOG.trace("Saving permissions for imported repository");
+    state.getLogger().step("setting permissions for repository from import");
     importRepositoryPermissions(state.getRepository(), state.getRepositoryPermissions());
   }
 

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLog.java

@@ -0,0 +1,86 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.importexport;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import static java.util.Collections.unmodifiableList;
+
+@XmlRootElement(name = "import")
+@XmlAccessorType(XmlAccessType.FIELD)
+@Getter
+@Setter
+class RepositoryImportLog {
+
+  private ImportType type;
+  private String repositoryType;
+  private String userName;
+  private String userId;
+  private String repositoryId;
+  private String namespace;
+  private String name;
+  private Boolean success;
+  @XmlElement(name = "entry")
+  private List<Entry> entries;
+
+  void addEntry(Entry entry) {
+    if (entries == null) {
+      entries = new ArrayList<>();
+    }
+    this.entries.add(entry);
+  }
+
+  public List<Entry> getEntries() {
+    return unmodifiableList(entries);
+  }
+
+  enum ImportType {
+    FULL, URL, DUMP
+  }
+
+  @XmlRootElement(name = "entry")
+  @XmlAccessorType(XmlAccessType.FIELD)
+  @SuppressWarnings("java:S1068") // unused fields will be serialized to xml
+  static class Entry {
+    private Date time = new Date();
+    private String message;
+
+    Entry() {
+    }
+
+    Entry(String message) {
+      this.message = message;
+    }
+  }
+}

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLogger.java

@@ -0,0 +1,83 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.importexport;
+
+import org.apache.shiro.SecurityUtils;
+import sonia.scm.importexport.RepositoryImportLog.ImportType;
+import sonia.scm.repository.Repository;
+import sonia.scm.store.DataStore;
+import sonia.scm.user.User;
+
+class RepositoryImportLogger {
+
+  private final DataStore<RepositoryImportLog> logStore;
+  private RepositoryImportLog log;
+  private String logId;
+
+  RepositoryImportLogger(DataStore<RepositoryImportLog> logStore) {
+    this.logStore = logStore;
+  }
+
+  void start(ImportType importType, Repository repository) {
+    User user = SecurityUtils.getSubject().getPrincipals().oneByType(User.class);
+    log = new RepositoryImportLog();
+    log.setType(importType);
+    log.setUserId(user.getId());
+    log.setUserName(user.getName());
+    log.setRepositoryType(repository.getType());
+    log.setNamespace(repository.getNamespace());
+    log.setName(repository.getName());
+    logId = logStore.put(log);
+    addLogEntry(new RepositoryImportLog.Entry("import started"));
+  }
+
+
+  public void finished() {
+    log.setSuccess(true);
+    step("import finished successfully");
+  }
+
+  public void failed(Exception e) {
+    log.setSuccess(false);
+    step("import failed (see next log entry)");
+    step(e.getMessage());
+  }
+
+  public void repositoryCreated(Repository createdRepository) {
+    log.setNamespace(createdRepository.getNamespace());
+    log.setName(createdRepository.getName());
+    log.setRepositoryId(createdRepository.getId());
+    step("created repository: " + createdRepository.getNamespaceAndName());
+  }
+
+  public void step(String message) {
+    addLogEntry(new RepositoryImportLog.Entry(message));
+  }
+
+  private void addLogEntry(RepositoryImportLog.Entry entry) {
+    log.addEntry(entry);
+    logStore.put(logId, log);
+  }
+}

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLoggerFactory.java

@@ -0,0 +1,43 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.importexport;
+
+import sonia.scm.store.DataStoreFactory;
+
+import javax.inject.Inject;
+
+class RepositoryImportLoggerFactory {
+
+  private final DataStoreFactory dataStoreFactory;
+
+  @Inject
+  RepositoryImportLoggerFactory(DataStoreFactory dataStoreFactory) {
+    this.dataStoreFactory = dataStoreFactory;
+  }
+
+  RepositoryImportLogger createLogger() {
+    return new RepositoryImportLogger(dataStoreFactory.withType(RepositoryImportLog.class).withName("imports").build());
+  }
+}

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportStep.java

@@ -62,12 +62,14 @@ class RepositoryImportStep implements ImportStep {
 
   @Override
   public boolean handle(TarArchiveEntry currentEntry, ImportState state, InputStream inputStream) {
-    if (!currentEntry.isDirectory()) {
+    if (!currentEntry.isDirectory() && !currentEntry.getName().contains("/")) {
       if (state.isStoreImported()) {
         LOG.trace("Importing directly from tar stream (entry '{}')", currentEntry.getName());
+        state.getLogger().step("directly importing repository data");
         unbundleRepository(state, inputStream);
       } else {
-        LOG.debug("Temporally storing tar entry '{}' in work dir", currentEntry.getName());
+        LOG.debug("Temporarily storing tar entry '{}' in work dir", currentEntry.getName());
+        state.getLogger().step("temporarily storing repository data for later import");
         Path path = saveRepositoryDataFromTarArchiveEntry(state.getRepository(), inputStream);
         state.setTemporaryRepositoryBundle(path);
       }
@@ -90,6 +92,7 @@ class RepositoryImportStep implements ImportStep {
 
   private void importFromTemporaryPath(ImportState state, Path path) {
     LOG.debug("Importing repository from temporary location in work dir");
+    state.getLogger().step("importing repository from temporary location");
     try {
       unbundleRepository(state, Files.newInputStream(path));
     } catch (IOException e) {

scm-webapp/src/main/java/sonia/scm/importexport/StoreImportStep.java

@@ -52,17 +52,18 @@ class StoreImportStep implements ImportStep {
   public boolean handle(TarArchiveEntry entry, ImportState state, InputStream inputStream) {
     if (entry.getName().equals(STORE_DATA_FILE_NAME) && !entry.isDirectory()) {
       LOG.trace("Importing store from tar");
+      state.getLogger().step("importing stores");
       // Inside the repository tar archive stream is another tar archive.
       // The nested tar archive is wrapped in another TarArchiveInputStream inside the storeImporter
-      importStores(state.getRepository(), inputStream);
+      importStores(state.getRepository(), inputStream, state.getLogger());
       state.storeImported();
       return true;
     }
     return false;
   }
 
-  private void importStores(Repository repository, InputStream inputStream) {
-    storeImporter.importFromTarArchive(repository, inputStream);
+  private void importStores(Repository repository, InputStream inputStream, RepositoryImportLogger logger) {
+    storeImporter.importFromTarArchive(repository, inputStream, logger);
     updateEngine.update(repository.getId());
   }
 }

scm-webapp/src/main/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporter.java

@@ -47,36 +47,40 @@ public class TarArchiveRepositoryStoreImporter {
     this.repositoryStoreImporter = repositoryStoreImporter;
   }
 
-  public void importFromTarArchive(Repository repository, InputStream inputStream) {
+  public void importFromTarArchive(Repository repository, InputStream inputStream, RepositoryImportLogger logger) {
     try (TarArchiveInputStream tais = new NoneClosingTarArchiveInputStream(inputStream)) {
       ArchiveEntry entry = tais.getNextEntry();
       while (entry != null) {
         String[] entryPathParts = entry.getName().split(File.separator);
         validateStorePath(repository, entryPathParts);
-        importStoreByType(repository, tais, entryPathParts);
+        importStoreByType(repository, tais, entryPathParts, logger);
         entry = tais.getNextEntry();
       }
     } catch (IOException e) {
-      throw  new ImportFailedException(ContextEntry.ContextBuilder.entity(repository).build(), "Could not import stores from metadata file.", e);
+      throw new ImportFailedException(ContextEntry.ContextBuilder.entity(repository).build(), "Could not import stores from metadata file.", e);
     }
   }
 
-  private void importStoreByType(Repository repository, TarArchiveInputStream tais, String[] entryPathParts) {
+  private void importStoreByType(Repository repository, TarArchiveInputStream tais, String[] entryPathParts, RepositoryImportLogger logger) {
     String storeType = entryPathParts[1];
+    String storeName = entryPathParts[2];
     if (isDataStore(storeType)) {
+      logger.step("importing data store entry for store " + storeName);
       repositoryStoreImporter
         .doImport(repository)
         .importStore(new StoreEntryMetaData(StoreType.DATA, entryPathParts[2]))
         .importEntry(entryPathParts[3], tais);
     } else if (isConfigStore(storeType)){
+      logger.step("importing data store entry for store " + storeName);
       repositoryStoreImporter
         .doImport(repository)
         .importStore(new StoreEntryMetaData(StoreType.CONFIG, ""))
-        .importEntry(entryPathParts[2], tais);
+        .importEntry(storeName, tais);
     } else if(isBlobStore(storeType)) {
+      logger.step("importing blob store entry for store " + storeName);
       repositoryStoreImporter
         .doImport(repository)
-        .importStore(new StoreEntryMetaData(StoreType.BLOB, entryPathParts[2]))
+        .importStore(new StoreEntryMetaData(StoreType.BLOB, storeName))
         .importEntry(entryPathParts[3], tais);
     }
   }

scm-webapp/src/test/java/sonia/scm/importexport/FullScmRepositoryImporterTest.java

@@ -94,6 +94,10 @@ class FullScmRepositoryImporterTest {
   private RepositoryImportExportEncryption repositoryImportExportEncryption;
   @Mock
   private WorkdirProvider workdirProvider;
+  @Mock
+  private RepositoryImportLogger logger;
+  @Mock
+  private RepositoryImportLoggerFactory loggerFactory;
 
   @InjectMocks
   private EnvironmentCheckStep environmentCheckStep;
@@ -124,6 +128,7 @@ class FullScmRepositoryImporterTest {
       repositoryImportStep,
       repositoryManager,
       repositoryImportExportEncryption,
+      loggerFactory,
       eventBus
     );
   }
@@ -132,6 +137,7 @@ class FullScmRepositoryImporterTest {
   void initRepositoryService() {
     lenient().when(serviceFactory.create(REPOSITORY)).thenReturn(service);
     lenient().when(service.getUnbundleCommand()).thenReturn(unbundleCommandBuilder);
+    lenient().when(loggerFactory.createLogger()).thenReturn(logger);
   }
 
   @Test
@@ -174,7 +180,7 @@ class FullScmRepositoryImporterTest {
       Repository repository = fullImporter.importFromStream(REPOSITORY, stream, "");
 
       assertThat(repository).isEqualTo(REPOSITORY);
-      verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class));
+      verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class), eq(logger));
       verify(repositoryManager).modify(REPOSITORY);
       Collection<RepositoryPermission> updatedPermissions = REPOSITORY.getPermissions();
       assertThat(updatedPermissions).hasSize(2);
@@ -207,7 +213,7 @@ class FullScmRepositoryImporterTest {
       Repository repository = fullImporter.importFromStream(REPOSITORY, stream, "");
 
       assertThat(repository).isEqualTo(REPOSITORY);
-      verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class));
+      verify(storeImporter).importFromTarArchive(eq(REPOSITORY), any(InputStream.class), eq(logger));
       verify(repositoryManager).modify(REPOSITORY);
       verify(unbundleCommandBuilder).unbundle((InputStream) argThat(argument -> argument.getClass().equals(NoneClosingInputStream.class)));
       verify(workdirProvider, never()).createNewWorkdir(REPOSITORY.getId());

scm-webapp/src/test/java/sonia/scm/importexport/TarArchiveRepositoryStoreImporterTest.java

@@ -53,6 +53,8 @@ class TarArchiveRepositoryStoreImporterTest {
 
   @Mock(answer = Answers.RETURNS_DEEP_STUBS)
   private RepositoryStoreImporter repositoryStoreImporter;
+  @Mock
+  private RepositoryImportLogger logger;
 
   @InjectMocks
   private TarArchiveRepositoryStoreImporter tarArchiveRepositoryStoreImporter;
@@ -60,20 +62,20 @@ class TarArchiveRepositoryStoreImporterTest {
   @Test
   void shouldDoNothingIfNoEntries() {
     ByteArrayInputStream bais = new ByteArrayInputStream("".getBytes());
-    tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, bais);
+    tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, bais, logger);
     verify(repositoryStoreImporter, never()).doImport(any(Repository.class));
   }
 
   @Test
   void shouldImportEachEntry() throws IOException {
     InputStream inputStream = Resources.getResource("sonia/scm/repository/import/scm-metadata.tar").openStream();
-    tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, inputStream);
+    tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, inputStream, logger);
     verify(repositoryStoreImporter, times(2)).doImport(repository);
   }
 
   @Test
   void shouldThrowImportFailedExceptionIfInvalidStorePath() throws IOException {
     InputStream inputStream = Resources.getResource("sonia/scm/repository/import/scm-metadata_invalid.tar").openStream();
-    assertThrows(ImportFailedException.class, () -> tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, inputStream));
+    assertThrows(ImportFailedException.class, () -> tarArchiveRepositoryStoreImporter.importFromTarArchive(repository, inputStream, logger));
   }
 }