User blob store instead of data store

scm-test/src/main/java/sonia/scm/store/InMemoryBlobStoreFactory.java

@@ -0,0 +1,60 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.store;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class InMemoryBlobStoreFactory implements BlobStoreFactory {
+
+  private final Map<String, BlobStore> stores = new HashMap<>();
+
+  private final BlobStore fixedStore;
+
+  public InMemoryBlobStoreFactory() {
+    this(null);
+  }
+
+  public InMemoryBlobStoreFactory(BlobStore fixedStore) {
+    this.fixedStore = fixedStore;
+  }
+
+  @Override
+  public BlobStore getStore(StoreParameters storeParameters) {
+    if (fixedStore == null) {
+      return stores.computeIfAbsent(computeKey(storeParameters), key -> new InMemoryBlobStore());
+    } else {
+      return fixedStore;
+    }
+  }
+
+  private String computeKey(StoreParameters storeParameters) {
+    if (storeParameters.getRepositoryId() == null) {
+      return storeParameters.getName();
+    } else {
+      return storeParameters.getName() + "/" + storeParameters.getRepositoryId();
+    }
+  }
+}

scm-webapp/src/main/java/sonia/scm/importexport/FromBundleImporter.java

@@ -50,7 +50,7 @@ import java.io.InputStream;
 import java.util.function.Consumer;
 
 import static java.util.Collections.singletonList;
-import static sonia.scm.importexport.RepositoryImportLog.ImportType.DUMP;
+import static sonia.scm.importexport.RepositoryImportLogger.ImportType.DUMP;
 import static sonia.scm.importexport.RepositoryTypeSupportChecker.checkSupport;
 import static sonia.scm.importexport.RepositoryTypeSupportChecker.type;
 

scm-webapp/src/main/java/sonia/scm/importexport/FromUrlImporter.java

@@ -30,6 +30,7 @@ import lombok.Setter;
 import org.apache.shiro.SecurityUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import sonia.scm.AlreadyExistsException;
 import sonia.scm.HandlerEventType;
 import sonia.scm.Type;
 import sonia.scm.event.ScmEventBus;
@@ -51,6 +52,7 @@ import java.util.function.Consumer;
 
 import static java.util.Collections.singletonList;
 import static sonia.scm.ContextEntry.ContextBuilder.noContext;
+import static sonia.scm.importexport.RepositoryImportLogger.ImportType.URL;
 import static sonia.scm.importexport.RepositoryTypeSupportChecker.checkSupport;
 import static sonia.scm.importexport.RepositoryTypeSupportChecker.type;
 
@@ -87,8 +89,12 @@ public class FromUrlImporter {
         repository,
         pullChangesFromRemoteUrl(parameters, logger)
       );
+    } catch (AlreadyExistsException e) {
+      throw e;
     } catch (Exception e) {
-      logger.failed(e);
+      if (logger.started()) {
+        logger.failed(e);
+      }
       eventBus.post(new RepositoryImportEvent(HandlerEventType.CREATE, repository, true));
       throw new ImportFailedException(noContext(), "Could not import repository from url " + parameters.getImportUrl(), e);
     }
@@ -98,7 +104,7 @@ public class FromUrlImporter {
 
   private Consumer<Repository> pullChangesFromRemoteUrl(RepositoryImportParameters parameters, RepositoryImportLogger logger) {
     return repository -> {
-      logger.start(RepositoryImportLog.ImportType.URL, repository);
+      logger.start(URL, repository);
       try (RepositoryService service = serviceFactory.create(repository)) {
         PullCommandBuilder pullCommand = service.getPullCommand();
         if (!Strings.isNullOrEmpty(parameters.getUsername()) && !Strings.isNullOrEmpty(parameters.getPassword())) {

scm-webapp/src/main/java/sonia/scm/importexport/FullScmRepositoryImporter.java

@@ -46,7 +46,7 @@ import java.io.InputStream;
 
 import static java.util.Arrays.stream;
 import static sonia.scm.ContextEntry.ContextBuilder.noContext;
-import static sonia.scm.importexport.RepositoryImportLog.ImportType.FULL;
+import static sonia.scm.importexport.RepositoryImportLogger.ImportType.FULL;
 import static sonia.scm.util.Archives.createTarInputStream;
 
 public class FullScmRepositoryImporter {

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLog.java

@@ -1,112 +0,0 @@
-/*
- * MIT License
- *
- * Copyright (c) 2020-present Cloudogu GmbH and Contributors
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package sonia.scm.importexport;
-
-import lombok.Getter;
-import lombok.Setter;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import static java.lang.String.format;
-import static java.util.Arrays.asList;
-import static java.util.Collections.unmodifiableList;
-
-@XmlRootElement(name = "import")
-@XmlAccessorType(XmlAccessType.FIELD)
-@Getter
-@Setter
-class RepositoryImportLog {
-
-  private ImportType type;
-  private String repositoryType;
-  private String userName;
-  private String userId;
-  private String repositoryId;
-  private String namespace;
-  private String name;
-  private Boolean success;
-  @XmlElement(name = "entry")
-  private List<Entry> entries;
-
-  void addEntry(Entry entry) {
-    if (entries == null) {
-      entries = new ArrayList<>();
-    }
-    this.entries.add(entry);
-  }
-
-  public List<Entry> getEntries() {
-    return unmodifiableList(entries);
-  }
-
-  public List<String> toLogHeader() {
-    return asList(
-      format("Import of repository %s/%s", namespace, name),
-      format("Repository type: %s", repositoryId),
-      format("Imported from: %s", type),
-      format("Imported by %s (%s)", userId, userName),
-      status()
-    );
-  }
-
-  private String status() {
-    if (success == null) {
-      return "Not finished";
-    } else if (success) {
-      return "Finished successful";
-    } else {
-      return "Import failed";
-    }
-  }
-
-  enum ImportType {
-    FULL, URL, DUMP
-  }
-
-  @XmlRootElement(name = "entry")
-  @XmlAccessorType(XmlAccessType.FIELD)
-  @SuppressWarnings("java:S1068") // unused fields will be serialized to xml
-  static class Entry {
-    private Date time = new Date();
-    private String message;
-
-    Entry() {
-    }
-
-    Entry(String message) {
-      this.message = message;
-    }
-
-    public String toLogMessage() {
-      return time + " - " + message;
-    }
-  }
-}

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLogger.java

@@ -25,65 +25,103 @@
 package sonia.scm.importexport;
 
 import org.apache.shiro.SecurityUtils;
-import sonia.scm.importexport.RepositoryImportLog.ImportType;
 import sonia.scm.repository.Repository;
-import sonia.scm.store.DataStore;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
 import sonia.scm.user.User;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.time.Instant;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 class RepositoryImportLogger {
 
-  private final DataStore<RepositoryImportLog> logStore;
-  private RepositoryImportLog log;
-  private String repositoryId;
+  private final BlobStore logStore;
+  private PrintWriter print;
+  private Blob blob;
 
-  RepositoryImportLogger(DataStore<RepositoryImportLog> logStore) {
+  RepositoryImportLogger(BlobStore logStore) {
     this.logStore = logStore;
   }
 
   void start(ImportType importType, Repository repository) {
     User user = SecurityUtils.getSubject().getPrincipals().oneByType(User.class);
-    repositoryId = repository.getId();
-    log = new RepositoryImportLog();
-    log.setType(importType);
-    log.setUserId(user.getId());
-    log.setUserName(user.getName());
-    log.setRepositoryType(repository.getType());
-    log.setNamespace(repository.getNamespace());
-    log.setName(repository.getName());
-    log.setRepositoryType(repository.getType());
-    logStore.put(repositoryId, log);
-    addLogEntry(new RepositoryImportLog.Entry("import started"));
+    blob = logStore.create(repository.getId());
+    OutputStream outputStream = getBlobOutputStream();
+    writeUser(user, outputStream);
+    print = new PrintWriter(outputStream);
+    print.printf("Import of repository %s/%s%n", repository.getNamespace(), repository.getName());
+    print.printf("Repository type: %s%n", repository.getType());
+    print.printf("Imported from: %s%n", importType);
+    print.printf("Imported by %s (%s)%n", user.getId(), user.getName());
+    print.println();
+
+    addLogEntry("import started");
+  }
+
+  private void writeUser(User user, OutputStream outputStream) {
+    try {
+      outputStream.write(user.getId().getBytes(UTF_8));
+      outputStream.write(0);
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+  }
+
+  private OutputStream getBlobOutputStream() {
+    try {
+      return blob.getOutputStream();
+    } catch (IOException e) {
+      e.printStackTrace();
+      return new OutputStream() {
+        @Override
+        public void write(int b) {
+          // this is a dummy
+        }
+      };
+    }
   }
 
   public void finished() {
-    log.setSuccess(true);
     step("import finished successfully");
     writeLog();
   }
 
   public void failed(Exception e) {
-    log.setSuccess(false);
     step("import failed (see next log entry)");
-    step(e.getMessage());
+    print.println(e.getMessage());
     writeLog();
   }
 
   public void repositoryCreated(Repository createdRepository) {
-    log.setNamespace(createdRepository.getNamespace());
-    log.setName(createdRepository.getName());
-    log.setRepositoryId(createdRepository.getId());
     step("created repository: " + createdRepository.getNamespaceAndName());
   }
 
   public void step(String message) {
-    addLogEntry(new RepositoryImportLog.Entry(message));
+    addLogEntry(message);
   }
 
   private void writeLog() {
-    logStore.put(repositoryId, log);
+    print.flush();
+    try {
+      blob.commit();
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
   }
 
-  private void addLogEntry(RepositoryImportLog.Entry entry) {
-    log.addEntry(entry);
+  private void addLogEntry(String message) {
+    print.printf("%s - %s%n", Instant.now(), message);
+  }
+
+  public boolean started() {
+    return blob != null;
+  }
+
+  enum ImportType {
+    FULL, URL, DUMP
   }
 }

scm-webapp/src/main/java/sonia/scm/importexport/RepositoryImportLoggerFactory.java

@@ -28,42 +28,54 @@ import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.authz.AuthorizationException;
 import org.apache.shiro.subject.Subject;
 import sonia.scm.NotFoundException;
-import sonia.scm.store.DataStore;
-import sonia.scm.store.DataStoreFactory;
+import sonia.scm.store.BlobStore;
+import sonia.scm.store.BlobStoreFactory;
+import sonia.scm.util.IOUtil;
 
 import javax.inject.Inject;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.PrintStream;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class RepositoryImportLoggerFactory {
 
-  private final DataStoreFactory dataStoreFactory;
+  private final BlobStoreFactory blobStoreFactory;
 
   @Inject
-  RepositoryImportLoggerFactory(DataStoreFactory dataStoreFactory) {
-    this.dataStoreFactory = dataStoreFactory;
+  RepositoryImportLoggerFactory(BlobStoreFactory blobStoreFactory) {
+    this.blobStoreFactory = blobStoreFactory;
   }
 
   RepositoryImportLogger createLogger() {
-    return new RepositoryImportLogger(dataStoreFactory.withType(RepositoryImportLog.class).withName("imports").build());
+    return new RepositoryImportLogger(blobStoreFactory.withName("imports").build());
   }
 
-  public void getLog(String logId, OutputStream out) {
-    DataStore<RepositoryImportLog> importStore = dataStoreFactory.withType(RepositoryImportLog.class).withName("imports").build();
-    RepositoryImportLog log = importStore.getOptional(logId).orElseThrow(() -> new NotFoundException("Log", logId));
+  public void getLog(String logId, OutputStream out) throws IOException {
+    BlobStore importStore = blobStoreFactory.withName("imports").build();
+    InputStream log = importStore
+      .getOptional(logId).orElseThrow(() -> new NotFoundException("Log", logId))
+      .getInputStream();
     checkPermission(log);
-    PrintStream printStream = new PrintStream(out);
-    log.toLogHeader().forEach(printStream::println);
-    log.getEntries()
-      .stream()
-      .map(RepositoryImportLog.Entry::toLogMessage)
-      .forEach(printStream::println);
+    IOUtil.copy(log, out);
   }
 
-  private void checkPermission(RepositoryImportLog log) {
+  private void checkPermission(InputStream log) throws IOException {
     Subject subject = SecurityUtils.getSubject();
-    if (!subject.isPermitted("only:admin:allowed") && !subject.getPrincipal().toString().equals(log.getUserId())) {
+    String logUser = readUserFrom(log);
+    if (!subject.isPermitted("only:admin:allowed") && !subject.getPrincipal().toString().equals(logUser)) {
       throw new AuthorizationException("not permitted");
     }
   }
+
+  private String readUserFrom(InputStream log) throws IOException {
+    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+    int b;
+    while ((b = log.read()) > 0) {
+      buffer.write(b);
+    }
+    return new String(buffer.toByteArray(), UTF_8);
+  }
 }

scm-webapp/src/test/java/sonia/scm/importexport/RepositoryImportLoggerFactoryTest.java

@@ -24,6 +24,7 @@
 
 package sonia.scm.importexport;
 
+import com.google.common.io.Resources;
 import org.apache.shiro.authz.AuthorizationException;
 import org.apache.shiro.subject.Subject;
 import org.apache.shiro.util.ThreadContext;
@@ -31,10 +32,12 @@ import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import sonia.scm.NotFoundException;
-import sonia.scm.store.InMemoryDataStore;
-import sonia.scm.store.InMemoryDataStoreFactory;
+import sonia.scm.store.Blob;
+import sonia.scm.store.InMemoryBlobStore;
+import sonia.scm.store.InMemoryBlobStoreFactory;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -47,8 +50,8 @@ class RepositoryImportLoggerFactoryTest {
 
   private final Subject subject = mock(Subject.class);
 
-  private final InMemoryDataStore<RepositoryImportLog> store = new InMemoryDataStore<>();
-  private final RepositoryImportLoggerFactory factory = new RepositoryImportLoggerFactory(new InMemoryDataStoreFactory(store));
+  private final InMemoryBlobStore store = new InMemoryBlobStore();
+  private final RepositoryImportLoggerFactory factory = new RepositoryImportLoggerFactory(new InMemoryBlobStoreFactory(store));
 
   @BeforeEach
   void initSubject() {
@@ -61,7 +64,7 @@ class RepositoryImportLoggerFactoryTest {
   }
 
   @Test
-  void shouldReadLogForExportingUser() {
+  void shouldReadLogForExportingUser() throws IOException {
     when(subject.getPrincipal()).thenReturn("dent");
 
     createLog();
@@ -70,19 +73,11 @@ class RepositoryImportLoggerFactoryTest {
 
     factory.getLog("42", out);
 
-    assertThat(out).asString().contains(
-      "Import of repository hitchhiker/HeartOfGold",
-      "Repository type: null",
-      "Imported from: null",
-      "Imported by dent (Arthur Dent)",
-      "Finished successful"
-    )
-      .containsPattern(".+ - import started")
-      .containsPattern(".+ - import finished");
+    assertLogReadCorrectly(out);
   }
 
   @Test
-  void shouldReadLogForAdmin() {
+  void shouldReadLogForAdmin() throws IOException {
     when(subject.getPrincipal()).thenReturn("trillian");
     when(subject.isPermitted(anyString())).thenReturn(true);
 
@@ -92,15 +87,20 @@ class RepositoryImportLoggerFactoryTest {
 
     factory.getLog("42", out);
 
+    assertLogReadCorrectly(out);
+  }
+
+  private void assertLogReadCorrectly(ByteArrayOutputStream out) {
     assertThat(out).asString().contains(
       "Import of repository hitchhiker/HeartOfGold",
-      "Repository type: null",
-      "Imported from: null",
+      "Repository type: git",
+      "Imported from: URL",
       "Imported by dent (Arthur Dent)",
-      "Finished successful"
-    )
-      .containsPattern(".+ - import started")
-      .containsPattern(".+ - import finished");
+      "",
+      "Thu Feb 25 11:11:07 CET 2021 - import started",
+      "Thu Feb 25 11:11:07 CET 2021 - pulling repository from https://github.com/scm-manager/scm-manager",
+      "Thu Feb 25 11:11:08 CET 2021 - import finished successfully"
+    );
   }
 
   @Test
@@ -111,7 +111,7 @@ class RepositoryImportLoggerFactoryTest {
   }
 
   @Test
-  void shouldFailWithoutPermission() {
+  void shouldFailWithoutPermission() throws IOException {
     when(subject.getPrincipal()).thenReturn("trillian");
     createLog();
 
@@ -122,18 +122,12 @@ class RepositoryImportLoggerFactoryTest {
     assertThrows(AuthorizationException.class, () -> factory.getLog("42", out));
   }
 
-  private void createLog() {
-    RepositoryImportLog log = new RepositoryImportLog();
-    log.setRepositoryType("git");
-    log.setNamespace("hitchhiker");
-    log.setName("HeartOfGold");
-    log.setUserId("dent");
-    log.setUserName("Arthur Dent");
-    log.setSuccess(true);
-
-    log.addEntry(new RepositoryImportLog.Entry("import started"));
-    log.addEntry(new RepositoryImportLog.Entry("import finished"));
-
-    store.put("42", log);
+  @SuppressWarnings("UnstableApiUsage")
+  private void createLog() throws IOException {
+    Blob blob = store.create("42");
+    Resources.copy(
+      Resources.getResource("sonia/scm/importexport/importLog.blob"),
+      blob.getOutputStream());
+    blob.commit();
   }
 }