Git import with lfs support (#2133)

This adds the possibility to load files managed by lfs to the repository import of git repositories.

Co-authored-by: Konstantin Schaper <konstantin.schaper@cloudogu.com>

scm-webapp/src/main/java/sonia/scm/api/v2/resources/NotificationDto.java

@@ -31,6 +31,7 @@ import sonia.scm.notifications.StoredNotification;
 import sonia.scm.notifications.Type;
 
 import java.time.Instant;
+import java.util.Map;
 
 @Data
 public class NotificationDto extends HalRepresentation {
@@ -39,12 +40,14 @@ public class NotificationDto extends HalRepresentation {
   private Type type;
   private String link;
   private String message;
+  private Map<String, String> parameters;
 
   public NotificationDto(StoredNotification notification, Links links) {
     super(links);
     this.type = notification.getType();
     this.link = notification.getLink();
     this.message = notification.getMessage();
+    this.parameters = notification.getParameters();
     this.createdAt = notification.getCreatedAt();
   }
 }

scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java

@@ -383,6 +383,7 @@ public class RepositoryImportResource {
     private String importUrl;
     private String username;
     private String password;
+    private boolean skipLfs;
   }
 
   @Getter
@@ -400,6 +401,8 @@ public class RepositoryImportResource {
     String getUsername();
 
     String getPassword();
+
+    boolean isSkipLfs();
   }
 
   interface ImportRepositoryFromFileDto extends CreateRepositoryDto {

scm-webapp/src/main/java/sonia/scm/importexport/FromUrlImporter.java

@@ -42,6 +42,7 @@ import sonia.scm.repository.RepositoryType;
 import sonia.scm.repository.api.Command;
 import sonia.scm.repository.api.ImportFailedException;
 import sonia.scm.repository.api.PullCommandBuilder;
+import sonia.scm.repository.api.PullResponse;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
 
@@ -63,13 +64,15 @@ public class FromUrlImporter {
   private final RepositoryServiceFactory serviceFactory;
   private final ScmEventBus eventBus;
   private final RepositoryImportLoggerFactory loggerFactory;
+  private final ImportNotificationHandler notificationHandler;
 
   @Inject
-  public FromUrlImporter(RepositoryManager manager, RepositoryServiceFactory serviceFactory, ScmEventBus eventBus, RepositoryImportLoggerFactory loggerFactory) {
+  public FromUrlImporter(RepositoryManager manager, RepositoryServiceFactory serviceFactory, ScmEventBus eventBus, RepositoryImportLoggerFactory loggerFactory, ImportNotificationHandler notificationHandler) {
     this.manager = manager;
     this.serviceFactory = serviceFactory;
     this.eventBus = eventBus;
     this.loggerFactory = loggerFactory;
+    this.notificationHandler = notificationHandler;
   }
 
   public Repository importFromUrl(RepositoryImportParameters parameters, Repository repository) {
@@ -112,21 +115,35 @@ public class FromUrlImporter {
             .withUsername(parameters.getUsername())
             .withPassword(parameters.getPassword());
         }
+        pullCommand.doFetchLfs(!parameters.isSkipLfs());
 
         logger.step("pulling repository from " + parameters.getImportUrl());
-        pullCommand.pull(parameters.getImportUrl());
+        PullResponse pullResponse = pullCommand.pull(parameters.getImportUrl());
         logger.finished();
+        handle(pullResponse, repository);
       } catch (IOException e) {
         throw new InternalRepositoryException(repository, "Failed to import from remote url: " + e.getMessage(), e);
+      } catch (ImportFailedException e) {
+        notificationHandler.handleFailedImport();
+        throw e;
       }
     };
   }
 
+  private void handle(PullResponse pullResponse, Repository repository) {
+    if (pullResponse.getLfsCount().getFailureCount() == 0) {
+      notificationHandler.handleSuccessfulImport(repository, pullResponse.getLfsCount());
+    } else {
+      notificationHandler.handleSuccessfulImportWithLfsFailures(repository, pullResponse.getLfsCount());
+    }
+  }
+
   @Getter
   @Setter
   public static class RepositoryImportParameters {
     private String importUrl;
     private String username;
     private String password;
+    private boolean skipLfs;
   }
 }

scm-webapp/src/main/java/sonia/scm/importexport/ImportNotificationHandler.java

@@ -0,0 +1,88 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.importexport;
+
+import sonia.scm.notifications.Notification;
+import sonia.scm.notifications.NotificationSender;
+import sonia.scm.notifications.Type;
+import sonia.scm.repository.Repository;
+import sonia.scm.repository.api.PullResponse;
+
+import javax.inject.Inject;
+import java.util.Map;
+
+class ImportNotificationHandler {
+
+  private final NotificationSender notificationSender;
+
+  @Inject
+  ImportNotificationHandler(NotificationSender notificationSender) {
+    this.notificationSender = notificationSender;
+  }
+
+  void handleSuccessfulImport(Repository repository) {
+    handleSuccessfulImport(repository, new PullResponse.LfsCount(0, 0));
+  }
+
+  void handleSuccessfulImport(Repository repository, PullResponse.LfsCount lfsCount) {
+    notificationSender.send(getImportSuccessfulNotification(repository, lfsCount));
+  }
+
+  void handleSuccessfulImportWithLfsFailures(Repository repository, PullResponse.LfsCount lfsCount) {
+    notificationSender.send(getImportLfsFailedNotification(repository, lfsCount));
+  }
+
+  void handleFailedImport() {
+    notificationSender.send(getImportFailedNotification());
+  }
+
+  private Notification getImportSuccessfulNotification(Repository repository, PullResponse.LfsCount lfsCount) {
+    if (lfsCount.getSuccessCount() > 0 || lfsCount.getFailureCount() > 0) {
+      return new Notification(Type.SUCCESS, createLink(repository), "importWithLfsFinished", createParameters(lfsCount));
+    } else {
+      return new Notification(Type.SUCCESS, createLink(repository), "importFinished");
+    }
+  }
+
+  private Notification getImportLfsFailedNotification(Repository repository, PullResponse.LfsCount lfsCount) {
+    return new Notification(Type.ERROR, createLink(repository), "importLfsFailed", createParameters(lfsCount));
+  }
+
+  private static Map<String, String> createParameters(PullResponse.LfsCount lfsCount) {
+    return Map.of(
+      "successCount", Integer.toString(lfsCount.getSuccessCount()),
+      "failureCount", Integer.toString(lfsCount.getFailureCount()),
+      "overallCount", Integer.toString(lfsCount.getSuccessCount() + lfsCount.getFailureCount())
+    );
+  }
+
+  private Notification getImportFailedNotification() {
+    return new Notification(Type.ERROR, null, "importFailed");
+  }
+
+  private static String createLink(Repository repository) {
+    return "/repo/" + repository.getNamespaceAndName();
+  }
+}

scm-webapp/src/main/java/sonia/scm/notifications/StoredNotification.java

@@ -33,6 +33,7 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 import java.time.Instant;
+import java.util.Map;
 
 @Data
 @NoArgsConstructor
@@ -44,6 +45,7 @@ public class StoredNotification {
   Type type;
   String link;
   String message;
+  Map<String, String> parameters;
   @XmlJavaTypeAdapter(XmlInstantAdapter.class)
   Instant createdAt;
 
@@ -53,5 +55,6 @@ public class StoredNotification {
     this.type = notification.getType();
     this.link = notification.getLink();
     this.message = notification.getMessage();
+    this.parameters = notification.getParameters();
   }
 }