Git import with lfs support (#2133)

This adds the possibility to load files managed by lfs to the repository import of git repositories.

Co-authored-by: Konstantin Schaper <konstantin.schaper@cloudogu.com>

scm-webapp/src/test/java/sonia/scm/api/v2/resources/NotificationResourceTest.java

@@ -33,7 +33,6 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
-import sonia.scm.notifications.Notification;
 import sonia.scm.notifications.NotificationStore;
 import sonia.scm.notifications.StoredNotification;
 import sonia.scm.notifications.Type;
@@ -50,6 +49,7 @@ import java.util.List;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -162,7 +162,7 @@ class NotificationResourceTest {
   }
 
   private StoredNotification notification(String m) {
-    return new StoredNotification(UUID.randomUUID().toString(), Type.INFO, "/notify", m, Instant.now());
+    return new StoredNotification(UUID.randomUUID().toString(), Type.INFO, "/notify", m, emptyMap(), Instant.now());
   }
 
   @Path("/api/v2")

scm-webapp/src/test/java/sonia/scm/importexport/FromUrlImporterTest.java

@@ -28,8 +28,10 @@ import org.apache.shiro.subject.Subject;
 import org.apache.shiro.util.ThreadContext;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Answers;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
@@ -42,6 +44,7 @@ import sonia.scm.repository.RepositoryTestData;
 import sonia.scm.repository.RepositoryType;
 import sonia.scm.repository.api.ImportFailedException;
 import sonia.scm.repository.api.PullCommandBuilder;
+import sonia.scm.repository.api.PullResponse;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
 
@@ -54,7 +57,7 @@ import static org.junit.Assert.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.argThat;
-import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -78,11 +81,18 @@ class FromUrlImporterTest {
   private RepositoryImportLogger logger;
   @Mock
   private Subject subject;
+  @Mock(answer = Answers.RETURNS_SELF)
+  private PullCommandBuilder pullCommandBuilder;
+  @Mock
+  private ImportNotificationHandler notificationHandler;
 
   @InjectMocks
   private FromUrlImporter importer;
 
   private final Repository repository = RepositoryTestData.createHeartOfGold("git");
+  private Repository createdRepository;
+
+  private PullResponse mockedResponse = new PullResponse();
 
   @BeforeEach
   void setUpMocks() {
@@ -91,12 +101,13 @@ class FromUrlImporterTest {
     when(manager.create(any(), any())).thenAnswer(
       invocation -> {
         Repository repository = invocation.getArgument(0, Repository.class);
-        Repository createdRepository = repository.clone();
+        createdRepository = repository.clone();
         createdRepository.setNamespace("created");
         invocation.getArgument(1, Consumer.class).accept(createdRepository);
         return createdRepository;
       }
     );
+    when(service.getPullCommand()).thenReturn(pullCommandBuilder);
   }
 
   @BeforeEach
@@ -119,50 +130,92 @@ class FromUrlImporterTest {
     ThreadContext.unbindSubject();
   }
 
-  @Test
-  void shouldPullChangesFromRemoteUrl() throws IOException {
-    PullCommandBuilder pullCommandBuilder = mock(PullCommandBuilder.class, RETURNS_SELF);
-    when(service.getPullCommand()).thenReturn(pullCommandBuilder);
+  @Nested
+  class ForSuccessfulImports {
 
-    FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
-    parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+    @BeforeEach
+    void mockImportResult() throws IOException {
+      when(pullCommandBuilder.pull(anyString())).thenAnswer(invocation -> mockedResponse);
+    }
 
-    Repository createdRepository = importer.importFromUrl(parameters, repository);
+    @Test
+    void shouldPullChangesFromRemoteUrl() throws IOException {
+      FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
+      parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
 
-    assertThat(createdRepository.getNamespace()).isEqualTo("created");
-    verify(pullCommandBuilder).pull("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
-    verify(logger).finished();
-    verify(eventBus).post(argThat(
-      event -> {
-        assertThat(event).isInstanceOf(RepositoryImportEvent.class);
-        RepositoryImportEvent repositoryImportEvent = (RepositoryImportEvent) event;
-        assertThat(repositoryImportEvent.getItem().getNamespace()).isEqualTo("created");
-        assertThat(repositoryImportEvent.isFailed()).isFalse();
-        return true;
-      }
-    ));
-  }
+      Repository createdRepository = importer.importFromUrl(parameters, repository);
 
-  @Test
-  void shouldPullChangesFromRemoteUrlWithCredentials() {
-    PullCommandBuilder pullCommandBuilder = mock(PullCommandBuilder.class, RETURNS_SELF);
-    when(service.getPullCommand()).thenReturn(pullCommandBuilder);
+      assertThat(createdRepository.getNamespace()).isEqualTo("created");
+      verify(pullCommandBuilder).pull("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+      verify(logger).finished();
+      verify(eventBus).post(argThat(
+        event -> {
+          assertThat(event).isInstanceOf(RepositoryImportEvent.class);
+          RepositoryImportEvent repositoryImportEvent = (RepositoryImportEvent) event;
+          assertThat(repositoryImportEvent.getItem().getNamespace()).isEqualTo("created");
+          assertThat(repositoryImportEvent.isFailed()).isFalse();
+          return true;
+        }
+      ));
+      verify(notificationHandler)
+        .handleSuccessfulImport(
+          eq(createdRepository),
+          argThat(argument -> argument.getSuccessCount() == 0));
+    }
 
-    FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
-    parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
-    parameters.setUsername("trillian");
-    parameters.setPassword("secret");
+    @Test
+    void shouldPullChangesFromRemoteUrlWithCredentials() {
+      FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
+      parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+      parameters.setUsername("trillian");
+      parameters.setPassword("secret");
 
-    importer.importFromUrl(parameters, repository);
+      importer.importFromUrl(parameters, repository);
 
-    verify(pullCommandBuilder).withUsername("trillian");
-    verify(pullCommandBuilder).withPassword("secret");
+      verify(pullCommandBuilder).withUsername("trillian");
+      verify(pullCommandBuilder).withPassword("secret");
+    }
+
+    @Test
+    void shouldPullChangesWithLfsIfNotDisabled() {
+      FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
+      parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+      parameters.setSkipLfs(false);
+
+      importer.importFromUrl(parameters, repository);
+
+      verify(pullCommandBuilder).doFetchLfs(true);
+    }
+
+    @Test
+    void shouldPullChangesWithoutLfsIfSelected() {
+      FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
+      parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+      parameters.setSkipLfs(true);
+
+      importer.importFromUrl(parameters, repository);
+
+      verify(pullCommandBuilder).doFetchLfs(false);
+    }
+
+    @Test
+    void shouldHandleFailedLfsFiles() {
+      FromUrlImporter.RepositoryImportParameters parameters = new FromUrlImporter.RepositoryImportParameters();
+      parameters.setImportUrl("https://scm-manager.org/scm/repo/scmadmin/scm-manager.git");
+      parameters.setSkipLfs(false);
+      mockedResponse = new PullResponse(42, new PullResponse.LfsCount(0, 1));
+
+      importer.importFromUrl(parameters, repository);
+
+      verify(notificationHandler).
+        handleSuccessfulImportWithLfsFailures(
+          eq(createdRepository),
+          argThat(argument -> argument.getFailureCount() == 1 && argument.getSuccessCount() == 0));
+    }
   }
 
   @Test
   void shouldThrowImportFailedEvent() throws IOException {
-    PullCommandBuilder pullCommandBuilder = mock(PullCommandBuilder.class, RETURNS_SELF);
-    when(service.getPullCommand()).thenReturn(pullCommandBuilder);
     doThrow(TestException.class).when(pullCommandBuilder).pull(anyString());
     when(logger.started()).thenReturn(true);
 

scm-webapp/src/test/java/sonia/scm/importexport/ImportNotificationHandlerTest.java

@@ -0,0 +1,110 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.importexport;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import sonia.scm.notifications.NotificationSender;
+import sonia.scm.notifications.Type;
+import sonia.scm.repository.Repository;
+import sonia.scm.repository.RepositoryTestData;
+import sonia.scm.repository.api.PullResponse;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+class ImportNotificationHandlerTest {
+
+  @Mock
+  private NotificationSender notificationSender;
+  @InjectMocks
+  private ImportNotificationHandler handler;
+
+  private final Repository repository = RepositoryTestData.createHeartOfGold();
+
+  @Test
+  void shouldCreateSuccessNotification() {
+    handler.handleSuccessfulImport(repository);
+
+    verify(notificationSender).send(argThat(
+      notification -> {
+        assertThat(notification.getType()).isEqualTo(Type.SUCCESS);
+        assertThat(notification.getMessage()).isEqualTo("importFinished");
+        assertThat(notification.getParameters()).isNull();
+        return true;
+      }
+    ));
+  }
+
+  @Test
+  void shouldCreateSuccessNotificationWithLfs() {
+    handler.handleSuccessfulImport(repository, new PullResponse.LfsCount(42, 0));
+
+    verify(notificationSender).send(argThat(
+      notification -> {
+        assertThat(notification.getType()).isEqualTo(Type.SUCCESS);
+        assertThat(notification.getMessage()).isEqualTo("importWithLfsFinished");
+        assertThat(notification.getParameters()).containsEntry("successCount", "42");
+        return true;
+      }
+    ));
+  }
+
+  @Test
+  void shouldCreateFailureNotification() {
+    handler.handleFailedImport();
+
+    verify(notificationSender).send(argThat(
+      notification -> {
+        assertThat(notification.getType()).isEqualTo(Type.ERROR);
+        assertThat(notification.getMessage()).isEqualTo("importFailed");
+        assertThat(notification.getParameters()).isNull();
+        return true;
+      }
+    ));
+  }
+
+  @Test
+  void shouldCreateLfsFailureNotification() {
+    handler.handleSuccessfulImportWithLfsFailures(repository, new PullResponse.LfsCount(42, 7));
+
+    verify(notificationSender).send(argThat(
+      notification -> {
+        assertThat(notification.getType()).isEqualTo(Type.ERROR);
+        assertThat(notification.getMessage()).isEqualTo("importLfsFailed");
+        assertThat(notification.getParameters())
+          .containsEntry("successCount", "42")
+          .containsEntry("failureCount", "7")
+          .containsEntry("overallCount", "49");
+        return true;
+      }
+    ));
+  }
+}