Add bundle endpoint to repository import resource

scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java

@@ -24,8 +24,13 @@
 
 package sonia.scm.api.v2.resources;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
+import com.google.common.io.ByteSource;
+import com.google.common.io.Files;
 import com.google.inject.Inject;
 import de.otto.edison.hal.Embedded;
 import de.otto.edison.hal.Links;
@@ -37,8 +42,13 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.shiro.SecurityUtils;
+import org.jboss.resteasy.plugins.providers.multipart.InputPart;
+import org.jboss.resteasy.plugins.providers.multipart.MultipartFormDataInput;
+import org.jboss.resteasy.plugins.providers.multipart.MultipartInputImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import sonia.scm.BadRequestException;
+import sonia.scm.ContextEntry;
 import sonia.scm.HandlerEventType;
 import sonia.scm.Type;
 import sonia.scm.event.ScmEventBus;
@@ -54,6 +64,7 @@ import sonia.scm.repository.api.Command;
 import sonia.scm.repository.api.PullCommandBuilder;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
+import sonia.scm.util.IOUtil;
 import sonia.scm.util.ValidationUtil;
 import sonia.scm.web.VndMediaType;
 
@@ -62,18 +73,30 @@ import javax.validation.constraints.Email;
 import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.Pattern;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriInfo;
+import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Consumer;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Collections.singletonList;
 
 public class RepositoryImportResource {
@@ -188,6 +211,139 @@ public class RepositoryImportResource {
     };
   }
 
+  /**
+   * Imports a external repository via dump. The method can
+   * only be used, if the repository type supports the {@link Command#UNBUNDLE}. The
+   * method will return a location header with the url to the imported
+   * repository.
+   *
+   * @param uriInfo     uri info
+   * @param type        repository type
+   * @param input       multi part form data which should contain a valid repository dto and the input stream of the bundle
+   * @param compressed  true if the bundle is gzip compressed
+   * @return empty response with location header which points to the imported
+   * repository
+   * @since 2.12.0
+   */
+  @POST
+  @Path("{type}/bundle")
+  @Consumes(MediaType.MULTIPART_FORM_DATA)
+  @Operation(summary = "Import repository from bundle", description = "Imports the repository from the provided bundle.", tags = "Repository")
+  @ApiResponse(
+    responseCode = "201",
+    description = "Repository import was successful"
+  )
+  @ApiResponse(
+    responseCode = "401",
+    description = "not authenticated / invalid credentials"
+  )
+  @ApiResponse(
+    responseCode = "403",
+    description = "not authorized, the current user has no privileges to read the repository"
+  )
+  @ApiResponse(
+    responseCode = "500",
+    description = "internal server error",
+    content = @Content(
+      mediaType = VndMediaType.ERROR_TYPE,
+      schema = @Schema(implementation = ErrorDto.class)
+    )
+  )
+  public Response importFromBundle(@Context UriInfo uriInfo,
+                                   @PathParam("type") String type,
+                                   MultipartFormDataInput input,
+                                   @QueryParam("compressed") @DefaultValue("false") boolean compressed) {
+    RepositoryPermissions.create().check();
+    Repository repository = doImportFromBundle(type, input, compressed);
+
+    return Response.created(URI.create(resourceLinks.repository().self(repository.getNamespace(), repository.getName()))).build();
+  }
+
+  /**
+   * Start bundle import.
+   *
+   * @param type        repository type
+   * @param input       multi part form data
+   * @param compressed  true if the bundle is gzip compressed
+   * @return imported repository
+   */
+  private Repository doImportFromBundle(String type, MultipartFormDataInput input, boolean compressed) {
+    Map<String, List<InputPart>> formParts = input.getFormDataMap();
+    RepositoryDto repositoryDto = extractFromInputPart(formParts.get("repository"), RepositoryDto.class);
+    InputStream inputStream = extractFromInputPart(formParts.get("bundle"), InputStream.class);
+
+    checkNotNull(repositoryDto, "repository data is required");
+    checkNotNull(inputStream, "bundle inputStream is required");
+    checkArgument(!Strings.isNullOrEmpty(repositoryDto.getName()), "request does not contain name of the repository");
+
+    Type t = type(type);
+
+    checkSupport(t, Command.UNBUNDLE, "bundle");
+
+    Repository repository = mapper.map(repositoryDto);
+    repository.setPermissions(singletonList(
+      new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false)
+    ));
+
+    try {
+      repository = manager.create(
+        repository,
+        unbundleImport(inputStream, compressed)
+      );
+      eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, false));
+
+    } catch (Exception e) {
+      eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, true));
+      throw e;
+    }
+
+    return repository;
+  }
+
+  @VisibleForTesting
+  Consumer<Repository> unbundleImport(InputStream inputStream, boolean compressed) {
+    return repository -> {
+      File file = null;
+      try (RepositoryService service = serviceFactory.create(repository)) {
+        file = File.createTempFile("scm-import-", ".bundle");
+        long length = Files.asByteSink(file).writeFrom(inputStream);
+        logger.info("copied {} bytes to temp, start bundle import", length);
+        service.getUnbundleCommand().setCompressed(compressed).unbundle(file);
+      } catch (IOException e) {
+        throw new InternalRepositoryException(repository, "Failed to import from bundle", e);
+      } finally {
+        try {
+          IOUtil.delete(file);
+        } catch (IOException ex) {
+          logger.warn("could not delete temporary file", ex);
+        }
+      }
+    };
+  }
+
+  private <T> T extractFromInputPart(List<InputPart> input, Class<T> type) {
+    try {
+      if (input != null && !input.isEmpty()) {
+        String content = new ByteSource() {
+          @Override
+          public InputStream openStream() throws IOException {
+            return ((MultipartInputImpl.PartImpl) input.get(0)).getBody();
+          }
+        }.asCharSource(UTF_8).read();
+        if (type == InputStream.class) {
+          return (T) new ByteArrayInputStream(StandardCharsets.UTF_8.encode(content).array());
+        }
+        try (JsonParser parser = new JsonFactory().createParser(content)) {
+          parser.setCodec(new ObjectMapper());
+          return parser.readValueAs(type);
+        }
+      }
+    } catch (IOException ex) {
+      logger.debug("Could not extract repository from input");
+    }
+    return null;
+  }
+
   /**
    * Check repository type for support for the given command.
    *
@@ -241,16 +397,23 @@ public class RepositoryImportResource {
 
   interface ImportRepositoryDto {
     String getNamespace();
+
     @Pattern(regexp = ValidationUtil.REGEX_REPOSITORYNAME)
     String getName();
+
     @NotEmpty
     String getType();
+
     @Email
     String getContact();
+
     String getDescription();
+
     @NotEmpty
     String getImportUrl();
+
     String getUsername();
+
     String getPassword();
   }
 }

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java

@@ -24,6 +24,8 @@
 
 package sonia.scm.api.v2.resources;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.sdorra.shiro.ShiroRule;
 import com.github.sdorra.shiro.SubjectAware;
 import com.google.common.collect.ImmutableSet;
@@ -58,17 +60,29 @@ import sonia.scm.repository.api.ImportFailedException;
 import sonia.scm.repository.api.PullCommandBuilder;
 import sonia.scm.repository.api.RepositoryService;
 import sonia.scm.repository.api.RepositoryServiceFactory;
+import sonia.scm.repository.api.UnbundleCommandBuilder;
+import sonia.scm.repository.api.UnbundleResponse;
 import sonia.scm.user.User;
 import sonia.scm.web.RestDispatcher;
 import sonia.scm.web.VndMediaType;
 
 import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.MediaType;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.StringWriter;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -565,6 +579,81 @@ public class RepositoryRootResourceTest extends RepositoryTestBase {
     assertThrows(ImportFailedException.class, () -> repositoryConsumer.accept(repository));
   }
 
+  @Test
+  public void shouldImportRepositoryFromBundle() throws IOException, URISyntaxException {
+    when(manager.getHandler("svn")).thenReturn(repositoryHandler);
+    when(repositoryHandler.getType()).thenReturn(new RepositoryType("svn", "svn", ImmutableSet.of(Command.UNBUNDLE)));
+    when(repositoryManager.create(any(), any())).thenReturn(RepositoryTestData.createHeartOfGold());
+
+    RepositoryDto repositoryDto = new RepositoryDto();
+    repositoryDto.setName("HeartOfGold");
+    repositoryDto.setNamespace("hitchhiker");
+    repositoryDto.setType("svn");
+
+    URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump");
+    byte[] svnDump = Resources.toByteArray(dumpUrl);
+
+    MockHttpRequest request = MockHttpRequest
+      .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "import/svn/bundle");
+    MockHttpResponse response = new MockHttpResponse();
+
+    multipartRequest(request, Collections.singletonMap("bundle", new ByteArrayInputStream(svnDump)), repositoryDto);
+
+    dispatcher.invoke(request, response);
+
+    assertEquals(HttpServletResponse.SC_CREATED, response.getStatus());
+    assertEquals("/v2/repositories/hitchhiker/HeartOfGold", response.getOutputHeaders().get("Location").get(0).toString());
+    ArgumentCaptor<RepositoryImportEvent> event = ArgumentCaptor.forClass(RepositoryImportEvent.class);
+    verify(eventBus).post(event.capture());
+    assertFalse(event.getValue().isFailed());
+  }
+
+  @Test
+  public void shouldThrowFailedEventOnImportRepositoryFromBundle() throws IOException, URISyntaxException {
+    when(manager.getHandler("svn")).thenReturn(repositoryHandler);
+    when(repositoryHandler.getType()).thenReturn(new RepositoryType("svn", "svn", ImmutableSet.of(Command.UNBUNDLE)));
+    doThrow(ImportFailedException.class).when(repositoryManager).create(any(), any());
+
+    RepositoryDto repositoryDto = new RepositoryDto();
+    repositoryDto.setName("HeartOfGold");
+    repositoryDto.setNamespace("hitchhiker");
+    repositoryDto.setType("svn");
+
+    URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump");
+    byte[] svnDump = Resources.toByteArray(dumpUrl);
+
+    MockHttpRequest request = MockHttpRequest
+      .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "import/svn/bundle");
+    MockHttpResponse response = new MockHttpResponse();
+
+    multipartRequest(request, Collections.singletonMap("bundle", new ByteArrayInputStream(svnDump)), repositoryDto);
+
+    dispatcher.invoke(request, response);
+
+    assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus());
+    ArgumentCaptor<RepositoryImportEvent> event = ArgumentCaptor.forClass(RepositoryImportEvent.class);
+    verify(eventBus).post(event.capture());
+    assertTrue(event.getValue().isFailed());
+  }
+
+  @Test
+  public void shouldImportCompressedBundle() throws IOException {
+    URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump.gz");
+    byte[] svnDump = Resources.toByteArray(dumpUrl);
+
+    UnbundleCommandBuilder ubc = mock(UnbundleCommandBuilder.class);
+    when(ubc.setCompressed(any())).thenReturn(ubc);
+    when(ubc.unbundle(any(File.class))).thenReturn(new UnbundleResponse(42));
+    RepositoryService service = mock(RepositoryService.class);
+    when(serviceFactory.create(any(Repository.class))).thenReturn(service);
+    when(service.getUnbundleCommand()).thenReturn(ubc);
+    InputStream in = new ByteArrayInputStream(svnDump);
+    repositoryImportResource.unbundleImport(in, true);
+
+    verify(ubc).setCompressed(true);
+    //TODO Enhance test
+  }
+
   private PageResult<Repository> createSingletonPageResult(Repository repository) {
     return new PageResult<>(singletonList(repository), 0);
   }
@@ -586,4 +675,49 @@ public class RepositoryRootResourceTest extends RepositoryTestBase {
     when(repositoryManager.get(id)).thenReturn(repository);
     return repository;
   }
+
+
+  /**
+   * This method is a slightly adapted copy of Lin Zaho's gist at https://gist.github.com/lin-zhao/9985191
+   */
+  private MockHttpRequest multipartRequest(MockHttpRequest request, Map<String, InputStream> files, RepositoryDto repository) throws IOException {
+    String boundary = UUID.randomUUID().toString();
+    request.contentType("multipart/form-data; boundary=" + boundary);
+
+    //Make sure this is deleted in afterTest()
+    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
+    try (OutputStreamWriter formWriter = new OutputStreamWriter(buffer)) {
+      formWriter.append("--").append(boundary);
+
+      for (Map.Entry<String, InputStream> entry : files.entrySet()) {
+        formWriter.append("\n");
+        formWriter.append(String.format("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"",
+          entry.getKey(), entry.getKey())).append("\n");
+        formWriter.append("Content-Type: application/octet-stream").append("\n\n");
+
+        InputStream stream = entry.getValue();
+        int b = stream.read();
+        while (b >= 0) {
+          formWriter.write(b);
+          b = stream.read();
+        }
+        stream.close();
+        formWriter.append("\n").append("--").append(boundary);
+      }
+
+      if (repository != null) {
+        formWriter.append("\n");
+        formWriter.append("Content-Disposition: form-data; name=\"repository\"").append("\n\n");
+        StringWriter repositoryWriter = new StringWriter();
+        new JsonFactory().createGenerator(repositoryWriter).setCodec(new ObjectMapper()).writeObject(repository);
+        formWriter.append(repositoryWriter.getBuffer().toString()).append("\n");
+        formWriter.append("--").append(boundary);
+      }
+
+      formWriter.append("--");
+      formWriter.flush();
+    }
+    request.setInputStream(new ByteArrayInputStream(buffer.toByteArray()));
+    return request;
+  }
 }

scm-webapp/src/test/resources/sonia/scm/api/v2/import-repo-from-bundle.json

@@ -0,0 +1,7 @@
+{
+  "contact": "none@example.com",
+  "description": "Test repository",
+  "namespace": "space",
+  "name": "repo",
+  "type": "svn"
+}

scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump

@@ -0,0 +1,83 @@
+SVN-fs-dump-format-version: 2
+
+UUID: dcaa635c-9a8d-4cd6-918f-250ca2f765ea
+
+Revision-number: 0
+Prop-content-length: 56
+Content-length: 56
+
+K 8
+svn:date
+V 27
+2020-12-09T13:42:16.879000Z
+PROPS-END
+
+Revision-number: 1
+Prop-content-length: 124
+Content-length: 124
+
+K 10
+svn:author
+V 8
+scmadmin
+K 8
+svn:date
+V 27
+2020-12-09T13:42:18.270000Z
+K 7
+svn:log
+V 21
+initialize repository
+PROPS-END
+
+Node-path: trunk
+Node-kind: dir
+Node-action: add
+Prop-content-length: 10
+Content-length: 10
+
+PROPS-END
+
+
+Node-path: trunk/README.md
+Node-kind: file
+Node-action: add
+Text-content-md5: fe6869009516b5517b13036294d05f83
+Text-content-sha1: 4e51754688703c31980541dbbb884671d92cf846
+Prop-content-length: 10
+Text-content-length: 9
+Content-length: 19
+
+PROPS-END
+# dump_me
+
+Revision-number: 2
+Prop-content-length: 106
+Content-length: 106
+
+K 10
+svn:author
+V 8
+scmadmin
+K 8
+svn:date
+V 27
+2020-12-09T13:42:38.170000Z
+K 7
+svn:log
+V 4
+test
+PROPS-END
+
+Node-path: trunk/second_one.txt
+Node-kind: file
+Node-action: add
+Text-content-md5: 098f6bcd4621d373cade4e832627b4f6
+Text-content-sha1: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3
+Prop-content-length: 10
+Text-content-length: 4
+Content-length: 14
+
+PROPS-END
+test
+