diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java index e06c23528d..17dd47352e 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java @@ -24,8 +24,13 @@ package sonia.scm.api.v2.resources; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import com.google.common.io.ByteSource; +import com.google.common.io.Files; import com.google.inject.Inject; import de.otto.edison.hal.Embedded; import de.otto.edison.hal.Links; @@ -37,8 +42,13 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import org.apache.shiro.SecurityUtils; +import org.jboss.resteasy.plugins.providers.multipart.InputPart; +import org.jboss.resteasy.plugins.providers.multipart.MultipartFormDataInput; +import org.jboss.resteasy.plugins.providers.multipart.MultipartInputImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sonia.scm.BadRequestException; +import sonia.scm.ContextEntry; import sonia.scm.HandlerEventType; import sonia.scm.Type; import sonia.scm.event.ScmEventBus; @@ -54,6 +64,7 @@ import sonia.scm.repository.api.Command; import sonia.scm.repository.api.PullCommandBuilder; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.util.IOUtil; import sonia.scm.util.ValidationUtil; import sonia.scm.web.VndMediaType; @@ -62,18 +73,30 @@ import javax.validation.constraints.Email; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.Pattern; import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; +import javax.ws.rs.QueryParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.singletonList; public class RepositoryImportResource { @@ -188,6 +211,139 @@ public class RepositoryImportResource { }; } + /** + * Imports a external repository via dump. The method can + * only be used, if the repository type supports the {@link Command#UNBUNDLE}. The + * method will return a location header with the url to the imported + * repository. + * + * @param uriInfo uri info + * @param type repository type + * @param input multi part form data which should contain a valid repository dto and the input stream of the bundle + * @param compressed true if the bundle is gzip compressed + * @return empty response with location header which points to the imported + * repository + * @since 2.12.0 + */ + @POST + @Path("{type}/bundle") + @Consumes(MediaType.MULTIPART_FORM_DATA) + @Operation(summary = "Import repository from bundle", description = "Imports the repository from the provided bundle.", tags = "Repository") + @ApiResponse( + responseCode = "201", + description = "Repository import was successful" + ) + @ApiResponse( + responseCode = "401", + description = "not authenticated / invalid credentials" + ) + @ApiResponse( + responseCode = "403", + description = "not authorized, the current user has no privileges to read the repository" + ) + @ApiResponse( + responseCode = "500", + description = "internal server error", + content = @Content( + mediaType = VndMediaType.ERROR_TYPE, + schema = @Schema(implementation = ErrorDto.class) + ) + ) + public Response importFromBundle(@Context UriInfo uriInfo, + @PathParam("type") String type, + MultipartFormDataInput input, + @QueryParam("compressed") @DefaultValue("false") boolean compressed) { + RepositoryPermissions.create().check(); + Repository repository = doImportFromBundle(type, input, compressed); + + return Response.created(URI.create(resourceLinks.repository().self(repository.getNamespace(), repository.getName()))).build(); + } + + /** + * Start bundle import. + * + * @param type repository type + * @param input multi part form data + * @param compressed true if the bundle is gzip compressed + * @return imported repository + */ + private Repository doImportFromBundle(String type, MultipartFormDataInput input, boolean compressed) { + Map> formParts = input.getFormDataMap(); + RepositoryDto repositoryDto = extractFromInputPart(formParts.get("repository"), RepositoryDto.class); + InputStream inputStream = extractFromInputPart(formParts.get("bundle"), InputStream.class); + + checkNotNull(repositoryDto, "repository data is required"); + checkNotNull(inputStream, "bundle inputStream is required"); + checkArgument(!Strings.isNullOrEmpty(repositoryDto.getName()), "request does not contain name of the repository"); + + Type t = type(type); + + checkSupport(t, Command.UNBUNDLE, "bundle"); + + Repository repository = mapper.map(repositoryDto); + repository.setPermissions(singletonList( + new RepositoryPermission(SecurityUtils.getSubject().getPrincipal().toString(), "OWNER", false) + )); + + try { + repository = manager.create( + repository, + unbundleImport(inputStream, compressed) + ); + eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, false)); + + } catch (Exception e) { + eventBus.post(new RepositoryImportEvent(HandlerEventType.MODIFY, repository, true)); + throw e; + } + + return repository; + } + + @VisibleForTesting + Consumer unbundleImport(InputStream inputStream, boolean compressed) { + return repository -> { + File file = null; + try (RepositoryService service = serviceFactory.create(repository)) { + file = File.createTempFile("scm-import-", ".bundle"); + long length = Files.asByteSink(file).writeFrom(inputStream); + logger.info("copied {} bytes to temp, start bundle import", length); + service.getUnbundleCommand().setCompressed(compressed).unbundle(file); + } catch (IOException e) { + throw new InternalRepositoryException(repository, "Failed to import from bundle", e); + } finally { + try { + IOUtil.delete(file); + } catch (IOException ex) { + logger.warn("could not delete temporary file", ex); + } + } + }; + } + + private T extractFromInputPart(List input, Class type) { + try { + if (input != null && !input.isEmpty()) { + String content = new ByteSource() { + @Override + public InputStream openStream() throws IOException { + return ((MultipartInputImpl.PartImpl) input.get(0)).getBody(); + } + }.asCharSource(UTF_8).read(); + if (type == InputStream.class) { + return (T) new ByteArrayInputStream(StandardCharsets.UTF_8.encode(content).array()); + } + try (JsonParser parser = new JsonFactory().createParser(content)) { + parser.setCodec(new ObjectMapper()); + return parser.readValueAs(type); + } + } + } catch (IOException ex) { + logger.debug("Could not extract repository from input"); + } + return null; + } + /** * Check repository type for support for the given command. * @@ -241,16 +397,23 @@ public class RepositoryImportResource { interface ImportRepositoryDto { String getNamespace(); + @Pattern(regexp = ValidationUtil.REGEX_REPOSITORYNAME) String getName(); + @NotEmpty String getType(); + @Email String getContact(); + String getDescription(); + @NotEmpty String getImportUrl(); + String getUsername(); + String getPassword(); } } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java index daabb9ae20..26f828571b 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java @@ -24,6 +24,8 @@ package sonia.scm.api.v2.resources; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.sdorra.shiro.ShiroRule; import com.github.sdorra.shiro.SubjectAware; import com.google.common.collect.ImmutableSet; @@ -58,17 +60,29 @@ import sonia.scm.repository.api.ImportFailedException; import sonia.scm.repository.api.PullCommandBuilder; import sonia.scm.repository.api.RepositoryService; import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.repository.api.UnbundleCommandBuilder; +import sonia.scm.repository.api.UnbundleResponse; import sonia.scm.user.User; import sonia.scm.web.RestDispatcher; import sonia.scm.web.VndMediaType; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.Collections; +import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.function.Consumer; import java.util.function.Predicate; @@ -565,6 +579,81 @@ public class RepositoryRootResourceTest extends RepositoryTestBase { assertThrows(ImportFailedException.class, () -> repositoryConsumer.accept(repository)); } + @Test + public void shouldImportRepositoryFromBundle() throws IOException, URISyntaxException { + when(manager.getHandler("svn")).thenReturn(repositoryHandler); + when(repositoryHandler.getType()).thenReturn(new RepositoryType("svn", "svn", ImmutableSet.of(Command.UNBUNDLE))); + when(repositoryManager.create(any(), any())).thenReturn(RepositoryTestData.createHeartOfGold()); + + RepositoryDto repositoryDto = new RepositoryDto(); + repositoryDto.setName("HeartOfGold"); + repositoryDto.setNamespace("hitchhiker"); + repositoryDto.setType("svn"); + + URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump"); + byte[] svnDump = Resources.toByteArray(dumpUrl); + + MockHttpRequest request = MockHttpRequest + .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "import/svn/bundle"); + MockHttpResponse response = new MockHttpResponse(); + + multipartRequest(request, Collections.singletonMap("bundle", new ByteArrayInputStream(svnDump)), repositoryDto); + + dispatcher.invoke(request, response); + + assertEquals(HttpServletResponse.SC_CREATED, response.getStatus()); + assertEquals("/v2/repositories/hitchhiker/HeartOfGold", response.getOutputHeaders().get("Location").get(0).toString()); + ArgumentCaptor event = ArgumentCaptor.forClass(RepositoryImportEvent.class); + verify(eventBus).post(event.capture()); + assertFalse(event.getValue().isFailed()); + } + + @Test + public void shouldThrowFailedEventOnImportRepositoryFromBundle() throws IOException, URISyntaxException { + when(manager.getHandler("svn")).thenReturn(repositoryHandler); + when(repositoryHandler.getType()).thenReturn(new RepositoryType("svn", "svn", ImmutableSet.of(Command.UNBUNDLE))); + doThrow(ImportFailedException.class).when(repositoryManager).create(any(), any()); + + RepositoryDto repositoryDto = new RepositoryDto(); + repositoryDto.setName("HeartOfGold"); + repositoryDto.setNamespace("hitchhiker"); + repositoryDto.setType("svn"); + + URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump"); + byte[] svnDump = Resources.toByteArray(dumpUrl); + + MockHttpRequest request = MockHttpRequest + .post("/" + RepositoryRootResource.REPOSITORIES_PATH_V2 + "import/svn/bundle"); + MockHttpResponse response = new MockHttpResponse(); + + multipartRequest(request, Collections.singletonMap("bundle", new ByteArrayInputStream(svnDump)), repositoryDto); + + dispatcher.invoke(request, response); + + assertEquals(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, response.getStatus()); + ArgumentCaptor event = ArgumentCaptor.forClass(RepositoryImportEvent.class); + verify(eventBus).post(event.capture()); + assertTrue(event.getValue().isFailed()); + } + + @Test + public void shouldImportCompressedBundle() throws IOException { + URL dumpUrl = Resources.getResource("sonia/scm/api/v2/svn.dump.gz"); + byte[] svnDump = Resources.toByteArray(dumpUrl); + + UnbundleCommandBuilder ubc = mock(UnbundleCommandBuilder.class); + when(ubc.setCompressed(any())).thenReturn(ubc); + when(ubc.unbundle(any(File.class))).thenReturn(new UnbundleResponse(42)); + RepositoryService service = mock(RepositoryService.class); + when(serviceFactory.create(any(Repository.class))).thenReturn(service); + when(service.getUnbundleCommand()).thenReturn(ubc); + InputStream in = new ByteArrayInputStream(svnDump); + repositoryImportResource.unbundleImport(in, true); + + verify(ubc).setCompressed(true); + //TODO Enhance test + } + private PageResult createSingletonPageResult(Repository repository) { return new PageResult<>(singletonList(repository), 0); } @@ -586,4 +675,49 @@ public class RepositoryRootResourceTest extends RepositoryTestBase { when(repositoryManager.get(id)).thenReturn(repository); return repository; } + + + /** + * This method is a slightly adapted copy of Lin Zaho's gist at https://gist.github.com/lin-zhao/9985191 + */ + private MockHttpRequest multipartRequest(MockHttpRequest request, Map files, RepositoryDto repository) throws IOException { + String boundary = UUID.randomUUID().toString(); + request.contentType("multipart/form-data; boundary=" + boundary); + + //Make sure this is deleted in afterTest() + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (OutputStreamWriter formWriter = new OutputStreamWriter(buffer)) { + formWriter.append("--").append(boundary); + + for (Map.Entry entry : files.entrySet()) { + formWriter.append("\n"); + formWriter.append(String.format("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"", + entry.getKey(), entry.getKey())).append("\n"); + formWriter.append("Content-Type: application/octet-stream").append("\n\n"); + + InputStream stream = entry.getValue(); + int b = stream.read(); + while (b >= 0) { + formWriter.write(b); + b = stream.read(); + } + stream.close(); + formWriter.append("\n").append("--").append(boundary); + } + + if (repository != null) { + formWriter.append("\n"); + formWriter.append("Content-Disposition: form-data; name=\"repository\"").append("\n\n"); + StringWriter repositoryWriter = new StringWriter(); + new JsonFactory().createGenerator(repositoryWriter).setCodec(new ObjectMapper()).writeObject(repository); + formWriter.append(repositoryWriter.getBuffer().toString()).append("\n"); + formWriter.append("--").append(boundary); + } + + formWriter.append("--"); + formWriter.flush(); + } + request.setInputStream(new ByteArrayInputStream(buffer.toByteArray())); + return request; + } } diff --git a/scm-webapp/src/test/resources/sonia/scm/api/v2/import-repo-from-bundle.json b/scm-webapp/src/test/resources/sonia/scm/api/v2/import-repo-from-bundle.json new file mode 100644 index 0000000000..5bc573f2a8 --- /dev/null +++ b/scm-webapp/src/test/resources/sonia/scm/api/v2/import-repo-from-bundle.json @@ -0,0 +1,7 @@ +{ + "contact": "none@example.com", + "description": "Test repository", + "namespace": "space", + "name": "repo", + "type": "svn" +} diff --git a/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump b/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump new file mode 100644 index 0000000000..f339c8635f --- /dev/null +++ b/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump @@ -0,0 +1,83 @@ +SVN-fs-dump-format-version: 2 + +UUID: dcaa635c-9a8d-4cd6-918f-250ca2f765ea + +Revision-number: 0 +Prop-content-length: 56 +Content-length: 56 + +K 8 +svn:date +V 27 +2020-12-09T13:42:16.879000Z +PROPS-END + +Revision-number: 1 +Prop-content-length: 124 +Content-length: 124 + +K 10 +svn:author +V 8 +scmadmin +K 8 +svn:date +V 27 +2020-12-09T13:42:18.270000Z +K 7 +svn:log +V 21 +initialize repository +PROPS-END + +Node-path: trunk +Node-kind: dir +Node-action: add +Prop-content-length: 10 +Content-length: 10 + +PROPS-END + + +Node-path: trunk/README.md +Node-kind: file +Node-action: add +Text-content-md5: fe6869009516b5517b13036294d05f83 +Text-content-sha1: 4e51754688703c31980541dbbb884671d92cf846 +Prop-content-length: 10 +Text-content-length: 9 +Content-length: 19 + +PROPS-END +# dump_me + +Revision-number: 2 +Prop-content-length: 106 +Content-length: 106 + +K 10 +svn:author +V 8 +scmadmin +K 8 +svn:date +V 27 +2020-12-09T13:42:38.170000Z +K 7 +svn:log +V 4 +test +PROPS-END + +Node-path: trunk/second_one.txt +Node-kind: file +Node-action: add +Text-content-md5: 098f6bcd4621d373cade4e832627b4f6 +Text-content-sha1: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 +Prop-content-length: 10 +Text-content-length: 4 +Content-length: 14 + +PROPS-END +test + diff --git a/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump.gz b/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump.gz new file mode 100644 index 0000000000..2ca3731608 Binary files /dev/null and b/scm-webapp/src/test/resources/sonia/scm/api/v2/svn.dump.gz differ