From 308b95d8c748b74aa308587c299d3f7ce1a1ba2f Mon Sep 17 00:00:00 2001 From: Eduard Heimbuch Date: Mon, 23 Nov 2020 18:20:48 +0100 Subject: [PATCH] Upgrade repository import api --- .../repository/spi/RemoteCommandRequest.java | 132 +--- .../scm/repository/spi/GitPullCommand.java | 2 + scm-ui/ui-components/src/forms/DropDown.tsx | 2 +- scm-ui/ui-webapp/public/locales/de/repos.json | 2 +- scm-ui/ui-webapp/public/locales/en/repos.json | 2 +- .../resources/RepositoryImportResource.java | 684 ------------------ .../resources/RepositoryImportResource.java | 591 +++++++++++++++ .../v2/resources/RepositoryRootResource.java | 17 +- .../api/v2/resources/RepositoryTestBase.java | 3 +- 9 files changed, 626 insertions(+), 809 deletions(-) delete mode 100644 scm-webapp/src/main/java/sonia/scm/api/rest/resources/RepositoryImportResource.java create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java diff --git a/scm-core/src/main/java/sonia/scm/repository/spi/RemoteCommandRequest.java b/scm-core/src/main/java/sonia/scm/repository/spi/RemoteCommandRequest.java index d2883e85a7..a95c3d02b5 100644 --- a/scm-core/src/main/java/sonia/scm/repository/spi/RemoteCommandRequest.java +++ b/scm-core/src/main/java/sonia/scm/repository/spi/RemoteCommandRequest.java @@ -21,57 +21,31 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.repository.spi; -//~--- non-JDK imports -------------------------------------------------------- - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import sonia.scm.repository.Repository; import java.net.URL; -//~--- JDK imports ------------------------------------------------------------ - /** - * * @author Sebastian Sdorra * @since 1.31 */ -public abstract class RemoteCommandRequest implements Resetable -{ +@Getter +@Setter +@EqualsAndHashCode +@ToString +public abstract class RemoteCommandRequest implements Resetable { - /** - * {@inheritDoc} - */ - @Override - public boolean equals(Object obj) - { - if (obj == null) - { - return false; - } - - if (getClass() != obj.getClass()) - { - return false; - } - - final RemoteCommandRequest other = (RemoteCommandRequest) obj; - - return Objects.equal(remoteRepository, other.remoteRepository) - && Objects.equal(remoteUrl, other.remoteUrl); - } - - /** - * {@inheritDoc} - */ - @Override - public int hashCode() - { - return Objects.hashCode(remoteRepository, remoteUrl); - } + protected Repository remoteRepository; + protected URL remoteUrl; + protected String username; + protected String password; /** * Resets the request object. @@ -79,82 +53,10 @@ public abstract class RemoteCommandRequest implements Resetable * @since 1.43 */ @Override - public void reset() - { + public void reset() { remoteRepository = null; remoteUrl = null; + username = null; + password = null; } - - /** - * {@inheritDoc} - */ - @Override - public String toString() - { - //J- - return MoreObjects.toStringHelper(this) - .add("remoteRepository", remoteRepository) - .add("remoteUrl", remoteUrl) - .toString(); - //J+ - } - - //~--- set methods ---------------------------------------------------------- - - /** - * Method description - * - * @param remoteRepository - */ - public void setRemoteRepository(Repository remoteRepository) - { - this.remoteRepository = remoteRepository; - } - - /** - * Method description - * - * - * @param remoteUrl - * - * @since 1.43 - */ - public void setRemoteUrl(URL remoteUrl) - { - this.remoteUrl = remoteUrl; - } - - //~--- get methods ---------------------------------------------------------- - - /** - * Method description - * - * - * @return - */ - Repository getRemoteRepository() - { - return remoteRepository; - } - - /** - * Method description - * - * - * @return - * - * @since 1.43 - */ - URL getRemoteUrl() - { - return remoteUrl; - } - - //~--- fields --------------------------------------------------------------- - - /** Field description */ - protected Repository remoteRepository; - - /** remote url */ - protected URL remoteUrl; } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitPullCommand.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitPullCommand.java index 422391fd19..5ef7defd34 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitPullCommand.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitPullCommand.java @@ -32,10 +32,12 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.TagOpt; import org.eclipse.jgit.transport.TrackingRefUpdate; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.repository.GitRepositoryHandler; diff --git a/scm-ui/ui-components/src/forms/DropDown.tsx b/scm-ui/ui-components/src/forms/DropDown.tsx index e801a0c2a8..bf4701ac43 100644 --- a/scm-ui/ui-components/src/forms/DropDown.tsx +++ b/scm-ui/ui-components/src/forms/DropDown.tsx @@ -42,7 +42,7 @@ class DropDown extends React.Component { render() { const { options, optionValues, preselectedOption, className, disabled } = this.props; - if (preselectedOption && options.filter(o => o === preselectedOption).length === 0) { + if (preselectedOption && options.some(o => o === preselectedOption)) { options.push(preselectedOption); } diff --git a/scm-ui/ui-webapp/public/locales/de/repos.json b/scm-ui/ui-webapp/public/locales/de/repos.json index d95ef44970..c70bbce695 100644 --- a/scm-ui/ui-webapp/public/locales/de/repos.json +++ b/scm-ui/ui-webapp/public/locales/de/repos.json @@ -42,7 +42,7 @@ "title": "Repositories", "subtitle": "Übersicht aller verfügbaren Repositories", "noRepositories": "Keine Repositories gefunden.", - "createButton": "Repository erstellen" + "createButton": "Repository hinzufügen" }, "create": { "title": "Repository erstellen", diff --git a/scm-ui/ui-webapp/public/locales/en/repos.json b/scm-ui/ui-webapp/public/locales/en/repos.json index 59981282df..532ded509d 100644 --- a/scm-ui/ui-webapp/public/locales/en/repos.json +++ b/scm-ui/ui-webapp/public/locales/en/repos.json @@ -42,7 +42,7 @@ "title": "Repositories", "subtitle": "Overview of available repositories", "noRepositories": "No repositories found.", - "createButton": "Create Repository" + "createButton": "Add Repository" }, "create": { "title": "Create Repository", diff --git a/scm-webapp/src/main/java/sonia/scm/api/rest/resources/RepositoryImportResource.java b/scm-webapp/src/main/java/sonia/scm/api/rest/resources/RepositoryImportResource.java deleted file mode 100644 index 78c0e728dd..0000000000 --- a/scm-webapp/src/main/java/sonia/scm/api/rest/resources/RepositoryImportResource.java +++ /dev/null @@ -1,684 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.api.rest.resources; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableList; -import com.google.common.io.Files; -import com.google.inject.Inject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import sonia.scm.FeatureNotSupportedException; -import sonia.scm.NotFoundException; -import sonia.scm.Type; -import sonia.scm.api.rest.RestActionUploadResult; -import sonia.scm.api.v2.resources.RepositoryResource; -import sonia.scm.repository.AdvancedImportHandler; -import sonia.scm.repository.ImportHandler; -import sonia.scm.repository.ImportResult; -import sonia.scm.repository.InternalRepositoryException; -import sonia.scm.repository.Repository; -import sonia.scm.repository.RepositoryHandler; -import sonia.scm.repository.RepositoryManager; -import sonia.scm.repository.RepositoryPermissions; -import sonia.scm.repository.RepositoryType; -import sonia.scm.repository.api.Command; -import sonia.scm.repository.api.RepositoryService; -import sonia.scm.repository.api.RepositoryServiceFactory; -import sonia.scm.repository.api.UnbundleCommandBuilder; -import sonia.scm.util.IOUtil; - -import javax.ws.rs.Consumes; -import javax.ws.rs.DefaultValue; -import javax.ws.rs.FormParam; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.GenericEntity; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Set; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Rest resource for importing repositories. - * - * @author Sebastian Sdorra - */ -// @Path("import/repositories") -public class RepositoryImportResource { - - /** - * the logger for RepositoryImportResource - */ - private static final Logger logger = - LoggerFactory.getLogger(RepositoryImportResource.class); - - //~--- constructors --------------------------------------------------------- - - /** - * Constructs a new repository import resource. - * - * @param manager repository manager - * @param serviceFactory - */ - @Inject - public RepositoryImportResource(RepositoryManager manager, - RepositoryServiceFactory serviceFactory) { - this.manager = manager; - this.serviceFactory = serviceFactory; - } - - //~--- methods -------------------------------------------------------------- - - /** - * Imports a repository type specific bundle. The bundle file is uploaded to - * the server which is running scm-manager. After the upload has finished, the - * bundle file is passed to the {@link UnbundleCommandBuilder}. Note: This method - * requires admin privileges. - * - * @param uriInfo uri info - * @param type repository type - * @param name name of the repository - * @param inputStream input bundle - * @param compressed true if the bundle is gzip compressed - * @return empty response with location header which points to the imported repository - * @since 1.43 - */ - @POST - @Path("{type}/bundle") - @Consumes(MediaType.MULTIPART_FORM_DATA) - public Response importFromBundle(@Context UriInfo uriInfo, - @PathParam("type") String type, @FormParam("name") String name, - @FormParam("bundle") InputStream inputStream, @QueryParam("compressed") - @DefaultValue("false") boolean compressed) { - Repository repository = doImportFromBundle(type, name, inputStream, - compressed); - - return buildResponse(uriInfo, repository); - } - - /** - * This method works exactly like - * {@link #importFromBundle(UriInfo, String, String, InputStream)}, but this - * method returns an html content-type. The method exists only for a - * workaround of the javascript ui extjs. Note: This method requires admin - * privileges. - * - * @param type repository type - * @param name name of the repository - * @param inputStream input bundle - * @param compressed true if the bundle is gzip compressed - * @return empty response with location header which points to the imported - * repository - * @since 1.43 - */ - @POST - @Path("{type}/bundle.html") - @Consumes(MediaType.MULTIPART_FORM_DATA) - @Produces(MediaType.TEXT_HTML) - public Response importFromBundleUI(@PathParam("type") String type, - @FormParam("name") String name, - @FormParam("bundle") InputStream inputStream, @QueryParam("compressed") - @DefaultValue("false") boolean compressed) { - Response response; - - try { - doImportFromBundle(type, name, inputStream, compressed); - response = Response.ok(new RestActionUploadResult(true)).build(); - } catch (WebApplicationException ex) { - logger.warn("error durring bundle import", ex); - response = Response.fromResponse(ex.getResponse()).entity( - new RestActionUploadResult(false)).build(); - } - - return response; - } - - /** - * Imports a external repository which is accessible via url. The method can - * only be used, if the repository type supports the {@link Command#PULL}. The - * method will return a location header with the url to the imported - * repository. Note: This method requires admin privileges. - * - * @param uriInfo uri info - * @param type repository type - * @param request request object - * @return empty response with location header which points to the imported - * repository - * @since 1.43 - */ - @POST - @Path("{type}/url") - @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response importFromUrl(@Context UriInfo uriInfo, - @PathParam("type") String type, UrlImportRequest request) { - RepositoryPermissions.create().check(); - checkNotNull(request, "request is required"); - checkArgument(!Strings.isNullOrEmpty(request.getName()), - "request does not contain name of the repository"); - checkArgument(!Strings.isNullOrEmpty(request.getUrl()), - "request does not contain url of the remote repository"); - - Type t = type(type); - - checkSupport(t, Command.PULL, request); - - logger.info("start {} import for external url {}", type, request.getUrl()); - - Repository repository = create(type, request.getName()); - RepositoryService service = null; - - try { - service = serviceFactory.create(repository); - service.getPullCommand().pull(request.getUrl()); - } catch (IOException ex) { - handleImportFailure(ex, repository); - } finally { - IOUtil.close(service); - } - - return buildResponse(uriInfo, repository); - } - - /** - * Imports repositories of the given type from the configured repository - * directory. Note: This method requires admin privileges. - * - * @param type repository type - * @return imported repositories - */ - @POST - @Path("{type}") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response importRepositories(@PathParam("type") String type) { - RepositoryPermissions.create().check(); - - List repositories = new ArrayList(); - - importFromDirectory(repositories, type); - - //J- - return Response.ok( - new GenericEntity>(repositories) { - } - ).build(); - //J+ - } - - /** - * Imports repositories of all supported types from the configured repository - * directories. Note: This method requires admin privileges. - * - * @return imported repositories - */ - @POST - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response importRepositories() { - RepositoryPermissions.create().check(); - - logger.info("start directory import for all supported repository types"); - - List repositories = new ArrayList(); - - for (Type t : findImportableTypes()) { - importFromDirectory(repositories, t.getName()); - } - - //J- - return Response.ok( - new GenericEntity>(repositories) { - } - ).build(); - //J+ - } - - /** - * Imports repositories of the given type from the configured repository - * directory. Returns a list of successfully imported directories and a list - * of failed directories. Note: This method requires admin privileges. - * - * @param type repository type - * @return imported repositories - * @since 1.43 - */ - @POST - @Path("{type}/directory") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response importRepositoriesFromDirectory( - @PathParam("type") String type) { - RepositoryPermissions.create().check(); - - Response response; - - RepositoryHandler handler = manager.getHandler(type); - - if (handler != null) { - logger.info("start directory import for repository type {}", type); - - try { - ImportResult result; - ImportHandler importHandler = handler.getImportHandler(); - - if (importHandler instanceof AdvancedImportHandler) { - logger.debug("start directory import, using advanced import handler"); - result = - ((AdvancedImportHandler) importHandler) - .importRepositoriesFromDirectory(manager); - } else { - logger.debug("start directory import, using normal import handler"); - result = new ImportResult(importHandler.importRepositories(manager), - ImmutableList.of()); - } - - response = Response.ok(result).build(); - } catch (FeatureNotSupportedException ex) { - logger - .warn( - "import feature is not supported by repository handler for type " - .concat(type), ex); - response = Response.status(Response.Status.BAD_REQUEST).build(); - } catch (IOException ex) { - logger.warn("exception occured durring directory import", ex); - response = Response.serverError().build(); - } - } else { - logger.warn("could not find reposiotry handler for type {}", type); - response = Response.status(Response.Status.BAD_REQUEST).build(); - } - - return response; - } - - //~--- get methods ---------------------------------------------------------- - - /** - * Returns a list of repository types, which support the directory import - * feature. Note: This method requires admin privileges. - * - * @return list of repository types - */ - @GET - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) - public Response getImportableTypes() { - RepositoryPermissions.create().check(); - - List types = findImportableTypes(); - - //J- - return Response.ok( - new GenericEntity>(types) { - } - ).build(); - //J+ - } - - //~--- methods -------------------------------------------------------------- - - /** - * Build rest response for repository. - * - * @param uriInfo uri info - * @param repository imported repository - * @return rest response - */ - private Response buildResponse(UriInfo uriInfo, Repository repository) { - URI location = uriInfo.getBaseUriBuilder().path( - RepositoryResource.class).path(repository.getId()).build(); - - return Response.created(location).build(); - } - - /** - * Check repository type for support for the given command. - * - * @param type repository type - * @param cmd command - * @param request request object - */ - private void checkSupport(Type type, Command cmd, Object request) { - if (!(type instanceof RepositoryType)) { - logger.warn("type {} is not a repository type", type.getName()); - - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } - - Set cmds = ((RepositoryType) type).getSupportedCommands(); - - if (!cmds.contains(cmd)) { - logger.warn("type {} does not support this type of import: {}", - type.getName(), request); - - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } - } - - /** - * Creates a new repository with the given name and type. - * - * @param type repository type - * @param name repository name - * @return newly created repository - */ - private Repository create(String type, String name) { - Repository repository = null; - - try { - // TODO #8783 -// repository = new Repository(null, type, name); - manager.create(repository); - } catch (InternalRepositoryException ex) { - handleGenericCreationFailure(ex, type, name); - } - - return repository; - } - - /** - * Start bundle import. - * - * @param type repository type - * @param name name of the repository - * @param inputStream bundle stream - * @param compressed true if the bundle is gzip compressed - * @return imported repository - */ - private Repository doImportFromBundle(String type, String name, - InputStream inputStream, boolean compressed) { - RepositoryPermissions.create().check(); - - checkArgument(!Strings.isNullOrEmpty(name), - "request does not contain name of the repository"); - checkNotNull(inputStream, "bundle inputStream is required"); - - Repository repository; - - try { - Type t = type(type); - - checkSupport(t, Command.UNBUNDLE, "bundle"); - - repository = create(type, name); - - RepositoryService service = null; - - File file = File.createTempFile("scm-import-", ".bundle"); - - try { - long length = Files.asByteSink(file).writeFrom(inputStream); - - logger.info("copied {} bytes to temp, start bundle import", length); - service = serviceFactory.create(repository); - service.getUnbundleCommand().setCompressed(compressed).unbundle(file); - } catch (InternalRepositoryException ex) { - handleImportFailure(ex, repository); - } finally { - IOUtil.close(service); - IOUtil.delete(file); - } - } catch (IOException ex) { - logger.warn("could not create temporary file", ex); - - throw new WebApplicationException(ex); - } - - return repository; - } - - /** - * Method description - * - * @return - */ - private List findImportableTypes() { - List types = new ArrayList(); - Collection handlerTypes = manager.getTypes(); - - for (Type t : handlerTypes) { - RepositoryHandler handler = manager.getHandler(t.getName()); - - if (handler != null) { - try { - if (handler.getImportHandler() != null) { - types.add(t); - } - } catch (FeatureNotSupportedException ex) { - if (logger.isTraceEnabled()) { - logger.trace("import handler is not supported", ex); - } else if (logger.isInfoEnabled()) { - logger.info("{} handler does not support import of repositories", - t.getName()); - } - } - } else if (logger.isWarnEnabled()) { - logger.warn("could not find handler for type {}", t.getName()); - } - } - - return types; - } - - /** - * Handle creation failures. - * - * @param ex exception - * @param type repository type - * @param name name of the repository - */ - private void handleGenericCreationFailure(Exception ex, String type, - String name) { - logger.error(String.format("could not create repository %s with type %s", - type, name), ex); - - throw new WebApplicationException(ex); - } - - /** - * Handle import failures. - * - * @param ex exception - * @param repository repository - */ - private void handleImportFailure(Exception ex, Repository repository) { - logger.error("import for repository failed, delete repository", ex); - - try { - manager.delete(repository); - } catch (InternalRepositoryException | NotFoundException e) { - logger.error("can not delete repository after import failure", e); - } - - throw new WebApplicationException(ex, - Response.Status.INTERNAL_SERVER_ERROR); - } - - /** - * Import repositories from a specific type. - * - * @param repositories repository list - * @param type type of repository - */ - private void importFromDirectory(List repositories, String type) { - RepositoryHandler handler = manager.getHandler(type); - - if (handler != null) { - logger.info("start directory import for repository type {}", type); - - try { - List repositoryNames = - handler.getImportHandler().importRepositories(manager); - - if (repositoryNames != null) { - for (String repositoryName : repositoryNames) { - // TODO #8783 - /*Repository repository = null; //manager.get(type, repositoryName); - - if (repository != null) - { - repositories.add(repository); - } - else if (logger.isWarnEnabled()) - { - logger.warn("could not find imported repository {}", - repositoryName); - }*/ - } - } - } catch (FeatureNotSupportedException ex) { - throw new WebApplicationException(ex, Response.Status.BAD_REQUEST); - } catch (IOException ex) { - throw new WebApplicationException(ex); - } catch (InternalRepositoryException ex) { - throw new WebApplicationException(ex); - } - } else { - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } - } - - /** - * Method description - * - * @param type - * @return - */ - private Type type(String type) { - RepositoryHandler handler = manager.getHandler(type); - - if (handler == null) { - logger.warn("no handler for type {} found", type); - - throw new WebApplicationException(Response.Status.NOT_FOUND); - } - - return handler.getType(); - } - - //~--- inner classes -------------------------------------------------------- - - /** - * Request for importing external repositories which are accessible via url. - */ - @XmlRootElement(name = "import") - @XmlAccessorType(XmlAccessType.FIELD) - public static class UrlImportRequest { - - /** - * Constructs ... - */ - public UrlImportRequest() { - } - - /** - * Constructs a new {@link UrlImportRequest} - * - * @param name name of the repository - * @param url external url of the repository - */ - public UrlImportRequest(String name, String url) { - this.name = name; - this.url = url; - } - - //~--- methods ------------------------------------------------------------ - - /** - * {@inheritDoc} - */ - @Override - public String toString() { - //J- - return MoreObjects.toStringHelper(this) - .add("name", name) - .add("url", url) - .toString(); - //J+ - } - - //~--- get methods -------------------------------------------------------- - - /** - * Returns name of the repository. - * - * @return name of the repository - */ - public String getName() { - return name; - } - - /** - * Returns external url of the repository. - * - * @return external url of the repository - */ - public String getUrl() { - return url; - } - - //~--- fields ------------------------------------------------------------- - - /** - * name of the repository - */ - private String name; - - /** - * external url of the repository - */ - private String url; - } - - - //~--- fields --------------------------------------------------------------- - - /** - * repository manager - */ - private final RepositoryManager manager; - - /** - * repository service factory - */ - private final RepositoryServiceFactory serviceFactory; -} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java new file mode 100644 index 0000000000..64399d30a5 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryImportResource.java @@ -0,0 +1,591 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import com.google.common.base.Strings; +import com.google.common.io.Files; +import com.google.inject.Inject; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import lombok.ToString; +import lombok.Value; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.FeatureNotSupportedException; +import sonia.scm.NotFoundException; +import sonia.scm.Type; +import sonia.scm.repository.InternalRepositoryException; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryHandler; +import sonia.scm.repository.RepositoryManager; +import sonia.scm.repository.RepositoryPermissions; +import sonia.scm.repository.RepositoryType; +import sonia.scm.repository.api.Command; +import sonia.scm.repository.api.RepositoryService; +import sonia.scm.repository.api.RepositoryServiceFactory; +import sonia.scm.util.IOUtil; +import sonia.scm.web.VndMediaType; + +import javax.ws.rs.Consumes; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +public class RepositoryImportResource { + + private static final Logger logger = LoggerFactory.getLogger(RepositoryImportResource.class); + + private final RepositoryManager manager; + private final RepositoryServiceFactory serviceFactory; + + @Inject + public RepositoryImportResource(RepositoryManager manager, + RepositoryServiceFactory serviceFactory) { + this.manager = manager; + this.serviceFactory = serviceFactory; + } + +// /** +// * Imports a repository type specific bundle. The bundle file is uploaded to +// * the server which is running scm-manager. After the upload has finished, the +// * bundle file is passed to the {@link UnbundleCommandBuilder}. Note: This method +// * requires admin privileges. +// * +// * @param uriInfo uri info +// * @param type repository type +// * @param name name of the repository +// * @param inputStream input bundle +// * @param compressed true if the bundle is gzip compressed +// * @return empty response with location header which points to the imported repository +// * @since 1.43 +// */ +// @POST +// @Path("{type}/bundle") +// @Consumes(MediaType.MULTIPART_FORM_DATA) +// public Response importFromBundle(@Context UriInfo uriInfo, +// @PathParam("type") String type, +// @FormParam("namespace") String namespace, +// @FormParam("name") String name, +// @FormParam("bundle") InputStream inputStream, +// @QueryParam("compressed") +// @DefaultValue("false") boolean compressed) { +// Repository repository = doImportFromBundle(type, namespace, name, inputStream, compressed); +// +// return buildResponse(uriInfo, repository); +// } +// +// /** +// * This method works exactly like +// * {@link #importFromBundle(UriInfo, String, String, String, InputStream, boolean)}, but this +// * method returns an html content-type. The method exists only for a +// * workaround of the javascript ui extjs. Note: This method requires admin +// * privileges. +// * +// * @param type repository type +// * @param name name of the repository +// * @param inputStream input bundle +// * @param compressed true if the bundle is gzip compressed +// * @return empty response with location header which points to the imported +// * repository +// * @since 1.43 +// */ +// @POST +// @Path("{type}/bundle.html") +// @Consumes(MediaType.MULTIPART_FORM_DATA) +// @Produces(MediaType.TEXT_HTML) +// public Response importFromBundleUI(@PathParam("type") String type, +// @FormParam("namespace") String namespace, +// @FormParam("name") String name, +// @FormParam("bundle") InputStream inputStream, +// @QueryParam("compressed") +// @DefaultValue("false") boolean compressed) { +// Response response; +// +// try { +// doImportFromBundle(type, namespace, name, inputStream, compressed); +// response = Response.ok(new RestActionUploadResult(true)).build(); +// } catch (WebApplicationException ex) { +// logger.warn("error durring bundle import", ex); +// response = Response.fromResponse(ex.getResponse()).entity( +// new RestActionUploadResult(false)).build(); +// } +// +// return response; +// } + + /** + * Imports a external repository which is accessible via url. The method can + * only be used, if the repository type supports the {@link Command#PULL}. The + * method will return a location header with the url to the imported + * repository. + * + * @param uriInfo uri info + * @param type repository type + * @param request request object + * @return empty response with location header which points to the imported + * repository + * @since 2.11.0 + */ + @POST + @Path("{type}/url") + @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) + @Operation(summary = "Import repository from url", description = "Imports the repository for the given url.", tags = "Repository") + @ApiResponse( + responseCode = "201", + description = "Repository import was successful", + content = @Content( + mediaType = VndMediaType.REPOSITORY, + schema = @Schema(implementation = RepositoryDto.class) + ) + ) + @ApiResponse( + responseCode = "401", + description = "not authenticated / invalid credentials" + ) + @ApiResponse( + responseCode = "403", + description = "not authorized, the current user has no privileges to read the repository" + ) + @ApiResponse( + responseCode = "500", + description = "internal server error", + content = @Content( + mediaType = VndMediaType.ERROR_TYPE, + schema = @Schema(implementation = ErrorDto.class) + ) + ) + public Response importFromUrl(@Context UriInfo uriInfo, + @PathParam("type") String type, UrlImportRequest request) { + RepositoryPermissions.create().check(); + checkNotNull(request, "request is required"); + checkArgument(!Strings.isNullOrEmpty(request.getName()), + "request does not contain name of the repository"); + checkArgument(!Strings.isNullOrEmpty(request.getUrl()), + "request does not contain url of the remote repository"); + + Type t = type(type); + + checkSupport(t, Command.PULL, request); + + logger.info("start {} import for external url {}", type, request.getUrl()); + + Repository repository = create(request.getNamespace(), request.getName(), type); + + try (RepositoryService service = serviceFactory.create(repository)) { + service.getPullCommand().pull(request.getUrl()); + } catch (IOException ex) { + handleImportFailure(ex, repository); + } + + return Response.created(createRepositoryLocation(uriInfo, repository)).build(); + } + + private URI createRepositoryLocation(UriInfo uriInfo, Repository repository) { + return URI.create( + String.format( + "%s/repos/%s", + uriInfo.getBaseUri().toString().replace("/api/", "/"), + repository.getNamespaceAndName() + ) + ); + } + +// /** +// * Imports repositories of the given type from the configured repository +// * directory. Note: This method requires admin privileges. +// * +// * @param type repository type +// * @return imported repositories +// */ +// @POST +// @Path("{type}") +// @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) +// public Response importRepositories(@PathParam("type") String type) { +// RepositoryPermissions.create().check(); +// +// List repositories = new ArrayList<>(); +// +// importFromDirectory(repositories, type); +// +// //J- +// return Response.ok( +// new GenericEntity>(repositories) { +// } +// ).build(); +// //J+ +// } +// +// /** +// * Imports repositories of all supported types from the configured repository +// * directories. Note: This method requires admin privileges. +// * +// * @return imported repositories +// */ +// @POST +// @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) +// public Response importRepositories() { +// RepositoryPermissions.create().check(); +// +// logger.info("start directory import for all supported repository types"); +// +// List repositories = new ArrayList(); +// +// for (Type t : findImportableTypes()) { +// importFromDirectory(repositories, t.getName()); +// } +// +// //J- +// return Response.ok( +// new GenericEntity>(repositories) { +// } +// ).build(); +// //J+ +// } +// +// /** +// * Imports repositories of the given type from the configured repository +// * directory. Returns a list of successfully imported directories and a list +// * of failed directories. Note: This method requires admin privileges. +// * +// * @param type repository type +// * @return imported repositories +// * @since 1.43 +// */ +// @POST +// @Path("{type}/directory") +// @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) +// public Response importRepositoriesFromDirectory( +// @PathParam("type") String type) { +// RepositoryPermissions.create().check(); +// +// Response response; +// +// RepositoryHandler handler = manager.getHandler(type); +// +// if (handler != null) { +// logger.info("start directory import for repository type {}", type); +// +// try { +// ImportResult result; +// ImportHandler importHandler = handler.getImportHandler(); +// +// if (importHandler instanceof AdvancedImportHandler) { +// logger.debug("start directory import, using advanced import handler"); +// result = +// ((AdvancedImportHandler) importHandler) +// .importRepositoriesFromDirectory(manager); +// } else { +// logger.debug("start directory import, using normal import handler"); +// result = new ImportResult(importHandler.importRepositories(manager), +// ImmutableList.of()); +// } +// +// response = Response.ok(result).build(); +// } catch (FeatureNotSupportedException ex) { +// logger +// .warn( +// "import feature is not supported by repository handler for type " +// .concat(type), ex); +// response = Response.status(Response.Status.BAD_REQUEST).build(); +// } catch (IOException ex) { +// logger.warn("exception occured durring directory import", ex); +// response = Response.serverError().build(); +// } +// } else { +// logger.warn("could not find reposiotry handler for type {}", type); +// response = Response.status(Response.Status.BAD_REQUEST).build(); +// } +// +// return response; +// } +// +// /** +// * Returns a list of repository types, which support the directory import +// * feature. Note: This method requires admin privileges. +// * +// * @return list of repository types +// */ +// @GET +// @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}) +// public Response getImportableTypes() { +// RepositoryPermissions.create().check(); +// +// List types = findImportableTypes(); +// +// //J- +// return Response.ok( +// new GenericEntity>(types) { +// } +// ).build(); +// //J+ +// } + + /** + * Check repository type for support for the given command. + * + * @param type repository type + * @param cmd command + * @param request request object + */ + private void checkSupport(Type type, Command cmd, Object request) { + if (!(type instanceof RepositoryType)) { + logger.warn("type {} is not a repository type", type.getName()); + + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + + Set cmds = ((RepositoryType) type).getSupportedCommands(); + + if (!cmds.contains(cmd)) { + logger.warn("type {} does not support this type of import: {}", + type.getName(), request); + + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + } + + /** + * Creates a new repository with the given namespace, name and type. + * + * @param namespace repository namespace + * @param name repository name + * @param type repository type + * @return newly created repository + */ + private Repository create(String namespace, String name, String type) { + Repository repository = null; + + try { + repository = new Repository(null, type, namespace, name); + manager.create(repository); + } catch (InternalRepositoryException ex) { + handleGenericCreationFailure(ex, type, name); + } + + return repository; + } + + /** + * Start bundle import. + * + * @param type repository type + * @param name name of the repository + * @param inputStream bundle stream + * @param compressed true if the bundle is gzip compressed + * @return imported repository + */ + private Repository doImportFromBundle(String type, String namespace, String name, + InputStream inputStream, boolean compressed) { + RepositoryPermissions.create().check(); + + checkArgument(!Strings.isNullOrEmpty(name), + "request does not contain name of the repository"); + checkNotNull(inputStream, "bundle inputStream is required"); + + Repository repository; + + try { + Type t = type(type); + checkSupport(t, Command.UNBUNDLE, "bundle"); + repository = create(namespace, name, type); + importFromBundle(repository, inputStream, compressed); + } catch (IOException ex) { + logger.warn("could not create temporary file", ex); + + throw new WebApplicationException(ex); + } + + return repository; + } + + private void importFromBundle(Repository repository, InputStream inputStream, boolean compressed) throws IOException { + File file = File.createTempFile("scm-import-", ".bundle"); + + try (RepositoryService service = serviceFactory.create(repository)) { + long length = Files.asByteSink(file).writeFrom(inputStream); + + logger.info("copied {} bytes to temp, start bundle import", length); + service.getUnbundleCommand().setCompressed(compressed).unbundle(file); + } catch (InternalRepositoryException ex) { + handleImportFailure(ex, repository); + } finally { + IOUtil.delete(file); + } + } + + private List findImportableTypes() { + List types = new ArrayList<>(); + Collection handlerTypes = manager.getTypes(); + + for (Type t : handlerTypes) { + RepositoryHandler handler = manager.getHandler(t.getName()); + + if (handler != null) { + try { + if (handler.getImportHandler() != null) { + types.add(t); + } + } catch (FeatureNotSupportedException ex) { + if (logger.isTraceEnabled()) { + logger.trace("import handler is not supported", ex); + } else if (logger.isInfoEnabled()) { + logger.info("{} handler does not support import of repositories", + t.getName()); + } + } + } else if (logger.isWarnEnabled()) { + logger.warn("could not find handler for type {}", t.getName()); + } + } + + return types; + } + + /** + * Handle creation failures. + * + * @param ex exception + * @param type repository type + * @param name name of the repository + */ + private void handleGenericCreationFailure(Exception ex, String type, + String name) { + logger.error(String.format("could not create repository %s with type %s", + type, name), ex); + + throw new WebApplicationException(ex); + } + + /** + * Handle import failures. + * + * @param ex exception + * @param repository repository + */ + private void handleImportFailure(Exception ex, Repository repository) { + logger.error("import for repository failed, delete repository", ex); + + try { + manager.delete(repository); + } catch (InternalRepositoryException | NotFoundException e) { + logger.error("can not delete repository after import failure", e); + } + + throw new WebApplicationException(ex, + Response.Status.INTERNAL_SERVER_ERROR); + } + + /** + * Import repositories from a specific type. + * + * @param repositories repository list + * @param type type of repository + */ + private void importFromDirectory(List repositories, String type) { + RepositoryHandler handler = manager.getHandler(type); + + if (handler != null) { + logger.info("start directory import for repository type {}", type); + + try { + List repositoryNames = + handler.getImportHandler().importRepositories(manager); + + if (repositoryNames != null) { + for (String repositoryName : repositoryNames) { + // TODO #8783 + /*Repository repository = null; //manager.get(type, repositoryName); + + if (repository != null) + { + repositories.add(repository); + } + else if (logger.isWarnEnabled()) + { + logger.warn("could not find imported repository {}", + repositoryName); + }*/ + } + } + } catch (FeatureNotSupportedException ex) { + throw new WebApplicationException(ex, Response.Status.BAD_REQUEST); + } catch (IOException ex) { + throw new WebApplicationException(ex); + } catch (InternalRepositoryException ex) { + throw new WebApplicationException(ex); + } + } else { + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + } + + private Type type(String type) { + RepositoryHandler handler = manager.getHandler(type); + + if (handler == null) { + logger.warn("no handler for type {} found", type); + + throw new WebApplicationException(Response.Status.NOT_FOUND); + } + + return handler.getType(); + } + + /** + * Request for importing external repositories which are accessible via url. + */ + @XmlRootElement(name = "import") + @XmlAccessorType(XmlAccessType.FIELD) + @Value + @ToString + public static class UrlImportRequest { + private String namespace; + private String name; + private String url; + private String username; + private String password; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryRootResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryRootResource.java index db2002cc4f..e65fee4f6d 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryRootResource.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryRootResource.java @@ -34,22 +34,22 @@ import javax.ws.rs.Path; /** * RESTful Web Service Resource to manage repositories. */ -@OpenAPIDefinition( - tags = { - @Tag(name = "Repository", description = "Repository related endpoints") - } -) +@OpenAPIDefinition(tags = { + @Tag(name = "Repository", description = "Repository related endpoints") +}) @Path(RepositoryRootResource.REPOSITORIES_PATH_V2) public class RepositoryRootResource { static final String REPOSITORIES_PATH_V2 = "v2/repositories/"; private final Provider repositoryResource; private final Provider repositoryCollectionResource; + private final Provider repositoryImportResource; @Inject - public RepositoryRootResource(Provider repositoryResource, Provider repositoryCollectionResource) { + public RepositoryRootResource(Provider repositoryResource, Provider repositoryCollectionResource, Provider repositoryImportResource) { this.repositoryResource = repositoryResource; this.repositoryCollectionResource = repositoryCollectionResource; + this.repositoryImportResource = repositoryImportResource; } @Path("{namespace}/{name}") @@ -61,4 +61,9 @@ public class RepositoryRootResource { public RepositoryCollectionResource getRepositoryCollectionResource() { return repositoryCollectionResource.get(); } + + @Path("import") + public RepositoryImportResource getRepositoryImportResource() { + return repositoryImportResource.get(); + } } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryTestBase.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryTestBase.java index f2276a6ebf..3c04fbac2b 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryTestBase.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryTestBase.java @@ -45,6 +45,7 @@ abstract class RepositoryTestBase { IncomingRootResource incomingRootResource; RepositoryCollectionResource repositoryCollectionResource; AnnotateResource annotateResource; + RepositoryImportResource repositoryImportResource; RepositoryRootResource getRepositoryRootResource() { RepositoryBasedResourceProvider repositoryBasedResourceProvider = new RepositoryBasedResourceProvider( @@ -65,6 +66,6 @@ abstract class RepositoryTestBase { dtoToRepositoryMapper, manager, repositoryBasedResourceProvider)), - of(repositoryCollectionResource)); + of(repositoryCollectionResource), of(repositoryImportResource)); } }