diff --git a/scm-core/src/main/java/sonia/scm/store/Blob.java b/scm-core/src/main/java/sonia/scm/store/Blob.java index 195697472a..a569e61ab5 100644 --- a/scm-core/src/main/java/sonia/scm/store/Blob.java +++ b/scm-core/src/main/java/sonia/scm/store/Blob.java @@ -85,4 +85,13 @@ public interface Blob * @throws IOException */ public OutputStream getOutputStream() throws IOException; + + /** + * + * Returns the size (in bytes) of the blob. + * @since 1.54 + */ + public long getSize(); + + } diff --git a/scm-dao-xml/src/main/java/sonia/scm/store/FileBlob.java b/scm-dao-xml/src/main/java/sonia/scm/store/FileBlob.java index 9c4cba88d9..33204536d5 100644 --- a/scm-dao-xml/src/main/java/sonia/scm/store/FileBlob.java +++ b/scm-dao-xml/src/main/java/sonia/scm/store/FileBlob.java @@ -119,6 +119,19 @@ public final class FileBlob implements Blob return new FileOutputStream(file); } + @Override + public long getSize() { + if (this.file.isFile()) { + + return this.file.length(); + } else { + + //to sum up all other cases, in which we cannot determine a size + return -1; + + } + } + //~--- fields --------------------------------------------------------------- /** Field description */ diff --git a/scm-plugins/scm-git-plugin/pom.xml b/scm-plugins/scm-git-plugin/pom.xml index 5ab04d1e1c..eb4469fcae 100644 --- a/scm-plugins/scm-git-plugin/pom.xml +++ b/scm-plugins/scm-git-plugin/pom.xml @@ -37,6 +37,12 @@ ${jgit.version} + + sonia.jgit + org.eclipse.jgit.lfs.server + ${jgit.version} + + commons-lang commons-lang @@ -51,6 +57,16 @@ 1.54-SNAPSHOT test + + sonia.scm + scm-dao-xml + 1.52-SNAPSHOT + + + sonia.scm + scm-dao-xml + 1.52-SNAPSHOT + diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitPermissionFilter.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitPermissionFilter.java index f682c355a6..3ff18e210e 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitPermissionFilter.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitPermissionFilter.java @@ -35,6 +35,7 @@ package sonia.scm.web; //~--- non-JDK imports -------------------------------------------------------- +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -127,13 +128,22 @@ public class GitPermissionFilter extends ProviderPermissionFilter * @return */ @Override - protected boolean isWriteRequest(HttpServletRequest request) - { + protected boolean isWriteRequest(HttpServletRequest request) { + String uri = request.getRequestURI(); - return uri.endsWith(URI_RECEIVE_PACK) - || (uri.endsWith(URI_REF_INFO) - && PARAMETER_VALUE_RECEIVE.equals( - request.getParameter(PARAMETER_SERVICE))); + return uri.endsWith(URI_RECEIVE_PACK) || + (uri.endsWith(URI_REF_INFO) && PARAMETER_VALUE_RECEIVE.equals(request.getParameter(PARAMETER_SERVICE))) || + isLfsFileUpload(request); + } + + @VisibleForTesting + static boolean isLfsFileUpload(HttpServletRequest request) { + String regex = String.format("^%s%s/.+(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH); + return request.getRequestURI().matches(regex) && "PUT".equals(request.getMethod()); + + } + + } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitServletModule.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitServletModule.java index fdb1ae52d9..3e85b4f3c2 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitServletModule.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitServletModule.java @@ -49,8 +49,11 @@ import sonia.scm.plugin.ext.Extension; public class GitServletModule extends ServletModule { + public static final String GIT_PATH = "/git"; + /** Field description */ - public static final String PATTERN_GIT = "/git/*"; + public static final String PATTERN_GIT = GIT_PATH + "/*"; + //~--- methods -------------------------------------------------------------- diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitUserAgentProvider.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitUserAgentProvider.java index 4fa7f15802..7d4111fa16 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitUserAgentProvider.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/GitUserAgentProvider.java @@ -53,6 +53,12 @@ public class GitUserAgentProvider implements UserAgentProvider false).basicAuthenticationCharset( Charsets.UTF_8).build(); + @VisibleForTesting + static final UserAgent GIT_LFS = UserAgent.builder("Git Lfs") + .browser(false) + .basicAuthenticationCharset(Charsets.UTF_8) + .build(); + /** Field description */ @VisibleForTesting static final UserAgent MSYSGIT = UserAgent.builder("msysGit").browser( @@ -60,10 +66,11 @@ public class GitUserAgentProvider implements UserAgentProvider Charsets.UTF_8).build(); /** Field description */ - private static final String PREFIX = "git/"; + private static final String PREFIX_REGULAR = "git/"; + private static final String PREFIX_LFS = "git-lfs/"; /** Field description */ - private static final String SUFFIX = "msysgit"; + private static final String SUFFIX_MSYSGIT = "msysgit"; //~--- methods -------------------------------------------------------------- @@ -80,16 +87,14 @@ public class GitUserAgentProvider implements UserAgentProvider { UserAgent ua = null; - if (userAgentString.startsWith(PREFIX)) - { - if (userAgentString.contains(SUFFIX)) - { + if (userAgentString.startsWith(PREFIX_REGULAR)) { + if (userAgentString.contains(SUFFIX_MSYSGIT)) { ua = MSYSGIT; - } - else - { + } else { ua = GIT; } + } else if (userAgentString.startsWith(PREFIX_LFS)) { + ua = GIT_LFS; } return ua; diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/ScmGitServlet.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/ScmGitServlet.java index 686e4dfbd7..5d3798ec3d 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/ScmGitServlet.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/ScmGitServlet.java @@ -35,23 +35,31 @@ package sonia.scm.web; //~--- non-JDK imports -------------------------------------------------------- +import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import com.google.inject.Singleton; import org.eclipse.jgit.http.server.GitServlet; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.slf4j.LoggerFactory.getLogger; + +import org.eclipse.jgit.lfs.lib.Constants; +import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON; + +import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryProvider; import sonia.scm.repository.RepositoryRequestListenerUtil; import sonia.scm.util.HttpUtil; +import sonia.scm.web.lfs.servlet.LfsServletFactory; //~--- JDK imports ------------------------------------------------------------ import java.io.IOException; import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import sonia.scm.repository.RepositoryException; @@ -73,7 +81,7 @@ public class ScmGitServlet extends GitServlet /** the logger for ScmGitServlet */ private static final Logger logger = - LoggerFactory.getLogger(ScmGitServlet.class); + getLogger(ScmGitServlet.class); //~--- constructors --------------------------------------------------------- @@ -90,14 +98,19 @@ public class ScmGitServlet extends GitServlet */ @Inject public ScmGitServlet(GitRepositoryResolver repositoryResolver, - GitReceivePackFactory receivePackFactory, - GitRepositoryViewer repositoryViewer, - RepositoryProvider repositoryProvider, - RepositoryRequestListenerUtil repositoryRequestListenerUtil) + GitReceivePackFactory receivePackFactory, + GitRepositoryViewer repositoryViewer, + RepositoryProvider repositoryProvider, + RepositoryRequestListenerUtil repositoryRequestListenerUtil, + LfsServletFactory lfsServletFactory, + UserAgentParser userAgentParser) { this.repositoryProvider = repositoryProvider; this.repositoryViewer = repositoryViewer; this.repositoryRequestListenerUtil = repositoryRequestListenerUtil; + this.lfsServletFactory = lfsServletFactory; + this.userAgentParser = userAgentParser; + setRepositoryResolver(repositoryResolver); setReceivePackFactory(receivePackFactory); } @@ -120,49 +133,44 @@ public class ScmGitServlet extends GitServlet throws ServletException, IOException { String uri = HttpUtil.getStrippedURI(request); + logger.trace("--request URI: {}", uri); - if (uri.matches(REGEX_GITHTTPBACKEND)) - { - sonia.scm.repository.Repository repository = repositoryProvider.get(); + //decide the type of response to be presented to the client + UserAgent userAgent = userAgentParser.parse(request); + if (userAgent.isBrowser()) { - if (repository != null) - { - if (repositoryRequestListenerUtil.callListeners(request, response, - repository)) - { - super.service(request, response); - } - else if (logger.isDebugEnabled()) - { + renderHtmlRepositryOverview(request, response); + } else { + + //service the request for a git client + final Repository repository = repositoryProvider.get(); + + if (repository == null) { + + //repository could not be matched found the current request + super.service(request, response); + } else { + + if (repositoryRequestListenerUtil.callListeners(request, response, repository)) { + handleRequest(request, response, repository); + } else if (logger.isDebugEnabled()) { logger.debug("request aborted by repository request listener"); } } - else - { - super.service(request, response); - } - } - else - { - printGitInformation(request, response); } } /** - * Method description - * - * - * + * This method renders basic information about the repository into the response. The result is meant to be viewed by + * browser. * @param request * @param response * * @throws IOException * @throws ServletException */ - private void printGitInformation(HttpServletRequest request, - HttpServletResponse response) - throws ServletException, IOException - { + private void renderHtmlRepositryOverview(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + sonia.scm.repository.Repository scmRepository = repositoryProvider.get(); if (scmRepository != null) @@ -186,6 +194,113 @@ public class ScmGitServlet extends GitServlet } } + /** + * Decides the type request being currently made and delegates it accordingly. + * + */ + private void handleRequest(HttpServletRequest request, HttpServletResponse response, Repository repository) throws ServletException, IOException { + + logger.trace("--- Repository is: {}", repository.getName()); + if (isLfsBatchApiRequest(request, repository.getName())) { + + logger.trace("--- detected LFS Batch API Request"); + HttpServlet servlet = lfsServletFactory.createProtocolServletFor(repository, request); + servlet.service(request, response); + } else if (isLfsFileTransferRequest(request, repository.getName())) { + + logger.trace("--- detected LFS File Transfer Request"); + HttpServlet servlet = lfsServletFactory.createFileLfsServletFor(repository, request); + servlet.service(request, response); + } else { + logger.trace("--- seems to be regular Git HTTP backend request: {}", request.getRequestURI()); + //continue to the regular HTTP Backend + super.service(request, response); + } + } + + /** + * Decides whether or not a request is for the LFS Batch API, + *

+ * - PUT or GET + * - exactly for this repository + * - Content Type is {@link Constants#HDR_APPLICATION_OCTET_STREAM}. + * + * @return Returns {@code false} if either of the conditions does not match. Returns true if all match. + */ + private static boolean isLfsFileTransferRequest(HttpServletRequest request, String repository) { + + String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/[a-z0-9]{64}$", request.getContextPath(), GitServletModule.GIT_PATH, repository); + boolean pathMatches = request.getRequestURI().matches(regex); + + boolean methodMatches = request.getMethod().equals("PUT") || request.getMethod().equals("GET"); + + return pathMatches && methodMatches; + } + + /** + * Decides whether or not a request is for the LFS Batch API, + *

+ * - POST + * - exactly for this repository + * - Content Type is {@link Constants#CONTENT_TYPE_GIT_LFS_JSON}. + * + * @return Returns {@code false} if either of the conditions does not match. Returns true if all match. + */ + private static boolean isLfsBatchApiRequest(HttpServletRequest request, String repository) { + + String regex = String.format("^%s%s/%s(\\.git)?/info/lfs/objects/batch$", request.getContextPath(), GitServletModule.GIT_PATH, repository); + boolean pathMatches = request.getRequestURI().matches(regex); + + boolean methodMatches = "POST".equals(request.getMethod()); + + boolean headerContentTypeMatches = isLfsContentHeaderField(request.getContentType(), CONTENT_TYPE_GIT_LFS_JSON); + boolean headerAcceptMatches = isLfsContentHeaderField(request.getHeader("Accept"), CONTENT_TYPE_GIT_LFS_JSON); + + return pathMatches && methodMatches && headerContentTypeMatches && headerAcceptMatches; + } + + /** + * Checks whether request is of the specific content type. + * + * @param request The HTTP request header value to be examined. + * @param expectedContentType The expected content type. + * @return Returns {@code true} if the request has the expected content type. Return {@code false} otherwise. + */ + @VisibleForTesting + static boolean isLfsContentHeaderField(String request, String expectedContentType) { + + if (request == null || request.isEmpty()) { + return false; + } + + String[] parts = request.split(" "); + for (String part : parts) { + if (part.startsWith(expectedContentType)) { + + return true; + } + } + + return false; + } + + //~--- fields --------------------------------------------------------------- /** Field description */ @@ -194,6 +309,14 @@ public class ScmGitServlet extends GitServlet /** Field description */ private final RepositoryRequestListenerUtil repositoryRequestListenerUtil; - /** Field description */ + /** + * Field description + */ private final GitRepositoryViewer repositoryViewer; + + private final LfsServletFactory lfsServletFactory; + + private final UserAgentParser userAgentParser; + + } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/ScmBlobLfsRepository.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/ScmBlobLfsRepository.java new file mode 100644 index 0000000000..46a58f6f07 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/ScmBlobLfsRepository.java @@ -0,0 +1,90 @@ +package sonia.scm.web.lfs; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.Response; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; + +import java.io.IOException; + +/** + * This LargeFileRepository is used for jGit-Servlet implementation. Under the jgit LFS Servlet hood, the + * SCM-Repository API is used to implement the Repository. + * + * @since 1.54 + * Created by omilke on 03.05.2017. + */ +public class ScmBlobLfsRepository implements LargeFileRepository { + + private final BlobStore blobStore; + + /** + * This URI is used to determine the actual URI for Upload / Download. Must be full URI (or rewritable by reverse + * proxy). + */ + private final String baseUri; + + /** + * Creates a {@link ScmBlobLfsRepository} for the provided repository. + * + * @param blobStore The SCM Blobstore used for this @{@link LargeFileRepository}. + * @param baseUri This URI is used to determine the actual URI for Upload / Download. Must be full URI (or + * rewritable by reverse proxy). + */ + + public ScmBlobLfsRepository(BlobStore blobStore, String baseUri) { + + this.blobStore = blobStore; + this.baseUri = baseUri; + } + + @Override + public Response.Action getDownloadAction(AnyLongObjectId id) { + + return getAction(id); + } + + @Override + public Response.Action getUploadAction(AnyLongObjectId id, long size) { + + return getAction(id); + } + + @Override + public Response.Action getVerifyAction(AnyLongObjectId id) { + + //validation is optional. We do not support it. + return null; + } + + @Override + public long getSize(AnyLongObjectId id) throws IOException { + + //this needs to be size of what is will be written into the response of the download. Clients are likely to + // verify it. + Blob blob = this.blobStore.get(id.getName()); + if (blob == null) { + + return -1; + } else { + + return blob.getSize(); + } + + } + + /** + * Constructs the Download / Upload actions to be supplied to the client. + */ + private Response.Action getAction(AnyLongObjectId id) { + + //LFS protocol has to provide the information on where to put or get the actual content, i. e. + //the actual URI for up- and download. + + Response.Action a = new Response.Action(); + a.href = baseUri + id.getName(); + + return a; + } +} diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/LfsServletFactory.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/LfsServletFactory.java new file mode 100644 index 0000000000..cbe4d1b99c --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/LfsServletFactory.java @@ -0,0 +1,98 @@ +package sonia.scm.web.lfs.servlet; + +import com.google.common.annotations.VisibleForTesting; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.LfsProtocolServlet; +import org.eclipse.jgit.lfs.server.fs.FileLfsServlet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.repository.Repository; +import sonia.scm.store.BlobStore; +import sonia.scm.store.BlobStoreFactory; +import sonia.scm.util.HttpUtil; +import sonia.scm.web.lfs.ScmBlobLfsRepository; + +import javax.inject.Inject; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; + +/** + * This factory class is a helper class to provide the {@link LfsProtocolServlet} and the {@link FileLfsServlet} + * belonging to a SCM Repository. + * + * @since 1.54 + * Created by omilke on 11.05.2017. + */ +public class LfsServletFactory { + + private static final String GIT_LFS_REPOSITORY_POSTFIX = "-git-lfs"; + + private static final Logger logger = LoggerFactory.getLogger(LfsServletFactory.class); + + private final BlobStoreFactory blobStoreFactory; + + @Inject + public LfsServletFactory(BlobStoreFactory blobStoreFactory) { + + this.blobStoreFactory = blobStoreFactory; + } + + /** + * Builds the {@link LfsProtocolServlet} (jgit API) for a SCM Repository. + * + * @param repository The SCM Repository to build the servlet for. + * @param request The {@link HttpServletRequest} the used to access the SCM Repository. + * @return The {@link LfsProtocolServlet} to provide the LFS Batch API for a SCM Repository. + */ + public LfsProtocolServlet createProtocolServletFor(Repository repository, HttpServletRequest request) { + + BlobStore blobStore = getBlobStore(repository); + String baseUri = buildBaseUri(repository, request); + + LargeFileRepository largeFileRepository = new ScmBlobLfsRepository(blobStore, baseUri); + return new ScmLfsProtocolServlet(largeFileRepository); + } + + /** + * Builds the {@link FileLfsServlet} (jgit API) for a SCM Repository. + * + * @param repository The SCM Repository to build the servlet for. + * @param request The {@link HttpServletRequest} the used to access the SCM Repository. + * @return The {@link FileLfsServlet} to provide the LFS Upload / Download API for a SCM Repository. + */ + public HttpServlet createFileLfsServletFor(Repository repository, HttpServletRequest request) { + + return new ScmFileTransferServlet(getBlobStore(repository)); + } + + /** + * Build the complete URI, under which the File Transfer API for this repository will be will be reachable. + * + * @param repository The repository to build the File Transfer URI for. + * @param request The request to construct the complete URI from. + */ + @VisibleForTesting + static String buildBaseUri(Repository repository, HttpServletRequest request) { + + return String.format("%s/git/%s.git/info/lfs/objects/", HttpUtil.getCompleteUrl(request), repository.getName()); + } + + /** + * Provides a {@link BlobStore} corresponding to the SCM Repository. + *

+ * git-lfs repositories should generally carry the same name as their regular SCM repository counterparts. However, + * we have decided to store them under their IDs instead of their names, since the names might change and provide + * other drawbacks, as well. + *

+ * These repositories will have {@linkplain #GIT_LFS_REPOSITORY_POSTFIX} appended to their IDs. + * + * @param repository The SCM Repository to provide a LFS {@link BlobStore} for. + */ + @VisibleForTesting + BlobStore getBlobStore(Repository repository) { + + return blobStoreFactory.getBlobStore(repository.getId() + GIT_LFS_REPOSITORY_POSTFIX); + } + + +} diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmFileTransferServlet.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmFileTransferServlet.java new file mode 100644 index 0000000000..2d0026ac9a --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmFileTransferServlet.java @@ -0,0 +1,278 @@ +package sonia.scm.web.lfs.servlet; + +import com.google.common.annotations.VisibleForTesting; +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.http.HttpStatus; +import org.eclipse.jgit.lfs.errors.CorruptLongObjectException; +import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lfs.server.LfsProtocolServlet; +import org.eclipse.jgit.lfs.server.fs.FileLfsServlet; +import org.eclipse.jgit.lfs.server.internal.LfsServerText; +import org.eclipse.jgit.util.HttpSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.util.IOUtil; + +import javax.servlet.ServletException; +import javax.servlet.ServletInputStream; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.text.MessageFormat; + +/** + * This Servlet provides the upload and download of files via git-lfs. + *

+ * This implementation is based on {@link FileLfsServlet} but adjusted to work with + * servlet-2.5 instead of servlet-3.1. + *

+ * + * @see FileLfsServlet + * @since 1.54 + * Created by omilke on 15.05.2017. + */ +public class ScmFileTransferServlet extends HttpServlet { + + private static final Logger logger = LoggerFactory.getLogger(ScmFileTransferServlet.class); + + private static final long serialVersionUID = 1L; + + /** + * Gson is used because the implementation was based on the jgit implementation. However the {@link LfsProtocolServlet} (which we do use in + * {@link ScmLfsProtocolServlet}) also uses Gson, which currently ties us to Gson anyway. + */ + private static Gson gson = createGson(); + + private final BlobStore blobStore; + + public ScmFileTransferServlet(BlobStore store) { + + this.blobStore = store; + } + + + /** + * Extracts the part after the last slash from path. + * + * @return Returns {@code null} if the part after the last slash is itself {@code null} or if its length is not 64. + */ + @VisibleForTesting + static String objectIdFromPath(String info) { + + int lastSlash = info.lastIndexOf('/'); + String potentialObjectId = info.substring(lastSlash + 1); + + if (potentialObjectId.length() != 64) { + return null; + + } else { + return potentialObjectId; + } + } + + /** + * Logs the message and provides it to the client. + * + * @param response The response + * @param status The HTTP Status Code to be provided to the client. + * @param message the message to used for server-side logging. It is also provided to the client. + */ + private static void sendErrorAndLog(HttpServletResponse response, int status, String message) throws IOException { + + logger.warn("Error occurred during git-lfs file transfer: {}", message); + + sendError(response, status, message); + } + + /** + * Logs the exception and provides only the message of the exception to the client. + * + * @param response The response + * @param status The HTTP Status Code to be provided to the client. + * @param exception An exception to used for server-side logging. + */ + private static void sendErrorAndLog(HttpServletResponse response, int status, Exception exception) throws IOException { + + logger.warn("Error occurred during git-lfs file transfer.", exception); + String message = exception.getMessage(); + + + sendError(response, status, message); + } + + private static void sendError(HttpServletResponse response, int status, String message) throws IOException { + + try (PrintWriter writer = response.getWriter()) { + + gson.toJson(new Error(message), writer); + + response.setStatus(status); + writer.flush(); + } + + response.flushBuffer(); + } + + private static Gson createGson() { + + GsonBuilder gb = new GsonBuilder().setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setPrettyPrinting().disableHtmlEscaping(); + return gb.create(); + } + + /** + * Provides a blob to download. + *

+ * Actual implementation is based on org.eclipse.jgit.lfs.server.fs.ObjectDownloadListener and adjusted + * to non-async as we're currently on servlet-2.5. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + AnyLongObjectId objectId = getObjectToTransfer(request, response); + if (objectId == null) { + + logInvalidObjectId(request.getRequestURI()); + } else { + + final String objectIdName = objectId.getName(); + logger.trace("---- providing download for LFS-Oid: {}", objectIdName); + + Blob savedBlob = blobStore.get(objectIdName); + if (isBlobPresent(savedBlob)) { + + logger.trace("----- Object {}: providing {} bytes", objectIdName, savedBlob.getSize()); + writeBlobIntoResponse(savedBlob, response); + } else { + + sendErrorAndLog(response, HttpStatus.SC_NOT_FOUND, MessageFormat.format(LfsServerText.get().objectNotFound, objectIdName)); + } + } + } + + /** + * Receives a blob from an upload. + *

+ * Actual implementation is based on org.eclipse.jgit.lfs.server.fs.ObjectUploadListener and adjusted + * to non-async as we're currently on servlet-2.5. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { + + AnyLongObjectId objectId = getObjectToTransfer(request, response); + if (objectId == null) { + + logInvalidObjectId(request.getRequestURI()); + } else { + + logger.trace("---- receiving upload for LFS-Oid: {}", objectId.getName()); + readBlobFromResponse(request, response, objectId); + } + } + + /** + * Extracts the {@link LongObjectId} from the request. Finishes the request, in case the {@link LongObjectId} cannot + * be extracted with an appropriate error. + * + * @throws IOException Thrown if the response could not be completed in an error case. + */ + private AnyLongObjectId getObjectToTransfer(HttpServletRequest request, HttpServletResponse response) throws IOException { + + String path = request.getPathInfo(); + + String objectIdFromPath = objectIdFromPath(path); + if (objectIdFromPath == null) { + + //ObjectId is not retrievable from URL + sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, MessageFormat.format(LfsServerText.get().invalidPathInfo, path)); + return null; + } else { + try { + return LongObjectId.fromString(objectIdFromPath); + } catch (InvalidLongObjectIdException e) { + + sendErrorAndLog(response, HttpStatus.SC_UNPROCESSABLE_ENTITY, e); + return null; + } + } + } + + private void logInvalidObjectId(String requestURI) { + + logger.warn("---- could not extract Oid from Request. Path seems to be invalid: {}", requestURI); + } + + private boolean isBlobPresent(Blob savedBlob) { + + return savedBlob != null && savedBlob.getSize() >= 0; + } + + private void writeBlobIntoResponse(Blob savedBlob, HttpServletResponse response) throws IOException { + + try (ServletOutputStream responseOutputStream = response.getOutputStream(); + InputStream savedBlobInputStream = savedBlob.getInputStream()) { + + response.addHeader(HttpSupport.HDR_CONTENT_LENGTH, String.valueOf(savedBlob.getSize())); + response.setContentType(Constants.HDR_APPLICATION_OCTET_STREAM); + + IOUtil.copy(savedBlobInputStream, responseOutputStream); + } catch (IOException ex) { + + sendErrorAndLog(response, HttpStatus.SC_INTERNAL_SERVER_ERROR, ex); + } + + } + + private void readBlobFromResponse(HttpServletRequest request, HttpServletResponse response, AnyLongObjectId objectId) throws IOException { + + try (OutputStream blobOutputStream = blobStore.create(objectId.getName()).getOutputStream(); + ServletInputStream requestInputStream = request.getInputStream()) { + + IOUtil.copy(requestInputStream, blobOutputStream); + + response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON); + response.setStatus(HttpServletResponse.SC_OK); + } catch (CorruptLongObjectException ex) { + + sendErrorAndLog(response, HttpStatus.SC_BAD_REQUEST, ex); + } + + + } + + /** + * Used for providing an error message. + */ + private static class Error { + String message; + + Error(String m) { + + this.message = m; + } + } + +} + + diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmLfsProtocolServlet.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmLfsProtocolServlet.java new file mode 100644 index 0000000000..332cf12e09 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/web/lfs/servlet/ScmLfsProtocolServlet.java @@ -0,0 +1,26 @@ +package sonia.scm.web.lfs.servlet; + +import org.eclipse.jgit.lfs.errors.LfsException; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.LfsProtocolServlet; + +/** + * Provides an implementation for the git-lfs Batch API. + * + * @since 1.54 + * Created by omilke on 11.05.2017. + */ +public class ScmLfsProtocolServlet extends LfsProtocolServlet { + + private final LargeFileRepository repository; + + public ScmLfsProtocolServlet(LargeFileRepository largeFileRepository) { + this.repository = largeFileRepository; + } + + + @Override + protected LargeFileRepository getLargeFileRepository(LfsRequest request, String path) throws LfsException { + return repository; + } +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitPermissionFilterTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitPermissionFilterTest.java new file mode 100644 index 0000000000..a107f35816 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitPermissionFilterTest.java @@ -0,0 +1,48 @@ +package sonia.scm.web; + +import org.junit.Test; + +import javax.servlet.http.HttpServletRequest; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Created by omilke on 19.05.2017. + */ +public class GitPermissionFilterTest { + + @Test + public void isLfsFileUpload() throws Exception { + + HttpServletRequest mockedRequest = getRequestWithMethodAndPathInfo("PUT", + "/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec"); + assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(true)); + + mockedRequest = getRequestWithMethodAndPathInfo("GET", + "/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec"); + assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false)); + + mockedRequest = getRequestWithMethodAndPathInfo("POST", + "/scm/git/git-lfs-demo.git/info/lfs/objects/8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec"); + assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false)); + + mockedRequest = getRequestWithMethodAndPathInfo("POST", + "/scm/git/git-lfs-demo.git/info/lfs/objects/batch"); + assertThat((GitPermissionFilter.isLfsFileUpload(mockedRequest)), is(false)); + } + + private HttpServletRequest getRequestWithMethodAndPathInfo(String method, String pathInfo) { + + HttpServletRequest mock = mock(HttpServletRequest.class); + + when(mock.getMethod()).thenReturn(method); + when(mock.getRequestURI()).thenReturn(pathInfo); + when(mock.getContextPath()).thenReturn("/scm"); + + return mock; + } + +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitUserAgentProviderTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitUserAgentProviderTest.java index b16580c4a5..67c2be69de 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitUserAgentProviderTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/GitUserAgentProviderTest.java @@ -58,6 +58,7 @@ public class GitUserAgentProviderTest public void testParseUserAgent() { assertEquals(GitUserAgentProvider.GIT, parse("git/1.7.9.5")); + assertEquals(GitUserAgentProvider.GIT_LFS, parse("git-lfs/2.0.1 (GitHub; windows amd64; go 1.8; git 678cdbd4)")); assertEquals(GitUserAgentProvider.MSYSGIT, parse("git/1.8.3.msysgit.0")); assertNull(parse("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36")); } diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/ScmGitServletTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/ScmGitServletTest.java new file mode 100644 index 0000000000..4013980cb1 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/ScmGitServletTest.java @@ -0,0 +1,26 @@ +package sonia.scm.web; + +import org.junit.Test; + +import static org.eclipse.jgit.lfs.lib.Constants.CONTENT_TYPE_GIT_LFS_JSON; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * Created by omilke on 11.05.2017. + */ +public class ScmGitServletTest { + + @Test + public void isContentTypeMatches() throws Exception { + + assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json", CONTENT_TYPE_GIT_LFS_JSON), is(true)); + assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json;", CONTENT_TYPE_GIT_LFS_JSON), is(true)); + assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs+json; charset=utf-8", CONTENT_TYPE_GIT_LFS_JSON), is(true)); + + assertThat(ScmGitServlet.isLfsContentHeaderField("application/vnd.git-lfs-json;", CONTENT_TYPE_GIT_LFS_JSON), is(false)); + assertThat(ScmGitServlet.isLfsContentHeaderField("", CONTENT_TYPE_GIT_LFS_JSON), is(false)); + assertThat(ScmGitServlet.isLfsContentHeaderField(null, CONTENT_TYPE_GIT_LFS_JSON), is(false)); + } + +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/LfsServletFactoryTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/LfsServletFactoryTest.java new file mode 100644 index 0000000000..09b3b13082 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/LfsServletFactoryTest.java @@ -0,0 +1,75 @@ +package sonia.scm.web.lfs.servlet; + +import org.junit.Test; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryTestData; +import sonia.scm.store.BlobStoreFactory; + +import javax.servlet.http.HttpServletRequest; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.*; +import static org.mockito.Matchers.matches; +import static org.mockito.Mockito.*; + +/** + * Created by omilke on 18.05.2017. + */ +public class LfsServletFactoryTest { + + @Test + public void buildBaseUri() throws Exception { + + String repositoryName = "git-lfs-demo"; + + String result = LfsServletFactory.buildBaseUri(new Repository("", "GIT", repositoryName), RequestWithUri(repositoryName, true)); + assertThat(result, is(equalTo("http://localhost:8081/scm/git/git-lfs-demo.git/info/lfs/objects/"))); + + + //result will be with dot-gix suffix, ide + result = LfsServletFactory.buildBaseUri(new Repository("", "GIT", repositoryName), RequestWithUri(repositoryName, false)); + assertThat(result, is(equalTo("http://localhost:8081/scm/git/git-lfs-demo.git/info/lfs/objects/"))); + } + + @Test + public void getBlobStore() throws Exception { + + BlobStoreFactory blobStoreFactoryMock = mock(BlobStoreFactory.class); + + //TODO #239: + RepositoryTestData repositoryTestData; + + + new LfsServletFactory(blobStoreFactoryMock).getBlobStore(new Repository("the-id", "GIT", "the-name")); + + //just make sure the right parameter is passed, as properly validating the return value is nearly impossible with the return value (and should not be + // part of this test) + verify(blobStoreFactoryMock).getBlobStore(matches("the-id-git-lfs")); + + //make sure there have been no further usages of the factory + verifyNoMoreInteractions(blobStoreFactoryMock); + } + + private HttpServletRequest RequestWithUri(String repositoryName, boolean withDotGitSuffix) { + + HttpServletRequest mockedRequest = mock(HttpServletRequest.class); + + final String suffix; + if (withDotGitSuffix) { + suffix = ".git"; + } else { + suffix = ""; + } + + //build from valid live request data + when(mockedRequest.getRequestURL()).thenReturn( + new StringBuffer(String.format("http://localhost:8081/scm/git/%s%s/info/lfs/objects/batch", repositoryName, suffix))); + when(mockedRequest.getRequestURI()).thenReturn(String.format("/scm/git/%s%s/info/lfs/objects/batch", repositoryName, suffix)); + when(mockedRequest.getContextPath()).thenReturn("/scm"); + + return mockedRequest; + } + + +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/ScmFileTransferServletTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/ScmFileTransferServletTest.java new file mode 100644 index 0000000000..3caa02b272 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/web/lfs/servlet/ScmFileTransferServletTest.java @@ -0,0 +1,42 @@ +package sonia.scm.web.lfs.servlet; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.junit.Assert.*; + +/** + * Created by omilke on 16.05.2017. + */ +public class ScmFileTransferServletTest { + + @Test + public void hasObjectId() throws Exception { + + String SAMPLE_OBJECT_ID = "8fcebeb5698230685f92028e560f8f1683ebc15ec82a620ffad5c12a3c19bdec"; + + String path = "/git-lfs-demo.git/info/lfs/objects/" + SAMPLE_OBJECT_ID; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID))); + + path = "/" + SAMPLE_OBJECT_ID; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID))); + + path = SAMPLE_OBJECT_ID; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(equalTo(SAMPLE_OBJECT_ID))); + + String nonObjectId = "this-ist-last-to-found"; + path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue())); + + nonObjectId = SAMPLE_OBJECT_ID.substring(1); + path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue())); + + nonObjectId = SAMPLE_OBJECT_ID + "X"; + path = "/git-lfs-demo.git/info/lfs/objects/" + nonObjectId; + assertThat(ScmFileTransferServlet.objectIdFromPath(path), is(nullValue())); + + } +}