diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/GitUtil.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/GitUtil.java index 7175d3b646..9bc40f409a 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/GitUtil.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/GitUtil.java @@ -42,7 +42,10 @@ import com.google.common.collect.Multimap; import org.eclipse.jgit.api.FetchCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.lfs.LfsPointer; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -55,6 +58,7 @@ import org.eclipse.jgit.transport.FetchResult; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.LfsFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.ContextEntry; @@ -65,10 +69,12 @@ import sonia.scm.web.GitUserAgentProvider; import javax.servlet.http.HttpServletRequest; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import static java.util.Optional.empty; import static java.util.Optional.of; //~--- JDK imports ------------------------------------------------------------ @@ -731,6 +737,20 @@ public final class GitUtil } } + public static Optional getLfsPointer(org.eclipse.jgit.lib.Repository repo, String path, RevCommit commit, TreeWalk treeWalk) throws IOException { + Attributes attributes = LfsFactory.getAttributesForPath(repo, path, commit); + + Attribute filter = attributes.get("filter"); + if (filter != null && "lfs".equals(filter.getValue())) { + ObjectId blobId = treeWalk.getObjectId(0); + try (InputStream is = repo.open(blobId, Constants.OBJ_BLOB).openStream()) { + return of(LfsPointer.parseLfsPointer(is)); + } + } else { + return empty(); + } + } + //~--- methods -------------------------------------------------------------- /** diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java index 2a254d96ce..5ec69cccdd 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitBrowseCommand.java @@ -38,6 +38,7 @@ package sonia.scm.repository.spi; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.eclipse.jgit.lfs.LfsPointer; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -57,13 +58,17 @@ import sonia.scm.repository.GitSubModuleParser; import sonia.scm.repository.GitUtil; import sonia.scm.repository.Repository; import sonia.scm.repository.SubRepository; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; import sonia.scm.util.Util; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import static sonia.scm.ContextEntry.ContextBuilder.entity; import static sonia.scm.NotFoundException.notFound; @@ -86,18 +91,20 @@ public class GitBrowseCommand extends AbstractGitCommand */ private static final Logger logger = LoggerFactory.getLogger(GitBrowseCommand.class); + private final LfsBlobStoreFactory lfsBlobStoreFactory; //~--- constructors --------------------------------------------------------- /** * Constructs ... - * - * @param context + * @param context * @param repository + * @param lfsBlobStoreFactory */ - public GitBrowseCommand(GitContext context, Repository repository) + public GitBrowseCommand(GitContext context, Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory) { super(context, repository); + this.lfsBlobStoreFactory = lfsBlobStoreFactory; } //~--- get methods ---------------------------------------------------------- @@ -167,7 +174,7 @@ public class GitBrowseCommand extends AbstractGitCommand * @throws IOException */ private FileObject createFileObject(org.eclipse.jgit.lib.Repository repo, - BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) + BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) throws IOException { FileObject file = new FileObject(); @@ -195,7 +202,6 @@ public class GitBrowseCommand extends AbstractGitCommand ObjectLoader loader = repo.open(treeWalk.getObjectId(0)); file.setDirectory(loader.getType() == Constants.OBJ_TREE); - file.setLength(loader.getSize()); // don't show message and date for directories to improve performance if (!file.isDirectory() &&!request.isDisableLastCommit()) @@ -203,6 +209,16 @@ public class GitBrowseCommand extends AbstractGitCommand logger.trace("fetch last commit for {} at {}", path, revId.getName()); RevCommit commit = getLatestCommit(repo, revId, path); + Optional lfsPointer = GitUtil.getLfsPointer(repo, path, commit, treeWalk); + + if (lfsPointer.isPresent()) { + BlobStore lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository); + Blob blob = lfsBlobStore.get(lfsPointer.get().getOid().getName()); + file.setLength(blob.getSize()); + } else { + file.setLength(loader.getSize()); + } + if (commit != null) { file.setLastModified(GitUtil.getCommitTime(commit)); @@ -232,7 +248,7 @@ public class GitBrowseCommand extends AbstractGitCommand * @return */ private RevCommit getLatestCommit(org.eclipse.jgit.lib.Repository repo, - ObjectId revId, String path) + ObjectId revId, String path) { RevCommit result = null; RevWalk walk = null; @@ -339,7 +355,7 @@ public class GitBrowseCommand extends AbstractGitCommand } private FileObject findFirstMatch(org.eclipse.jgit.lib.Repository repo, - BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) throws IOException { + BrowseCommandRequest request, ObjectId revId, TreeWalk treeWalk) throws IOException { String[] pathElements = request.getPath().split("/"); int currentDepth = 0; int limit = pathElements.length; @@ -364,7 +380,7 @@ public class GitBrowseCommand extends AbstractGitCommand @SuppressWarnings("unchecked") private Map getSubRepositories(org.eclipse.jgit.lib.Repository repo, - ObjectId revision) + ObjectId revision) throws IOException { if (logger.isDebugEnabled()) { @@ -375,7 +391,7 @@ public class GitBrowseCommand extends AbstractGitCommand Map subRepositories; try ( ByteArrayOutputStream baos = new ByteArrayOutputStream() ) { - new GitCatCommand(context, repository).getContent(repo, revision, + new GitCatCommand(context, repository, lfsBlobStoreFactory).getContent(repo, revision, PATH_MODULES, baos); subRepositories = GitSubModuleParser.parse(baos.toString()); } @@ -389,7 +405,7 @@ public class GitBrowseCommand extends AbstractGitCommand } private SubRepository getSubRepository(org.eclipse.jgit.lib.Repository repo, - ObjectId revId, String path) + ObjectId revId, String path) throws IOException { Map subRepositories = subrepositoryCache.get(revId); @@ -410,7 +426,7 @@ public class GitBrowseCommand extends AbstractGitCommand } //~--- fields --------------------------------------------------------------- - + /** sub repository cache */ private final Map> subrepositoryCache = Maps.newHashMap(); } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java index 7477e0aee3..35ff4d6ac2 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitCatCommand.java @@ -33,6 +33,7 @@ package sonia.scm.repository.spi; import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lfs.LfsPointer; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -45,13 +46,18 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.repository.GitUtil; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.util.IOUtil; import sonia.scm.util.Util; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.Closeable; import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.Optional; import static sonia.scm.ContextEntry.ContextBuilder.entity; import static sonia.scm.NotFoundException.notFound; @@ -61,15 +67,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand { private static final Logger logger = LoggerFactory.getLogger(GitCatCommand.class); - public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository) { + private final LfsBlobStoreFactory lfsBlobStoreFactory; + + public GitCatCommand(GitContext context, sonia.scm.repository.Repository repository, LfsBlobStoreFactory lfsBlobStoreFactory) { super(context, repository); + this.lfsBlobStoreFactory = lfsBlobStoreFactory; } @Override public void getCatResult(CatCommandRequest request, OutputStream output) throws IOException { logger.debug("try to read content for {}", request); - try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(request)) { - closableObjectLoaderContainer.objectLoader.copyTo(output); + try (Loader closableObjectLoaderContainer = getLoader(request)) { + closableObjectLoaderContainer.copyTo(output); } } @@ -80,18 +89,18 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand { } void getContent(org.eclipse.jgit.lib.Repository repo, ObjectId revId, String path, OutputStream output) throws IOException { - try (ClosableObjectLoaderContainer closableObjectLoaderContainer = getLoader(repo, revId, path)) { - closableObjectLoaderContainer.objectLoader.copyTo(output); + try (Loader closableObjectLoaderContainer = getLoader(repo, revId, path)) { + closableObjectLoaderContainer.copyTo(output); } } - private ClosableObjectLoaderContainer getLoader(CatCommandRequest request) throws IOException { + private Loader getLoader(CatCommandRequest request) throws IOException { org.eclipse.jgit.lib.Repository repo = open(); ObjectId revId = getCommitOrDefault(repo, request.getRevision()); return getLoader(repo, revId, request.getPath()); } - private ClosableObjectLoaderContainer getLoader(Repository repo, ObjectId revId, String path) throws IOException { + private Loader getLoader(Repository repo, ObjectId revId, String path) throws IOException { TreeWalk treeWalk = new TreeWalk(repo); treeWalk.setRecursive(Util.nonNull(path).contains("/")); @@ -116,21 +125,67 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand { treeWalk.setFilter(PathFilter.create(path)); if (treeWalk.next() && treeWalk.getFileMode(0).getObjectType() == Constants.OBJ_BLOB) { - ObjectId blobId = treeWalk.getObjectId(0); - ObjectLoader loader = repo.open(blobId); - - return new ClosableObjectLoaderContainer(loader, treeWalk, revWalk); + Optional lfsPointer = GitUtil.getLfsPointer(repo, path, entry, treeWalk); + if (lfsPointer.isPresent()) { + return loadFromLfsStore(treeWalk, revWalk, lfsPointer.get()); + } else { + return loadFromGit(repo, treeWalk, revWalk); + } } else { throw notFound(entity("Path", path).in("Revision", revId.getName()).in(repository)); } } - private static class ClosableObjectLoaderContainer implements Closeable { + private Loader loadFromGit(Repository repo, TreeWalk treeWalk, RevWalk revWalk) throws IOException { + ObjectId blobId = treeWalk.getObjectId(0); + ObjectLoader loader = repo.open(blobId); + + return new GitObjectLoaderWrapper(loader, treeWalk, revWalk); + } + + private Loader loadFromLfsStore(TreeWalk treeWalk, RevWalk revWalk, LfsPointer lfsPointer) throws IOException { + BlobStore lfsBlobStore = lfsBlobStoreFactory.getLfsBlobStore(repository); + Blob blob = lfsBlobStore.get(lfsPointer.getOid().getName()); + GitUtil.release(revWalk); + GitUtil.release(treeWalk); + return new BlobLoader(blob); + } + + private interface Loader extends Closeable { + void copyTo(OutputStream output) throws IOException; + + InputStream openStream() throws IOException; + } + + private static class BlobLoader implements Loader { + private final InputStream inputStream; + + private BlobLoader(Blob blob) throws IOException { + this.inputStream = blob.getInputStream(); + } + + @Override + public void copyTo(OutputStream output) throws IOException { + IOUtil.copy(inputStream, output); + } + + @Override + public InputStream openStream() { + return inputStream; + } + + @Override + public void close() throws IOException { + this.inputStream.close(); + } + } + + private static class GitObjectLoaderWrapper implements Loader { private final ObjectLoader objectLoader; private final TreeWalk treeWalk; private final RevWalk revWalk; - private ClosableObjectLoaderContainer(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) { + private GitObjectLoaderWrapper(ObjectLoader objectLoader, TreeWalk treeWalk, RevWalk revWalk) { this.objectLoader = objectLoader; this.treeWalk = treeWalk; this.revWalk = revWalk; @@ -141,14 +196,22 @@ public class GitCatCommand extends AbstractGitCommand implements CatCommand { GitUtil.release(revWalk); GitUtil.release(treeWalk); } + + public void copyTo(OutputStream output) throws IOException { + this.objectLoader.copyTo(output); + } + + public InputStream openStream() throws IOException { + return objectLoader.openStream(); + } } private static class InputStreamWrapper extends FilterInputStream { - private final ClosableObjectLoaderContainer container; + private final Loader container; - private InputStreamWrapper(ClosableObjectLoaderContainer container) throws IOException { - super(container.objectLoader.openStream()); + private InputStreamWrapper(Loader container) throws IOException { + super(container.openStream()); this.container = container; } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java index 24dcff01d8..4c02a3a73c 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java @@ -39,6 +39,7 @@ import sonia.scm.repository.Feature; import sonia.scm.repository.GitRepositoryHandler; import sonia.scm.repository.Repository; import sonia.scm.repository.api.Command; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.IOException; import java.util.EnumSet; @@ -76,9 +77,10 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider //~--- constructors --------------------------------------------------------- - public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider) { + public GitRepositoryServiceProvider(GitRepositoryHandler handler, Repository repository, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) { this.handler = handler; this.repository = repository; + this.lfsBlobStoreFactory = lfsBlobStoreFactory; this.context = new GitContext(handler.getDirectory(repository.getId()), repository, storeProvider); } @@ -143,7 +145,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider @Override public BrowseCommand getBrowseCommand() { - return new GitBrowseCommand(context, repository); + return new GitBrowseCommand(context, repository, lfsBlobStoreFactory); } /** @@ -155,7 +157,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider @Override public CatCommand getCatCommand() { - return new GitCatCommand(context, repository); + return new GitCatCommand(context, repository, lfsBlobStoreFactory); } /** @@ -281,11 +283,13 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider //~--- fields --------------------------------------------------------------- /** Field description */ - private GitContext context; + private final GitContext context; /** Field description */ - private GitRepositoryHandler handler; + private final GitRepositoryHandler handler; /** Field description */ - private Repository repository; + private final Repository repository; + + private final LfsBlobStoreFactory lfsBlobStoreFactory; } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java index 0730ffc9cf..547c6b25f8 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceResolver.java @@ -39,6 +39,7 @@ import sonia.scm.api.v2.resources.GitRepositoryConfigStoreProvider; import sonia.scm.plugin.Extension; import sonia.scm.repository.GitRepositoryHandler; import sonia.scm.repository.Repository; +import sonia.scm.web.lfs.LfsBlobStoreFactory; /** * @@ -49,11 +50,13 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver { private final GitRepositoryHandler handler; private final GitRepositoryConfigStoreProvider storeProvider; + private final LfsBlobStoreFactory lfsBlobStoreFactory; @Inject - public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider) { + public GitRepositoryServiceResolver(GitRepositoryHandler handler, GitRepositoryConfigStoreProvider storeProvider, LfsBlobStoreFactory lfsBlobStoreFactory) { this.handler = handler; this.storeProvider = storeProvider; + this.lfsBlobStoreFactory = lfsBlobStoreFactory; } @Override @@ -61,7 +64,7 @@ public class GitRepositoryServiceResolver implements RepositoryServiceResolver { GitRepositoryServiceProvider provider = null; if (GitRepositoryHandler.TYPE_NAME.equalsIgnoreCase(repository.getType())) { - provider = new GitRepositoryServiceProvider(handler, repository, storeProvider); + provider = new GitRepositoryServiceProvider(handler, repository, storeProvider, lfsBlobStoreFactory); } return provider; diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java index 1feceba652..4b854f6209 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitBrowseCommandTest.java @@ -171,6 +171,6 @@ public class GitBrowseCommandTest extends AbstractGitCommandTestBase { } private GitBrowseCommand createCommand() { - return new GitBrowseCommand(createContext(), repository); + return new GitBrowseCommand(createContext(), repository, null); } } diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java index 0418bc3e61..eea8bc0017 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitCatCommandTest.java @@ -39,12 +39,18 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import sonia.scm.NotFoundException; import sonia.scm.repository.GitRepositoryConfig; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.web.lfs.LfsBlobStoreFactory; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Unit tests for {@link GitCatCommand}. @@ -136,7 +142,7 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase { CatCommandRequest request = new CatCommandRequest(); request.setPath("b.txt"); - InputStream catResultStream = new GitCatCommand(createContext(), repository).getCatResultStream(request); + InputStream catResultStream = new GitCatCommand(createContext(), repository, null).getCatResultStream(request); assertEquals('b', catResultStream.read()); assertEquals('\n', catResultStream.read()); @@ -145,13 +151,38 @@ public class GitCatCommandTest extends AbstractGitCommandTestBase { catResultStream.close(); } + @Test + public void testLfsStream() throws IOException { + LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + BlobStore blobStore = mock(BlobStore.class); + Blob blob = mock(Blob.class); + when(lfsBlobStoreFactory.getLfsBlobStore(repository)).thenReturn(blobStore); + when(blobStore.get("d2252bd9fde1bb2ae7531b432c48262c3cbe4df4376008986980de40a7c9cf8b")) + .thenReturn(blob); + when(blob.getInputStream()).thenReturn(new ByteArrayInputStream(new byte[]{'i', 's'})); + + CatCommandRequest request = new CatCommandRequest(); + request.setRevision("lfs-test"); + request.setPath("lfs-image.png"); + + InputStream catResultStream = new GitCatCommand(createContext(), repository, lfsBlobStoreFactory) + .getCatResultStream(request); + + assertEquals('i', catResultStream.read()); + assertEquals('s', catResultStream.read()); + + assertEquals(-1, catResultStream.read()); + + catResultStream.close(); + } + private String execute(CatCommandRequest request) throws IOException { String content = null; ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { - new GitCatCommand(createContext(), repository).getCatResult(request, + new GitCatCommand(createContext(), repository, null).getCatResult(request, baos); } finally diff --git a/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-test.zip b/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-test.zip index 8f689e9664..addda86090 100644 Binary files a/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-test.zip and b/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-test.zip differ