Merged in feature/lfs_upload (pull request #332)

LFS upload

pom.xml

@@ -843,7 +843,7 @@
     <shiro.version>1.4.0</shiro.version>
 
     <!-- repository libraries -->
-    <jgit.version>v5.4.0.201906121030-r-scm1</jgit.version>
+    <jgit.version>v5.4.0.201906121030-r-scm2</jgit.version>
     <svnkit.version>1.9.0-scm3</svnkit.version>
 
     <!-- util libraries -->

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitLfsFilterContextListener.java

@@ -0,0 +1,77 @@
+package sonia.scm.repository.spi;
+
+import com.google.common.io.ByteStreams;
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.util.FS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sonia.scm.SCMContextProvider;
+import sonia.scm.plugin.Extension;
+
+import javax.inject.Inject;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+
+@Extension
+public class GitLfsFilterContextListener implements ServletContextListener {
+
+  public static final String GITCONFIG = "[filter \"lfs\"]\n" +
+        "clean = git-lfs clean -- %f\n" +
+        "smudge = git-lfs smudge -- %f\n" +
+        "process = git-lfs filter-process\n" +
+        "required = true\n";
+  public static final Pattern COMMAND_NAME_PATTERN = Pattern.compile("git-lfs (smudge|clean) -- .*");
+
+  private static final Logger LOG = LoggerFactory.getLogger(GitLfsFilterContextListener.class);
+
+  private final SCMContextProvider contextProvider;
+
+  @Inject
+  public GitLfsFilterContextListener(SCMContextProvider contextProvider) {
+    this.contextProvider = contextProvider;
+  }
+
+  @Override
+  public void contextInitialized(ServletContextEvent sce) {
+    Path gitconfig = contextProvider.getBaseDirectory().toPath().resolve("gitconfig");
+    try {
+      Files.write(gitconfig, GITCONFIG.getBytes(Charset.defaultCharset()), TRUNCATE_EXISTING, CREATE);
+      FS.DETECTED.setGitSystemConfig(gitconfig.toFile());
+      LOG.info("wrote git config file: {}", gitconfig);
+    } catch (IOException e) {
+      LOG.error("could not write git config in path {}; git lfs support may not work correctly", gitconfig, e);
+    }
+    FilterCommandRegistry.register(COMMAND_NAME_PATTERN, NoOpFilterCommand::new);
+  }
+
+  @Override
+  public void contextDestroyed(ServletContextEvent sce) {
+    FilterCommandRegistry.unregister(COMMAND_NAME_PATTERN);
+  }
+
+  private static class NoOpFilterCommand extends FilterCommand {
+    NoOpFilterCommand(Repository db, InputStream in, OutputStream out) {
+      super(in, out);
+    }
+
+    @Override
+    public int run() throws IOException {
+      ByteStreams.copy(in, out);
+      in.close();
+      out.close();
+      return -1;
+    }
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java

@@ -1,27 +1,38 @@
 package sonia.scm.repository.spi;
 
+import com.google.common.util.concurrent.Striped;
 import org.apache.commons.lang.StringUtils;
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.attributes.FilterCommandRegistry;
 import org.eclipse.jgit.revwalk.RevCommit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import sonia.scm.ConcurrentModificationException;
 import sonia.scm.NoChangesMadeException;
 import sonia.scm.repository.GitWorkdirFactory;
 import sonia.scm.repository.InternalRepositoryException;
 import sonia.scm.repository.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Optional;
+import java.util.concurrent.locks.Lock;
 
 public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand {
 
-  private final GitWorkdirFactory workdirFactory;
+  private static final Logger LOG = LoggerFactory.getLogger(GitModifyCommand.class);
+  private static final Striped<Lock> REGISTER_LOCKS = Striped.lock(5);
 
-  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) {
+  private final GitWorkdirFactory workdirFactory;
+  private final LfsBlobStoreFactory lfsBlobStoreFactory;
+
+  GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) {
     super(context, repository);
     this.workdirFactory = workdirFactory;
+    this.lfsBlobStoreFactory = lfsBlobStoreFactory;
   }
 
   @Override
@@ -43,10 +54,9 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
     @Override
     String run() throws IOException {
       getClone().getRepository().getFullBranch();
-      if (!StringUtils.isEmpty(request.getExpectedRevision())) {
-        if (!request.getExpectedRevision().equals(getCurrentRevision().getName())) {
-          throw new ConcurrentModificationException("branch", request.getBranch() == null? "default": request.getBranch());
-        }
+      if (!StringUtils.isEmpty(request.getExpectedRevision())
+        && !request.getExpectedRevision().equals(getCurrentRevision().getName())) {
+        throw new ConcurrentModificationException("branch", request.getBranch() == null ? "default" : request.getBranch());
       }
       for (ModifyCommandRequest.PartialRequest r : request.getRequests()) {
         r.execute(this);
@@ -59,10 +69,27 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
 
     @Override
     public void addFileToScm(String name, Path file) {
+      addToGitWithLfsSupport(name, file);
+    }
+
+    private void addToGitWithLfsSupport(String path, Path targetFile) {
+      REGISTER_LOCKS.get(targetFile).lock();
       try {
-        addFileToGit(name);
-      } catch (GitAPIException e) {
-        throwInternalRepositoryException("could not add new file to index", e);
+        LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile);
+
+        String registerKey = "git-lfs clean -- '" + path + "'";
+        LOG.debug("register lfs filter command factory for command '{}'", registerKey);
+        FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter);
+        try {
+          addFileToGit(path);
+        } catch (GitAPIException e) {
+          throwInternalRepositoryException("could not add file to index", e);
+        } finally {
+          LOG.debug("unregister lfs filter command factory for command \"{}\"", registerKey);
+          FilterCommandRegistry.unregister(registerKey);
+        }
+      } finally {
+        REGISTER_LOCKS.get(targetFile).unlock();
       }
     }
 
@@ -100,9 +127,9 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman
       }
       return path;
     }
-  }
 
-  private String throwInternalRepositoryException(String message, Exception e) {
-    throw new InternalRepositoryException(context.getRepository(), message, e);
+    private String throwInternalRepositoryException(String message, Exception e) {
+      throw new InternalRepositoryException(context.getRepository(), message, e);
+    }
   }
 }

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java

@@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider
 
   @Override
   public ModifyCommand getModifyCommand() {
-    return new GitModifyCommand(context, repository, handler.getWorkdirFactory());
+    return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory);
   }
 
   @Override

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java

@@ -0,0 +1,83 @@
+package sonia.scm.repository.spi;
+
+import com.google.common.io.ByteStreams;
+import org.eclipse.jgit.attributes.FilterCommand;
+import org.eclipse.jgit.lfs.LfsPointer;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.util.IOUtil;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.DigestOutputStream;
+
+/**
+ * Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the
+ * lfs file directly to the lfs blob store.
+ */
+class LfsBlobStoreCleanFilter extends FilterCommand {
+
+  private static final Logger LOG = LoggerFactory.getLogger(LfsBlobStoreCleanFilter.class);
+
+  private final BlobStore lfsBlobStore;
+  private final Path targetFile;
+
+  LfsBlobStoreCleanFilter(InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile) {
+    super(in, out);
+    this.lfsBlobStore = lfsBlobStore;
+    this.targetFile = targetFile;
+  }
+
+  @Override
+  // Suppress warning for RuntimeException after check for wrong size, because mathematicians say this will never happen
+  @SuppressWarnings("squid:S00112")
+  public int run() throws IOException {
+    LOG.debug("running scm lfs filter for file {}", targetFile);
+    DigestOutputStream digestOutputStream = createDigestStream();
+    try {
+      long size = ByteStreams.copy(in, digestOutputStream);
+      AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest());
+      String hash = loid.getName();
+
+      Blob existingBlob = lfsBlobStore.get(hash);
+      if (existingBlob != null) {
+        LOG.debug("found existing lfs blob for oid {}", hash);
+        long blobSize = existingBlob.getSize();
+        if (blobSize != size) {
+          throw new RuntimeException("lfs entry already exists for loid " + hash + " but has wrong size");
+        }
+      } else {
+        LOG.debug("uploading new lfs blob for oid {}", hash);
+        Blob newBlob = lfsBlobStore.create(hash);
+        OutputStream outputStream = newBlob.getOutputStream();
+        Files.copy(targetFile, outputStream);
+        newBlob.commit();
+      }
+
+      LfsPointer lfsPointer = new LfsPointer(loid, size);
+      lfsPointer.encode(out);
+      return -1;
+    } finally {
+      IOUtil.close(digestOutputStream);
+      IOUtil.close(in);
+      IOUtil.close(out);
+    }
+  }
+
+  private DigestOutputStream createDigestStream() {
+    return new DigestOutputStream(new OutputStream() {
+      @Override
+      public void write(int b) {
+        // no further target here, we are just interested in the digest
+      }
+    }, Constants.newMessageDigest());
+  }
+}

scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java

@@ -0,0 +1,27 @@
+package sonia.scm.repository.spi;
+
+import org.eclipse.jgit.lib.Repository;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Path;
+
+class LfsBlobStoreCleanFilterFactory {
+
+  private final LfsBlobStoreFactory blobStoreFactory;
+  private final sonia.scm.repository.Repository repository;
+  private final Path targetFile;
+
+  LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) {
+    this.blobStoreFactory = blobStoreFactory;
+    this.repository = repository;
+    this.targetFile = targetFile;
+  }
+
+  @SuppressWarnings("squid:S1172") // suppress unused parameter to keep the api compatible to jgit's FilterCommandFactory
+  LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) {
+    return new LfsBlobStoreCleanFilter(in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile);
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java

@@ -20,12 +20,14 @@ import sonia.scm.NotFoundException;
 import sonia.scm.ScmConstraintViolationException;
 import sonia.scm.repository.Person;
 import sonia.scm.repository.util.WorkdirProvider;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
 
 @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
 public class GitModifyCommandTest extends AbstractGitCommandTestBase {
@@ -37,6 +39,8 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
   @Rule
   public ShiroRule shiro = new ShiroRule();
 
+  private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+
   @Test
   public void shouldCreateCommit() throws IOException, GitAPIException {
     File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
@@ -296,7 +300,7 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase {
   }
 
   private GitModifyCommand createCommand() {
-    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()));
+    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
   }
 
   @FunctionalInterface

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java

@@ -0,0 +1,116 @@
+package sonia.scm.repository.spi;
+
+import com.github.sdorra.shiro.ShiroRule;
+import com.github.sdorra.shiro.SubjectAware;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import sonia.scm.repository.Person;
+import sonia.scm.repository.util.WorkdirProvider;
+import sonia.scm.store.Blob;
+import sonia.scm.store.BlobStore;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
+public class GitModifyCommand_LFSTest extends AbstractGitCommandTestBase {
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+  @Rule
+  public BindTransportProtocolRule transportProtocolRule = new BindTransportProtocolRule();
+  @Rule
+  public ShiroRule shiro = new ShiroRule();
+
+  private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+
+  @Before
+  public void registerFilter() {
+    new GitLfsFilterContextListener(contextProvider).contextInitialized(null);
+  }
+
+  @After
+  public void unregisterFilter() {
+    new GitLfsFilterContextListener(contextProvider).contextDestroyed(null);
+  }
+
+  @Test
+  public void shouldCreateCommit() throws IOException, GitAPIException {
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    String newRef = createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", outputStream);
+
+    try (Git git = new Git(createContext().open())) {
+      RevCommit lastCommit = getLastCommit(git);
+      assertThat(lastCommit.getFullMessage()).isEqualTo("test commit");
+      assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently");
+      assertThat(newRef).isEqualTo(lastCommit.toObjectId().name());
+    }
+
+    assertThat(outputStream.toString()).isEqualTo("new content");
+  }
+
+  @Test
+  public void shouldCreateSecondCommits() throws IOException, GitAPIException {
+    createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", new ByteArrayOutputStream());
+
+    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+    String newRef = createCommit("more_lfs.png", "more content", "2c2316737c9313956dfc0083da3a2a62ce259f66484f3e26440f0d1b02dd4128", outputStream);
+
+    try (Git git = new Git(createContext().open())) {
+      RevCommit lastCommit = getLastCommit(git);
+      assertThat(lastCommit.getFullMessage()).isEqualTo("test commit");
+      assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently");
+      assertThat(newRef).isEqualTo(lastCommit.toObjectId().name());
+    }
+
+    assertThat(outputStream.toString()).isEqualTo("more content");
+  }
+
+  private String createCommit(String fileName, String content, String hashOfContent, ByteArrayOutputStream outputStream) throws IOException {
+    BlobStore blobStore = mock(BlobStore.class);
+    Blob blob = mock(Blob.class);
+    when(lfsBlobStoreFactory.getLfsBlobStore(any())).thenReturn(blobStore);
+    when(blobStore.create(hashOfContent)).thenReturn(blob);
+    when(blobStore.get(hashOfContent)).thenReturn(null, blob);
+    when(blob.getOutputStream()).thenReturn(outputStream);
+    when(blob.getSize()).thenReturn((long) content.length());
+
+    File newFile = Files.write(temporaryFolder.newFile().toPath(), content.getBytes()).toFile();
+
+    GitModifyCommand command = createCommand();
+
+    ModifyCommandRequest request = new ModifyCommandRequest();
+    request.setCommitMessage("test commit");
+    request.addRequest(new ModifyCommandRequest.CreateFileRequest(fileName, newFile, false));
+    request.setAuthor(new Person("Dirk Gently", "dirk@holistic.det"));
+
+    return command.execute(request);
+  }
+
+  private RevCommit getLastCommit(Git git) throws GitAPIException {
+    return git.log().setMaxCount(1).call().iterator().next();
+  }
+
+  private GitModifyCommand createCommand() {
+    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
+  }
+
+  @Override
+  protected String getZippedRepositoryResource() {
+    return "sonia/scm/repository/spi/scm-git-spi-lfs-test.zip";
+  }
+}

scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_withEmptyRepositoryTest.java

@@ -16,12 +16,14 @@ import org.junit.rules.TemporaryFolder;
 import sonia.scm.ScmConstraintViolationException;
 import sonia.scm.repository.Person;
 import sonia.scm.repository.util.WorkdirProvider;
+import sonia.scm.web.lfs.LfsBlobStoreFactory;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
 
 @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret")
 public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommandTestBase {
@@ -33,6 +35,8 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand
   @Rule
   public ShiroRule shiro = new ShiroRule();
 
+  private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class);
+
   @Test
   public void shouldCreateNewFileInEmptyRepository() throws IOException, GitAPIException {
     File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile();
@@ -74,7 +78,7 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand
   }
 
   private GitModifyCommand createCommand() {
-    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()));
+    return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory);
   }
 
   @FunctionalInterface