diff --git a/pom.xml b/pom.xml index 3ab3cb82a3..c14ce8b39b 100644 --- a/pom.xml +++ b/pom.xml @@ -843,7 +843,7 @@ 1.4.0 - v5.4.0.201906121030-r-scm1 + v5.4.0.201906121030-r-scm2 1.9.0-scm3 diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitLfsFilterContextListener.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitLfsFilterContextListener.java new file mode 100644 index 0000000000..101c242d65 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitLfsFilterContextListener.java @@ -0,0 +1,77 @@ +package sonia.scm.repository.spi; + +import com.google.common.io.ByteStreams; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.FS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.SCMContextProvider; +import sonia.scm.plugin.Extension; + +import javax.inject.Inject; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Pattern; + +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; + +@Extension +public class GitLfsFilterContextListener implements ServletContextListener { + + public static final String GITCONFIG = "[filter \"lfs\"]\n" + + "clean = git-lfs clean -- %f\n" + + "smudge = git-lfs smudge -- %f\n" + + "process = git-lfs filter-process\n" + + "required = true\n"; + public static final Pattern COMMAND_NAME_PATTERN = Pattern.compile("git-lfs (smudge|clean) -- .*"); + + private static final Logger LOG = LoggerFactory.getLogger(GitLfsFilterContextListener.class); + + private final SCMContextProvider contextProvider; + + @Inject + public GitLfsFilterContextListener(SCMContextProvider contextProvider) { + this.contextProvider = contextProvider; + } + + @Override + public void contextInitialized(ServletContextEvent sce) { + Path gitconfig = contextProvider.getBaseDirectory().toPath().resolve("gitconfig"); + try { + Files.write(gitconfig, GITCONFIG.getBytes(Charset.defaultCharset()), TRUNCATE_EXISTING, CREATE); + FS.DETECTED.setGitSystemConfig(gitconfig.toFile()); + LOG.info("wrote git config file: {}", gitconfig); + } catch (IOException e) { + LOG.error("could not write git config in path {}; git lfs support may not work correctly", gitconfig, e); + } + FilterCommandRegistry.register(COMMAND_NAME_PATTERN, NoOpFilterCommand::new); + } + + @Override + public void contextDestroyed(ServletContextEvent sce) { + FilterCommandRegistry.unregister(COMMAND_NAME_PATTERN); + } + + private static class NoOpFilterCommand extends FilterCommand { + NoOpFilterCommand(Repository db, InputStream in, OutputStream out) { + super(in, out); + } + + @Override + public int run() throws IOException { + ByteStreams.copy(in, out); + in.close(); + out.close(); + return -1; + } + } +} diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java index da7f0150f5..a68be0a4da 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java @@ -1,27 +1,38 @@ package sonia.scm.repository.spi; +import com.google.common.util.concurrent.Striped; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.revwalk.RevCommit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import sonia.scm.ConcurrentModificationException; import sonia.scm.NoChangesMadeException; import sonia.scm.repository.GitWorkdirFactory; import sonia.scm.repository.InternalRepositoryException; import sonia.scm.repository.Repository; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.Optional; +import java.util.concurrent.locks.Lock; public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand { - private final GitWorkdirFactory workdirFactory; + private static final Logger LOG = LoggerFactory.getLogger(GitModifyCommand.class); + private static final Striped REGISTER_LOCKS = Striped.lock(5); - GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) { + private final GitWorkdirFactory workdirFactory; + private final LfsBlobStoreFactory lfsBlobStoreFactory; + + GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) { super(context, repository); this.workdirFactory = workdirFactory; + this.lfsBlobStoreFactory = lfsBlobStoreFactory; } @Override @@ -43,10 +54,9 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman @Override String run() throws IOException { getClone().getRepository().getFullBranch(); - if (!StringUtils.isEmpty(request.getExpectedRevision())) { - if (!request.getExpectedRevision().equals(getCurrentRevision().getName())) { - throw new ConcurrentModificationException("branch", request.getBranch() == null? "default": request.getBranch()); - } + if (!StringUtils.isEmpty(request.getExpectedRevision()) + && !request.getExpectedRevision().equals(getCurrentRevision().getName())) { + throw new ConcurrentModificationException("branch", request.getBranch() == null ? "default" : request.getBranch()); } for (ModifyCommandRequest.PartialRequest r : request.getRequests()) { r.execute(this); @@ -59,10 +69,27 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman @Override public void addFileToScm(String name, Path file) { + addToGitWithLfsSupport(name, file); + } + + private void addToGitWithLfsSupport(String path, Path targetFile) { + REGISTER_LOCKS.get(targetFile).lock(); try { - addFileToGit(name); - } catch (GitAPIException e) { - throwInternalRepositoryException("could not add new file to index", e); + LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile); + + String registerKey = "git-lfs clean -- '" + path + "'"; + LOG.debug("register lfs filter command factory for command '{}'", registerKey); + FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter); + try { + addFileToGit(path); + } catch (GitAPIException e) { + throwInternalRepositoryException("could not add file to index", e); + } finally { + LOG.debug("unregister lfs filter command factory for command \"{}\"", registerKey); + FilterCommandRegistry.unregister(registerKey); + } + } finally { + REGISTER_LOCKS.get(targetFile).unlock(); } } @@ -100,9 +127,9 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman } return path; } - } - private String throwInternalRepositoryException(String message, Exception e) { - throw new InternalRepositoryException(context.getRepository(), message, e); + private String throwInternalRepositoryException(String message, Exception e) { + throw new InternalRepositoryException(context.getRepository(), message, e); + } } } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java index 4c02a3a73c..dc43b8d9b7 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java @@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider @Override public ModifyCommand getModifyCommand() { - return new GitModifyCommand(context, repository, handler.getWorkdirFactory()); + return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory); } @Override diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java new file mode 100644 index 0000000000..bb54ea162a --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java @@ -0,0 +1,83 @@ +package sonia.scm.repository.spi; + +import com.google.common.io.ByteStreams; +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.util.IOUtil; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestOutputStream; + +/** + * Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the + * lfs file directly to the lfs blob store. + */ +class LfsBlobStoreCleanFilter extends FilterCommand { + + private static final Logger LOG = LoggerFactory.getLogger(LfsBlobStoreCleanFilter.class); + + private final BlobStore lfsBlobStore; + private final Path targetFile; + + LfsBlobStoreCleanFilter(InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile) { + super(in, out); + this.lfsBlobStore = lfsBlobStore; + this.targetFile = targetFile; + } + + @Override + // Suppress warning for RuntimeException after check for wrong size, because mathematicians say this will never happen + @SuppressWarnings("squid:S00112") + public int run() throws IOException { + LOG.debug("running scm lfs filter for file {}", targetFile); + DigestOutputStream digestOutputStream = createDigestStream(); + try { + long size = ByteStreams.copy(in, digestOutputStream); + AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest()); + String hash = loid.getName(); + + Blob existingBlob = lfsBlobStore.get(hash); + if (existingBlob != null) { + LOG.debug("found existing lfs blob for oid {}", hash); + long blobSize = existingBlob.getSize(); + if (blobSize != size) { + throw new RuntimeException("lfs entry already exists for loid " + hash + " but has wrong size"); + } + } else { + LOG.debug("uploading new lfs blob for oid {}", hash); + Blob newBlob = lfsBlobStore.create(hash); + OutputStream outputStream = newBlob.getOutputStream(); + Files.copy(targetFile, outputStream); + newBlob.commit(); + } + + LfsPointer lfsPointer = new LfsPointer(loid, size); + lfsPointer.encode(out); + return -1; + } finally { + IOUtil.close(digestOutputStream); + IOUtil.close(in); + IOUtil.close(out); + } + } + + private DigestOutputStream createDigestStream() { + return new DigestOutputStream(new OutputStream() { + @Override + public void write(int b) { + // no further target here, we are just interested in the digest + } + }, Constants.newMessageDigest()); + } +} diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java new file mode 100644 index 0000000000..d6de8e83df --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java @@ -0,0 +1,27 @@ +package sonia.scm.repository.spi; + +import org.eclipse.jgit.lib.Repository; +import sonia.scm.web.lfs.LfsBlobStoreFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; + +class LfsBlobStoreCleanFilterFactory { + + private final LfsBlobStoreFactory blobStoreFactory; + private final sonia.scm.repository.Repository repository; + private final Path targetFile; + + LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) { + this.blobStoreFactory = blobStoreFactory; + this.repository = repository; + this.targetFile = targetFile; + } + + @SuppressWarnings("squid:S1172") // suppress unused parameter to keep the api compatible to jgit's FilterCommandFactory + LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) { + return new LfsBlobStoreCleanFilter(in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile); + } +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java index 501134026d..fd007d0f48 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java @@ -20,12 +20,14 @@ import sonia.scm.NotFoundException; import sonia.scm.ScmConstraintViolationException; import sonia.scm.repository.Person; import sonia.scm.repository.util.WorkdirProvider; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret") public class GitModifyCommandTest extends AbstractGitCommandTestBase { @@ -37,6 +39,8 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase { @Rule public ShiroRule shiro = new ShiroRule(); + private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + @Test public void shouldCreateCommit() throws IOException, GitAPIException { File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile(); @@ -296,7 +300,7 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase { } private GitModifyCommand createCommand() { - return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider())); + return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory); } @FunctionalInterface diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java new file mode 100644 index 0000000000..b1a5c7bbcc --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java @@ -0,0 +1,116 @@ +package sonia.scm.repository.spi; + +import com.github.sdorra.shiro.ShiroRule; +import com.github.sdorra.shiro.SubjectAware; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import sonia.scm.repository.Person; +import sonia.scm.repository.util.WorkdirProvider; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.web.lfs.LfsBlobStoreFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret") +public class GitModifyCommand_LFSTest extends AbstractGitCommandTestBase { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public BindTransportProtocolRule transportProtocolRule = new BindTransportProtocolRule(); + @Rule + public ShiroRule shiro = new ShiroRule(); + + private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + + @Before + public void registerFilter() { + new GitLfsFilterContextListener(contextProvider).contextInitialized(null); + } + + @After + public void unregisterFilter() { + new GitLfsFilterContextListener(contextProvider).contextDestroyed(null); + } + + @Test + public void shouldCreateCommit() throws IOException, GitAPIException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + String newRef = createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", outputStream); + + try (Git git = new Git(createContext().open())) { + RevCommit lastCommit = getLastCommit(git); + assertThat(lastCommit.getFullMessage()).isEqualTo("test commit"); + assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently"); + assertThat(newRef).isEqualTo(lastCommit.toObjectId().name()); + } + + assertThat(outputStream.toString()).isEqualTo("new content"); + } + + @Test + public void shouldCreateSecondCommits() throws IOException, GitAPIException { + createCommit("new_lfs.png", "new content", "fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601", new ByteArrayOutputStream()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + String newRef = createCommit("more_lfs.png", "more content", "2c2316737c9313956dfc0083da3a2a62ce259f66484f3e26440f0d1b02dd4128", outputStream); + + try (Git git = new Git(createContext().open())) { + RevCommit lastCommit = getLastCommit(git); + assertThat(lastCommit.getFullMessage()).isEqualTo("test commit"); + assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently"); + assertThat(newRef).isEqualTo(lastCommit.toObjectId().name()); + } + + assertThat(outputStream.toString()).isEqualTo("more content"); + } + + private String createCommit(String fileName, String content, String hashOfContent, ByteArrayOutputStream outputStream) throws IOException { + BlobStore blobStore = mock(BlobStore.class); + Blob blob = mock(Blob.class); + when(lfsBlobStoreFactory.getLfsBlobStore(any())).thenReturn(blobStore); + when(blobStore.create(hashOfContent)).thenReturn(blob); + when(blobStore.get(hashOfContent)).thenReturn(null, blob); + when(blob.getOutputStream()).thenReturn(outputStream); + when(blob.getSize()).thenReturn((long) content.length()); + + File newFile = Files.write(temporaryFolder.newFile().toPath(), content.getBytes()).toFile(); + + GitModifyCommand command = createCommand(); + + ModifyCommandRequest request = new ModifyCommandRequest(); + request.setCommitMessage("test commit"); + request.addRequest(new ModifyCommandRequest.CreateFileRequest(fileName, newFile, false)); + request.setAuthor(new Person("Dirk Gently", "dirk@holistic.det")); + + return command.execute(request); + } + + private RevCommit getLastCommit(Git git) throws GitAPIException { + return git.log().setMaxCount(1).call().iterator().next(); + } + + private GitModifyCommand createCommand() { + return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory); + } + + @Override + protected String getZippedRepositoryResource() { + return "sonia/scm/repository/spi/scm-git-spi-lfs-test.zip"; + } +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_withEmptyRepositoryTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_withEmptyRepositoryTest.java index ff409e470d..89362d65cd 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_withEmptyRepositoryTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_withEmptyRepositoryTest.java @@ -16,12 +16,14 @@ import org.junit.rules.TemporaryFolder; import sonia.scm.ScmConstraintViolationException; import sonia.scm.repository.Person; import sonia.scm.repository.util.WorkdirProvider; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret") public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommandTestBase { @@ -33,6 +35,8 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand @Rule public ShiroRule shiro = new ShiroRule(); + private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + @Test public void shouldCreateNewFileInEmptyRepository() throws IOException, GitAPIException { File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile(); @@ -74,7 +78,7 @@ public class GitModifyCommand_withEmptyRepositoryTest extends AbstractGitCommand } private GitModifyCommand createCommand() { - return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider())); + return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory); } @FunctionalInterface diff --git a/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-lfs-test.zip b/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-lfs-test.zip new file mode 100644 index 0000000000..b97f519684 Binary files /dev/null and b/scm-plugins/scm-git-plugin/src/test/resources/sonia/scm/repository/spi/scm-git-spi-lfs-test.zip differ