diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java index 0f7c9296a7..19f18f0625 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitModifyCommand.java @@ -3,6 +3,7 @@ package sonia.scm.repository.spi; import org.apache.commons.lang.StringUtils; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.attributes.FilterCommandRegistry; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevCommit; @@ -12,6 +13,7 @@ import sonia.scm.ContextEntry; import sonia.scm.repository.GitWorkdirFactory; import sonia.scm.repository.InternalRepositoryException; import sonia.scm.repository.Repository; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.File; import java.io.IOException; @@ -30,10 +32,12 @@ import static sonia.scm.ScmConstraintViolationException.Builder.doThrow; public class GitModifyCommand extends AbstractGitCommand implements ModifyCommand { private final GitWorkdirFactory workdirFactory; + private final LfsBlobStoreFactory lfsBlobStoreFactory; - GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory) { + GitModifyCommand(GitContext context, Repository repository, GitWorkdirFactory workdirFactory, LfsBlobStoreFactory lfsBlobStoreFactory) { super(context, repository); this.workdirFactory = workdirFactory; + this.lfsBlobStoreFactory = lfsBlobStoreFactory; } @Override @@ -87,10 +91,17 @@ public class GitModifyCommand extends AbstractGitCommand implements ModifyComman throw alreadyExists(createFileContext(toBeCreated)); } } + + LfsBlobStoreCleanFilterFactory cleanFilterFactory = new LfsBlobStoreCleanFilterFactory(lfsBlobStoreFactory, repository, targetFile); + + String registerKey = "git-lfs clean -- '" + toBeCreated + "'"; + FilterCommandRegistry.register(registerKey, cleanFilterFactory::createFilter); try { addFileToGit(toBeCreated); } catch (GitAPIException e) { throwInternalRepositoryException("could not add new file to index", e); + } finally { + FilterCommandRegistry.unregister(registerKey); } } diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java index 4c02a3a73c..dc43b8d9b7 100644 --- a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/GitRepositoryServiceProvider.java @@ -273,7 +273,7 @@ public class GitRepositoryServiceProvider extends RepositoryServiceProvider @Override public ModifyCommand getModifyCommand() { - return new GitModifyCommand(context, repository, handler.getWorkdirFactory()); + return new GitModifyCommand(context, repository, handler.getWorkdirFactory(), lfsBlobStoreFactory); } @Override diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java new file mode 100644 index 0000000000..d47673c970 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilter.java @@ -0,0 +1,101 @@ +package sonia.scm.repository.spi; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.lfs.Lfs; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lib.Repository; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import static org.eclipse.jgit.lfs.lib.Constants.LONG_HASH_FUNCTION; + +/** + * Adapted version of JGit's {@link org.eclipse.jgit.lfs.CleanFilter} to write the + * lfs file directly to the lfs blob store. + */ +public class LfsBlobStoreCleanFilter extends FilterCommand { + + + private Lfs lfsUtil; + private final BlobStore lfsBlobStore; + private final Path targetFile; + private final DigestOutputStream digestOutputStream; + + private long size; + + public LfsBlobStoreCleanFilter(Repository db, InputStream in, OutputStream out, BlobStore lfsBlobStore, Path targetFile) + throws IOException { + super(in, out); + lfsUtil = new Lfs(db); + this.lfsBlobStore = lfsBlobStore; + this.targetFile = targetFile; + Files.createDirectories(lfsUtil.getLfsTmpDir()); + + + MessageDigest md ; + try { + md = MessageDigest.getInstance(LONG_HASH_FUNCTION); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + digestOutputStream = new DigestOutputStream(new OutputStream() { + @Override + public void write(int b) { + } + }, md); + + } + + @Override + public int run() throws IOException { + try { + byte[] buf = new byte[8192]; + int length = in.read(buf); + if (length != -1) { + digestOutputStream.write(buf, 0, length); + size += length; + return length; + } else { + digestOutputStream.close(); + AnyLongObjectId loid = LongObjectId.fromRaw(digestOutputStream.getMessageDigest().digest()); + + Blob existingBlob = lfsBlobStore.get(loid.getName()); + if (existingBlob != null) { + long blobSize = existingBlob.getSize(); + if (blobSize != size) { + throw new RuntimeException("lfs entry already exists for loid " + loid.getName() + " but has wrong size"); + } + } else { + Blob newBlob = lfsBlobStore.create(loid.getName()); + OutputStream outputStream = newBlob.getOutputStream(); + Files.copy(targetFile, outputStream); + newBlob.commit(); + } + + LfsPointer lfsPointer = new LfsPointer(loid, size); + lfsPointer.encode(out); + in.close(); + out.close(); + return -1; + } + } catch (IOException e) { + if (digestOutputStream != null) { + digestOutputStream.close(); + } + in.close(); + out.close(); + throw e; + } + } +} diff --git a/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java new file mode 100644 index 0000000000..6951ec1b14 --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/main/java/sonia/scm/repository/spi/LfsBlobStoreCleanFilterFactory.java @@ -0,0 +1,26 @@ +package sonia.scm.repository.spi; + +import org.eclipse.jgit.lib.Repository; +import sonia.scm.web.lfs.LfsBlobStoreFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; + +public class LfsBlobStoreCleanFilterFactory { + + private final LfsBlobStoreFactory blobStoreFactory; + private final sonia.scm.repository.Repository repository; + private final Path targetFile; + + public LfsBlobStoreCleanFilterFactory(LfsBlobStoreFactory blobStoreFactory, sonia.scm.repository.Repository repository, Path targetFile) { + this.blobStoreFactory = blobStoreFactory; + this.repository = repository; + this.targetFile = targetFile; + } + + LfsBlobStoreCleanFilter createFilter(Repository db, InputStream in, OutputStream out) throws IOException { + return new LfsBlobStoreCleanFilter(db, in, out, blobStoreFactory.getLfsBlobStore(repository), targetFile); + } +} diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java index 4fbf7eaf02..e26458a4ec 100644 --- a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommandTest.java @@ -20,12 +20,14 @@ import sonia.scm.NotFoundException; import sonia.scm.ScmConstraintViolationException; import sonia.scm.repository.Person; import sonia.scm.repository.util.WorkdirProvider; +import sonia.scm.web.lfs.LfsBlobStoreFactory; import java.io.File; import java.io.IOException; import java.nio.file.Files; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; @SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret") public class GitModifyCommandTest extends AbstractGitCommandTestBase { @@ -37,6 +39,8 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase { @Rule public ShiroRule shiro = new ShiroRule(); + private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + @Test public void shouldCreateCommit() throws IOException, GitAPIException { File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile(); @@ -296,7 +300,7 @@ public class GitModifyCommandTest extends AbstractGitCommandTestBase { } private GitModifyCommand createCommand() { - return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider())); + return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory); } @FunctionalInterface diff --git a/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java new file mode 100644 index 0000000000..9414497bcb --- /dev/null +++ b/scm-plugins/scm-git-plugin/src/test/java/sonia/scm/repository/spi/GitModifyCommand_LFSTest.java @@ -0,0 +1,83 @@ +package sonia.scm.repository.spi; + +import com.github.sdorra.shiro.ShiroRule; +import com.github.sdorra.shiro.SubjectAware; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import sonia.scm.repository.Person; +import sonia.scm.repository.util.WorkdirProvider; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.web.lfs.LfsBlobStoreFactory; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@SubjectAware(configuration = "classpath:sonia/scm/configuration/shiro.ini", username = "admin", password = "secret") +public class GitModifyCommand_LFSTest extends AbstractGitCommandTestBase { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public BindTransportProtocolRule transportProtocolRule = new BindTransportProtocolRule(); + @Rule + public ShiroRule shiro = new ShiroRule(); + + private final LfsBlobStoreFactory lfsBlobStoreFactory = mock(LfsBlobStoreFactory.class); + + @Test + public void shouldCreateCommit() throws IOException, GitAPIException { + BlobStore blobStore = mock(BlobStore.class); + Blob blob = mock(Blob.class); + when(lfsBlobStoreFactory.getLfsBlobStore(any())).thenReturn(blobStore); + when(blobStore.create("fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601")).thenReturn(blob); + when(blobStore.get("fe32608c9ef5b6cf7e3f946480253ff76f24f4ec0678f3d0f07f9844cbff9601")).thenReturn(null, blob); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + when(blob.getOutputStream()).thenReturn(outputStream); + when(blob.getSize()).thenReturn((long) "new content".length()); + + File newFile = Files.write(temporaryFolder.newFile().toPath(), "new content".getBytes()).toFile(); + + GitModifyCommand command = createCommand(); + + ModifyCommandRequest request = new ModifyCommandRequest(); + request.setCommitMessage("test commit"); + request.addRequest(new ModifyCommandRequest.CreateFileRequest("new_lfs.png", newFile, false)); + request.setAuthor(new Person("Dirk Gently", "dirk@holistic.det")); + + String newRef = command.execute(request); + + try (Git git = new Git(createContext().open())) { + RevCommit lastCommit = getLastCommit(git); + assertThat(lastCommit.getFullMessage()).isEqualTo("test commit"); + assertThat(lastCommit.getAuthorIdent().getName()).isEqualTo("Dirk Gently"); + assertThat(newRef).isEqualTo(lastCommit.toObjectId().name()); + } + + assertThat(outputStream.toString()).isEqualTo("new content"); + } + + private RevCommit getLastCommit(Git git) throws GitAPIException { + return git.log().setMaxCount(1).call().iterator().next(); + } + + private GitModifyCommand createCommand() { + return new GitModifyCommand(createContext(), repository, new SimpleGitWorkdirFactory(new WorkdirProvider()), lfsBlobStoreFactory); + } + + @Override + protected String getZippedRepositoryResource() { + return "sonia/scm/repository/spi/scm-git-spi-lfs-test.zip"; + } +}