improve archive download

src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala

@@ -1,8 +1,8 @@
 package gitbucket.core.controller
 
 import java.io.File
-import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
 
+import javax.servlet.http.{HttpServletRequest, HttpServletResponse}
 import gitbucket.core.plugin.PluginRegistry
 import gitbucket.core.repo.html
 import gitbucket.core.helper
@@ -17,6 +17,13 @@ import gitbucket.core.model.{Account, CommitState, CommitStatus, WebHook}
 import gitbucket.core.service.WebHookService._
 import gitbucket.core.view
 import gitbucket.core.view.helpers
+import org.apache.commons.compress.archivers.{ArchiveEntry, ArchiveOutputStream}
+import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream}
+import org.apache.commons.compress.archivers.zip.{ZipArchiveEntry, ZipArchiveOutputStream}
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream
+import org.apache.commons.compress.utils.IOUtils
 import org.scalatra.forms._
 import org.apache.commons.io.FileUtils
 import org.eclipse.jgit.api.{ArchiveCommand, Git}
@@ -24,7 +31,10 @@ import org.eclipse.jgit.archive.{TgzFormat, ZipFormat}
 import org.eclipse.jgit.dircache.{DirCache, DirCacheBuilder}
 import org.eclipse.jgit.errors.MissingObjectException
 import org.eclipse.jgit.lib._
+import org.eclipse.jgit.revwalk.RevWalk
 import org.eclipse.jgit.transport.{ReceiveCommand, ReceivePack}
+import org.eclipse.jgit.treewalk.TreeWalk
+import org.eclipse.jgit.treewalk.filter.PathFilter
 import org.json4s.jackson.Serialization
 import org.scalatra._
 import org.scalatra.i18n.Messages
@@ -813,16 +823,31 @@ trait RepositoryViewerControllerBase extends ControllerBase {
   })
 
   /**
-   * Download repository contents as an archive.
+   * Download repository contents as an archive as compatible URL.
    */
-  get("/:owner/:repository/archive/*")(referrersOnly { repository =>
-    multiParams("splat").head match {
-      case name if name.endsWith(".zip") =>
-        archiveRepository(name, ".zip", repository)
-      case name if name.endsWith(".tar.gz") =>
-        archiveRepository(name, ".tar.gz", repository)
-      case _ => BadRequest()
-    }
+  get("/:owner/:repository/archive/:branch.:suffix")(referrersOnly { repository =>
+    val branch = params("branch")
+    val suffix = params("suffix")
+    archiveRepository(branch, branch + "." + suffix, repository, "")
+  })
+
+  /**
+   * Download all repository contents as an archive.
+   */
+  get("/:owner/:repository/archive/:branch/:name")(referrersOnly { repository =>
+    val branch = params("branch")
+    val name = params("name")
+    archiveRepository(branch, name, repository, "")
+  })
+
+  /**
+   * Download repositories subtree contents as an archive.
+   */
+  get("/:owner/:repository/archive/:branch/*/:name")(referrersOnly { repository =>
+    val branch = params("branch")
+    val name = params("name")
+    val path = multiParams("splat").head
+    archiveRepository(branch, name, repository, path)
   })
 
   get("/:owner/:repository/network/members")(referrersOnly { repository =>
@@ -1110,26 +1135,88 @@ trait RepositoryViewerControllerBase extends ControllerBase {
     }
   }
 
-  private def archiveRepository(name: String, suffix: String, repository: RepositoryService.RepositoryInfo): Unit = {
-    val revision = name.stripSuffix(suffix)
-
+  def archiveRepository(
+    revision: String,
+    filename: String,
+    repository: RepositoryService.RepositoryInfo,
+    path: String
+  ): Unit = {
+    def archive(archiveFormat: String, archive: ArchiveOutputStream)(
+      entryCreator: (String, Long, Int) => ArchiveEntry
+    ): Unit = {
       using(Git.open(getRepositoryDir(repository.owner, repository.name))) { git =>
         val oid = git.getRepository.resolve(revision)
         val revCommit = JGitUtil.getRevCommitFromId(git, oid)
         val sha1 = oid.getName()
         val repositorySuffix = (if (sha1.startsWith(revision)) sha1 else revision).replace('/', '-')
-      val filename = repository.name + "-" + repositorySuffix + suffix
+        val pathSuffix = if (path.isEmpty) "" else '-' + path.replace('/', '-')
+        val baseName = repository.name + "-" + repositorySuffix + pathSuffix
+        val filename = baseName + archiveFormat
 
-      contentType = "application/octet-stream"
+        using(new RevWalk(git.getRepository)) { revWalk =>
+          using(new TreeWalk(git.getRepository)) { treeWalk =>
+            treeWalk.addTree(revCommit.getTree)
+            treeWalk.setRecursive(true)
+            if (!path.isEmpty) {
+              treeWalk.setFilter(PathFilter.create(path))
+            }
+            if (treeWalk != null) {
+              while (treeWalk.next()) {
+                val entryPath =
+                  if (path.isEmpty) baseName + "/" + treeWalk.getPathString
+                  else path.split("/").last + treeWalk.getPathString.substring(path.length)
+                val size = JGitUtil.getFileSize(git, repository, treeWalk)
+                val mode = treeWalk.getFileMode.getBits
+                val entry: ArchiveEntry = entryCreator(entryPath, size, mode)
+                JGitUtil.openFile(git, repository, revCommit.getTree, treeWalk.getPathString) { in =>
+                  archive.putArchiveEntry(entry)
+                  IOUtils.copy(in, archive)
+                  archive.closeArchiveEntry()
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    val tarRe = """\.tar\.(gz|bz2|xz)$""".r
+
+    filename match {
+      case name if name.endsWith(".zip") =>
         response.setHeader("Content-Disposition", s"attachment; filename=${filename}")
+        contentType = "application/octet-stream"
         response.setBufferSize(1024 * 1024);
-
-      git.archive
-        .setFormat(suffix.tail)
-        .setPrefix(repository.name + "-" + repositorySuffix + "/")
-        .setTree(revCommit)
-        .setOutputStream(response.getOutputStream)
-        .call()
+        using(new ZipArchiveOutputStream(response.getOutputStream)) { zip =>
+          archive(".zip", zip) { (path, size, mode) =>
+            val entry = new ZipArchiveEntry(path)
+            entry.setSize(size)
+            entry.setUnixMode(mode)
+            entry
+          }
+        }
+      case tarRe(compressor) =>
+        response.setHeader("Content-Disposition", s"attachment; filename=${filename}")
+        contentType = "application/octet-stream"
+        response.setBufferSize(1024 * 1024)
+        using(compressor match {
+          case "gz"  => new GzipCompressorOutputStream(response.getOutputStream)
+          case "bz2" => new BZip2CompressorOutputStream(response.getOutputStream)
+          case "xz"  => new XZCompressorOutputStream(response.getOutputStream)
+        }) { compressorOutputStream =>
+          using(new TarArchiveOutputStream(compressorOutputStream)) { tar =>
+            tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR)
+            tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU)
+            tar.setAddPaxHeadersForNonAsciiNames(true)
+            archive(".tar.gz", tar) { (path, size, mode) =>
+              val entry = new TarArchiveEntry(path)
+              entry.setSize(size)
+              entry.setMode(mode)
+              entry
+            }
+          }
+        }
+      case _ =>
+        BadRequest()
     }
   }
 

src/main/scala/gitbucket/core/util/JGitUtil.scala

@@ -1,6 +1,6 @@
 package gitbucket.core.util
 
-import java.io.ByteArrayOutputStream
+import java.io.{ByteArrayOutputStream, File, FileInputStream, InputStream}
 
 import gitbucket.core.service.RepositoryService
 import org.eclipse.jgit.api.Git
@@ -1220,4 +1220,60 @@ object JGitUtil {
       Option(git.getRepository.resolve(revstr)).map(ObjectId.toString(_))
     }
   }
+
+  def getFileSize(git: Git, repository: RepositoryService.RepositoryInfo, treeWalk: TreeWalk): Long = {
+    val attrs = treeWalk.getAttributes
+    val loader = git.getRepository.open(treeWalk.getObjectId(0))
+    if (attrs.containsKey("filter") && attrs.get("filter").getValue == "lfs") {
+      val lfsAttrs = getLfsAttributes(loader)
+      lfsAttrs.get("size").map(_.toLong).get
+    } else {
+      loader.getSize
+    }
+  }
+
+  def getFileSize(git: Git, repository: RepositoryService.RepositoryInfo, tree: RevTree, path: String): Long = {
+    using(TreeWalk.forPath(git.getRepository, path, tree)) { treeWalk =>
+      getFileSize(git, repository, treeWalk)
+    }
+  }
+
+  def openFile[T](git: Git, repository: RepositoryService.RepositoryInfo, treeWalk: TreeWalk)(
+    f: InputStream => T
+  ): T = {
+    val attrs = treeWalk.getAttributes
+    val loader = git.getRepository.open(treeWalk.getObjectId(0))
+    if (attrs.containsKey("filter") && attrs.get("filter").getValue == "lfs") {
+      val lfsAttrs = getLfsAttributes(loader)
+      if (lfsAttrs.nonEmpty) {
+        val oid = lfsAttrs("oid").split(":")(1)
+
+        val file = new File(FileUtil.getLfsFilePath(repository.owner, repository.name, oid))
+        using(new FileInputStream(FileUtil.getLfsFilePath(repository.owner, repository.name, oid))) { in =>
+          f(in)
+        }
+      } else {
+        throw new RuntimeException
+      }
+    } else {
+      using(loader.openStream()) { in =>
+        f(in)
+      }
+    }
+  }
+
+  def openFile[T](git: Git, repository: RepositoryService.RepositoryInfo, tree: RevTree, path: String)(
+    f: InputStream => T
+  ): T = {
+    using(TreeWalk.forPath(git.getRepository, path, tree)) { treeWalk =>
+      openFile(git, repository, treeWalk)(f)
+    }
+  }
+
+  private def getLfsAttributes(loader: ObjectLoader): Map[String, String] = {
+    val bytes = loader.getCachedBytes
+    val text = new String(bytes, "UTF-8")
+
+    JGitUtil.getLfsObjects(text)
+  }
 }

src/main/twirl/gitbucket/core/repo/files.scala.html

@@ -33,6 +33,7 @@
     <div class="head" style="height: 24px;">
       <div class="pull-right">
         <div class="btn-group">
+          <a href="@{helpers.url(repository)}/archive/@{helpers.encodeRefName(branch)}@if(pathList.length > 0){/@pathList.mkString("/")}/@{helpers.encodeRefName(branch)}.zip" class="btn btn-sm btn-default"><i class="octicon octicon-cloud-download"></i> Download ZIP</a>
           <a href="@helpers.url(repository)/find/@helpers.encodeRefName(branch)" class="btn btn-sm btn-default" data-hotkey="t"><i class="octicon octicon-search"></i></a>
           <a href="@helpers.url(repository)/commits/@helpers.encodeRefName((branch :: pathList).mkString("/"))" class="btn btn-sm btn-default"><i class="octicon octicon-history"></i> @if(commitCount > 10000){10000+} else {@commitCount} @helpers.plural(commitCount, "commit")</a>
         </div>
@@ -44,7 +45,6 @@
             @if(context.platform != "linux" && context.platform != null){
               <a href="@RepositoryService.openRepoUrl(repository.httpUrl)" id="repository-clone-url" class="btn btn-sm btn-default"><i class="octicon octicon-desktop-download"></i></a>
             }
-            <a href="@{helpers.url(repository)}/archive/@{helpers.encodeRefName(branch)}.zip" class="btn btn-sm btn-default"><i class="octicon octicon-cloud-download"></i> Download ZIP</a>
           </div>
         </div>
         <div class="pull-right pc">