From e1802978d3c880ba79cdc76e2e935db5867c8b40 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 3 Jan 2017 03:09:50 +0900 Subject: [PATCH 01/13] (refs #1101)Experimental implementation of GitLFS Batch API --- .../core/servlet/GitLfsBatchServlet.scala | 97 +++++++++++++++++++ src/main/webapp/WEB-INF/web.xml | 11 +++ 2 files changed, 108 insertions(+) create mode 100644 src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala new file mode 100644 index 000000000..d59229909 --- /dev/null +++ b/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala @@ -0,0 +1,97 @@ +package gitbucket.core.servlet + +import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} + +import org.json4s._ +import org.json4s.jackson.Serialization.{read, write} + +import java.util.Date + +/** + * Provides GitLFS Batch API. + * + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + */ +class GitLfsBatchServlet extends HttpServlet { + + // TODO GitLFS server url must be configurable + private val GitLfsServerUrl = "http://localhost:9090/git-lfs" + + private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats + + override protected def doPost(req: HttpServletRequest, res: HttpServletResponse): Unit = { + val batchRequest = read[BatchRequest](req.getInputStream) + + val batchResponse = batchRequest.operation match { + case "upload" => + BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + BatchResponseObject( + requestObject.oid, + requestObject.size, + true, + Actions( + upload = Some(Action( + href = GitLfsServerUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000) + )) + ) + ) + }) + case "download" => + BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + BatchResponseObject( + requestObject.oid, + requestObject.size, + true, + Actions( + download = Some(Action( + href = GitLfsServerUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000) + )) + ) + ) + }) + } + + res.setContentType("application/vnd.git-lfs+json") + + val out = res.getWriter + out.print(write(batchResponse)) + out.flush() + } + +} + +case class BatchRequest( + operation: String, + transfers: Seq[String], + objects: Seq[BatchRequestObject] +) + +case class BatchRequestObject( + oid: String, + size: Long +) + +case class BatchUploadResponse( + transfer: String, + objects: Seq[BatchResponseObject] +) + +case class BatchResponseObject( + oid: String, + size: Long, + authenticated: Boolean, + actions: Actions +) + +case class Actions( + download: Option[Action] = None, + upload: Option[Action] = None +) + +case class Action( + href: String, + header: Map[String, String] = Map.empty, + expires_at: Date +) \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 9634066c2..066f99f52 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -46,6 +46,17 @@ /git/* + + GitLfsBatchServlet + gitbucket.core.servlet.GitLfsBatchServlet + true + + + + GitLfsBatchServlet + /git/root/git-lfs-test.git/info/lfs/* + + From c67441b6d4bd311d7ebf526082f7422cc1c391bc Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 3 Jan 2017 13:40:35 +0900 Subject: [PATCH 02/13] (refs #1101)Add GitLFS setting --- .../controller/SystemSettingsController.scala | 5 +- .../core/service/SystemSettingsService.scala | 13 ++- .../core/servlet/GitLfsBatchServlet.scala | 79 +++++++++---------- .../scala/gitbucket/core/util/Directory.scala | 4 +- .../gitbucket/core/admin/system.scala.html | 45 ++++++----- 5 files changed, 82 insertions(+), 64 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala index bfca8582d..96a33e78c 100644 --- a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala +++ b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala @@ -58,7 +58,10 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { "tls" -> trim(label("Enable TLS", optional(boolean()))), "ssl" -> trim(label("Enable SSL", optional(boolean()))), "keystore" -> trim(label("Keystore", optional(text()))) - )(Ldap.apply)) + )(Ldap.apply)), + "lfs" -> mapping( + "serverUrl" -> trim(label("LDAP host", optional(text()))) + )(Lfs.apply) )(SystemSettings.apply).verifying { settings => Vector( if(settings.ssh && settings.baseUrl.isEmpty){ diff --git a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala index c2aa28ded..00199a472 100644 --- a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala +++ b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala @@ -53,6 +53,7 @@ trait SystemSettingsService { ldap.keystore.foreach(x => props.setProperty(LdapKeystore, x)) } } + settings.lfs.serverUrl.foreach { x => props.setProperty(LfsServerUrl, x) } using(new java.io.FileOutputStream(GitBucketConf)){ out => props.store(out, null) } @@ -109,7 +110,10 @@ trait SystemSettingsService { getOptionValue(props, LdapKeystore, None))) } else { None - } + }, + Lfs( + getOptionValue(props, LfsServerUrl, None) + ) ) } } @@ -134,7 +138,8 @@ object SystemSettingsService { useSMTP: Boolean, smtp: Option[Smtp], ldapAuthentication: Boolean, - ldap: Option[Ldap]){ + ldap: Option[Ldap], + lfs: Lfs){ def baseUrl(request: HttpServletRequest): String = baseUrl.fold(request.baseUrl)(_.stripSuffix("/")) def sshAddress:Option[SshAddress] = @@ -176,6 +181,9 @@ object SystemSettingsService { port:Int, genericUser:String) + case class Lfs( + serverUrl: Option[String]) + val DefaultSshPort = 29418 val DefaultSmtpPort = 25 val DefaultLdapPort = 389 @@ -212,6 +220,7 @@ object SystemSettingsService { private val LdapTls = "ldap.tls" private val LdapSsl = "ldap.ssl" private val LdapKeystore = "ldap.keystore" + private val LfsServerUrl = "lfs.server_url" private def getValue[A: ClassTag](props: java.util.Properties, key: String, default: A): A = defining(props.getProperty(key)){ value => diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala index d59229909..7b53ddf53 100644 --- a/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala @@ -4,60 +4,59 @@ import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} import org.json4s._ import org.json4s.jackson.Serialization.{read, write} - import java.util.Date +import gitbucket.core.service.SystemSettingsService + /** * Provides GitLFS Batch API. * * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md */ -class GitLfsBatchServlet extends HttpServlet { +class GitLfsBatchServlet extends HttpServlet with SystemSettingsService { - // TODO GitLFS server url must be configurable - private val GitLfsServerUrl = "http://localhost:9090/git-lfs" - private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats override protected def doPost(req: HttpServletRequest, res: HttpServletResponse): Unit = { val batchRequest = read[BatchRequest](req.getInputStream) + val settings = loadSystemSettings() - val batchResponse = batchRequest.operation match { - case "upload" => - BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - BatchResponseObject( - requestObject.oid, - requestObject.size, - true, - Actions( - upload = Some(Action( - href = GitLfsServerUrl + "/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000) - )) - ) - ) - }) - case "download" => - BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - BatchResponseObject( - requestObject.oid, - requestObject.size, - true, - Actions( - download = Some(Action( - href = GitLfsServerUrl + "/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000) - )) - ) - ) - }) + settings.lfs.serverUrl match { + case None => + throw new IllegalStateException("lfs.server_url is not configured.") + + case Some(serverUrl) => + val batchResponse = batchRequest.operation match { + case "upload" => + BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + BatchResponseObject(requestObject.oid, requestObject.size, true, + Actions( + upload = Some(Action( + href = serverUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000L) + )) + ) + ) + }) + case "download" => + BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + BatchResponseObject(requestObject.oid, requestObject.size, true, + Actions( + download = Some(Action( + href = serverUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000L) + )) + ) + ) + }) + } + + res.setContentType("application/vnd.git-lfs+json") + + val out = res.getWriter + out.print(write(batchResponse)) + out.flush() } - - res.setContentType("application/vnd.git-lfs+json") - - val out = res.getWriter - out.print(write(batchResponse)) - out.flush() } } diff --git a/src/main/scala/gitbucket/core/util/Directory.scala b/src/main/scala/gitbucket/core/util/Directory.scala index 6e2410013..a1d603e5c 100644 --- a/src/main/scala/gitbucket/core/util/Directory.scala +++ b/src/main/scala/gitbucket/core/util/Directory.scala @@ -1,11 +1,9 @@ package gitbucket.core.util import java.io.File -import ControlUtil._ -import org.apache.commons.io.FileUtils /** - * Provides directories used by GitBucket. + * Provides directory locations used by GitBucket. */ object Directory { diff --git a/src/main/twirl/gitbucket/core/admin/system.scala.html b/src/main/twirl/gitbucket/core/admin/system.scala.html index b2a582d5a..fd63210b1 100644 --- a/src/main/twirl/gitbucket/core/admin/system.scala.html +++ b/src/main/twirl/gitbucket/core/admin/system.scala.html @@ -123,27 +123,25 @@
- +
- +
-

- Both of SSH host and Base URL are required if SSH access is enabled. -

@@ -157,14 +155,14 @@
- +
- +
@@ -178,7 +176,7 @@
- +
@@ -259,31 +257,32 @@
- +
- +
- +
- +
@@ -295,13 +294,13 @@
- +
- +
@@ -311,10 +310,20 @@
- -

- Enable notification not only SMTP configuration if you want to send notification email. -

+
+ + + +
+ +
+ +
+ + +
From 88e72bee2c9675b8733e8e63e5ce898039303bf8 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 3 Jan 2017 22:38:09 +0900 Subject: [PATCH 03/13] (refs #1101)Support LFS files in the blob view --- .../RepositoryViewerController.scala | 68 ++++++++++++++----- .../scala/gitbucket/core/util/Directory.scala | 2 + .../twirl/gitbucket/core/repo/blob.scala.html | 6 +- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index a6a460a21..e336a8ed4 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -1,6 +1,7 @@ package gitbucket.core.controller -import javax.servlet.http.{HttpServletResponse, HttpServletRequest} +import java.io.FileInputStream +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import gitbucket.core.plugin.PluginRegistry import gitbucket.core.repo.html @@ -16,9 +17,8 @@ import gitbucket.core.model.{Account, WebHook} import gitbucket.core.service.WebHookService._ import gitbucket.core.view import gitbucket.core.view.helpers - import io.github.gitbucket.scalatra.forms._ -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{FileUtils, IOUtils} import org.eclipse.jgit.api.{ArchiveCommand, Git} import org.eclipse.jgit.archive.{TgzFormat, ZipFormat} import org.eclipse.jgit.dircache.DirCache @@ -255,13 +255,9 @@ trait RepositoryViewerControllerBase extends ControllerBase { val (id, path) = repository.splitPath(multiParams("splat").head) using(Git.open(getRepositoryDir(repository.owner, repository.name))){ git => val revCommit = JGitUtil.getRevCommitFromId(git, git.getRepository.resolve(id)) - getPathObjectId(git, path, revCommit).flatMap { objectId => - JGitUtil.getObjectLoaderFromId(git, objectId){ loader => - contentType = FileUtil.getMimeType(path) - response.setContentLength(loader.getSize.toInt) - loader.copyTo(response.outputStream) - () - } + + getPathObjectId(git, path, revCommit).map { objectId => + responseRawFile(git, objectId, path) } getOrElse NotFound() } }) @@ -277,23 +273,61 @@ trait RepositoryViewerControllerBase extends ControllerBase { getPathObjectId(git, path, revCommit).map { objectId => if(raw){ // Download (This route is left for backword compatibility) - JGitUtil.getObjectLoaderFromId(git, objectId){ loader => - contentType = FileUtil.getMimeType(path) - response.setContentLength(loader.getSize.toInt) - loader.copyTo(response.outputStream) - () - } getOrElse NotFound() + responseRawFile(git, objectId, path) } else { html.blob(id, repository, path.split("/").toList, JGitUtil.getContentInfo(git, path, objectId), new JGitUtil.CommitInfo(JGitUtil.getLastModifiedCommit(git, revCommit, path)), hasDeveloperRole(repository.owner, repository.name, context.loginAccount), - request.paths(2) == "blame") + request.paths(2) == "blame", + isLfsFile(git, objectId)) } } getOrElse NotFound() } }) + private def isLfsFile(git: Git, objectId: ObjectId): Boolean = { + JGitUtil.getObjectLoaderFromId(git, objectId){ loader => + if(loader.isLarge){ + false + } else { + new String(loader.getCachedBytes, "UTF-8").startsWith("version https://git-lfs.github.com/spec/v1") + } + }.getOrElse(false) + } + + private def responseRawFile(git: Git, objectId: ObjectId, path: String): Unit = { + JGitUtil.getObjectLoaderFromId(git, objectId){ loader => + contentType = FileUtil.getMimeType(path) + + if(loader.isLarge || context.settings.lfs.serverUrl.isEmpty){ + response.setContentLength(loader.getSize.toInt) + loader.copyTo(response.outputStream) + } else { + val bytes = loader.getCachedBytes + val text = new String(bytes, "UTF-8") + + if(text.startsWith("version https://git-lfs.github.com/spec/v1")){ + // LFS objects + val attrs = text.split("\n").map { line => + val dim = line.split(" ") + dim(0) -> dim(1) + }.toMap + + response.setContentLength(attrs("size").toInt) + val hash = attrs("oid").split(":")(1) + + using(new FileInputStream(Directory.LfsHome + "/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash)){ in => + IOUtils.copy(in, response.getOutputStream) + } + } else { + response.setContentLength(loader.getSize.toInt) + response.getOutputStream.write(bytes) + } + } + } + } + get("/:owner/:repository/blame/*"){ blobRoute.action() } diff --git a/src/main/scala/gitbucket/core/util/Directory.scala b/src/main/scala/gitbucket/core/util/Directory.scala index a1d603e5c..671344350 100644 --- a/src/main/scala/gitbucket/core/util/Directory.scala +++ b/src/main/scala/gitbucket/core/util/Directory.scala @@ -36,6 +36,8 @@ object Directory { val TemporaryHome = s"${GitBucketHome}/tmp" + val LfsHome = s"${GitBucketHome}/lfs" + /** * Substance directory of the repository. */ diff --git a/src/main/twirl/gitbucket/core/repo/blob.scala.html b/src/main/twirl/gitbucket/core/repo/blob.scala.html index 9ae999d06..e5a8e6355 100644 --- a/src/main/twirl/gitbucket/core/repo/blob.scala.html +++ b/src/main/twirl/gitbucket/core/repo/blob.scala.html @@ -4,7 +4,8 @@ content: gitbucket.core.util.JGitUtil.ContentInfo, latestCommit: gitbucket.core.util.JGitUtil.CommitInfo, hasWritePermission: Boolean, - isBlame: Boolean)(implicit context: gitbucket.core.controller.Context) + isBlame: Boolean, + isLfsFile: Boolean)(implicit context: gitbucket.core.controller.Context) @import gitbucket.core.view.helpers @gitbucket.core.html.main(s"${(repository.name :: pathList).mkString("/")} at ${helpers.encodeRefName(branch)} - ${repository.owner}/${repository.name}", Some(repository)) { @gitbucket.core.html.menu("files", repository){ @@ -45,6 +46,9 @@ @section / } } + @if(isLfsFile){ + LFS + }
@helpers.avatar(latestCommit, 28) From 30c8d3c39c660978c39b35537817c427475178c9 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Tue, 3 Jan 2017 22:44:30 +0900 Subject: [PATCH 04/13] (refs #1101)Fix testcase --- .../scala/gitbucket/core/view/AvatarImageProviderSpec.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala b/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala index d30eb7fc5..5120a5604 100644 --- a/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala +++ b/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala @@ -5,7 +5,7 @@ import java.util.Date import gitbucket.core.model.Account import gitbucket.core.service.{RequestCache, SystemSettingsService} import gitbucket.core.controller.Context -import SystemSettingsService.SystemSettings +import SystemSettingsService.{Lfs, SystemSettings} import javax.servlet.http.{HttpServletRequest, HttpSession} import play.twirl.api.Html @@ -112,7 +112,8 @@ class AvatarImageProviderSpec extends FunSpec with MockitoSugar { useSMTP = false, smtp = None, ldapAuthentication = false, - ldap = None) + ldap = None, + lfs = Lfs(None)) /** * Adapter to test AvatarImageProviderImpl. From bfc88a489a565f85d986ec330a572bef5400e69b Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 06:17:35 +0900 Subject: [PATCH 05/13] (refs #1101)Fix Batch API url mapping --- .../core/servlet/GitLfsBatchServlet.scala | 96 ------------------- .../core/servlet/GitRepositoryServlet.scala | 96 ++++++++++++++++++- src/main/webapp/WEB-INF/web.xml | 11 --- 3 files changed, 91 insertions(+), 112 deletions(-) delete mode 100644 src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala deleted file mode 100644 index 7b53ddf53..000000000 --- a/src/main/scala/gitbucket/core/servlet/GitLfsBatchServlet.scala +++ /dev/null @@ -1,96 +0,0 @@ -package gitbucket.core.servlet - -import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} - -import org.json4s._ -import org.json4s.jackson.Serialization.{read, write} -import java.util.Date - -import gitbucket.core.service.SystemSettingsService - -/** - * Provides GitLFS Batch API. - * - * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md - */ -class GitLfsBatchServlet extends HttpServlet with SystemSettingsService { - - private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats - - override protected def doPost(req: HttpServletRequest, res: HttpServletResponse): Unit = { - val batchRequest = read[BatchRequest](req.getInputStream) - val settings = loadSystemSettings() - - settings.lfs.serverUrl match { - case None => - throw new IllegalStateException("lfs.server_url is not configured.") - - case Some(serverUrl) => - val batchResponse = batchRequest.operation match { - case "upload" => - BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - BatchResponseObject(requestObject.oid, requestObject.size, true, - Actions( - upload = Some(Action( - href = serverUrl + "/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000L) - )) - ) - ) - }) - case "download" => - BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - BatchResponseObject(requestObject.oid, requestObject.size, true, - Actions( - download = Some(Action( - href = serverUrl + "/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000L) - )) - ) - ) - }) - } - - res.setContentType("application/vnd.git-lfs+json") - - val out = res.getWriter - out.print(write(batchResponse)) - out.flush() - } - } - -} - -case class BatchRequest( - operation: String, - transfers: Seq[String], - objects: Seq[BatchRequestObject] -) - -case class BatchRequestObject( - oid: String, - size: Long -) - -case class BatchUploadResponse( - transfer: String, - objects: Seq[BatchResponseObject] -) - -case class BatchResponseObject( - oid: String, - size: Long, - authenticated: Boolean, - actions: Actions -) - -case class Actions( - download: Option[Action] = None, - upload: Option[Action] = None -) - -case class Action( - href: String, - header: Map[String, String] = Map.empty, - expires_at: Date -) \ No newline at end of file diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index e85804eae..879ab5c40 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -1,6 +1,7 @@ package gitbucket.core.servlet import java.io.File +import java.util.Date import gitbucket.core.api import gitbucket.core.model.{Session, WebHook} @@ -11,16 +12,16 @@ import gitbucket.core.service._ import gitbucket.core.util.ControlUtil._ import gitbucket.core.util.Implicits._ import gitbucket.core.util._ - import org.eclipse.jgit.api.Git import org.eclipse.jgit.http.server.GitServlet import org.eclipse.jgit.lib._ import org.eclipse.jgit.transport._ import org.eclipse.jgit.transport.resolver._ import org.slf4j.LoggerFactory - import javax.servlet.ServletConfig -import javax.servlet.http.{HttpServletResponse, HttpServletRequest} +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +import org.json4s.jackson.Serialization._ /** @@ -32,7 +33,8 @@ import javax.servlet.http.{HttpServletResponse, HttpServletRequest} class GitRepositoryServlet extends GitServlet with SystemSettingsService { private val logger = LoggerFactory.getLogger(classOf[GitRepositoryServlet]) - + private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats + override def init(config: ServletConfig): Unit = { setReceivePackFactory(new GitBucketReceivePackFactory()) @@ -45,15 +47,61 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { override def service(req: HttpServletRequest, res: HttpServletResponse): Unit = { val agent = req.getHeader("USER-AGENT") val index = req.getRequestURI.indexOf(".git") - if(index >= 0 && (agent == null || agent.toLowerCase.indexOf("git/") < 0)){ + if(index >= 0 && (agent == null || agent.toLowerCase.indexOf("git") < 0)){ // redirect for browsers val paths = req.getRequestURI.substring(0, index).split("/") res.sendRedirect(baseUrl(req) + "/" + paths.dropRight(1).last + "/" + paths.last) + + } else if(req.getMethod.toUpperCase == "POST" && req.getRequestURI.endsWith("/info/lfs/objects/batch")){ + serviceGitLfsBatchAPI(req, res) + } else { // response for git client super.service(req, res) } } + + protected def serviceGitLfsBatchAPI(req: HttpServletRequest, res: HttpServletResponse): Unit = { + val batchRequest = read[GitLfs.BatchRequest](req.getInputStream) + val settings = loadSystemSettings() + + settings.lfs.serverUrl match { + case None => + throw new IllegalStateException("lfs.server_url is not configured.") + + case Some(serverUrl) => + val batchResponse = batchRequest.operation match { + case "upload" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + upload = Some(GitLfs.Action( + href = serverUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000L) + )) + ) + ) + }) + case "download" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + download = Some(GitLfs.Action( + href = serverUrl + "/" + requestObject.oid, + expires_at = new Date(System.currentTimeMillis + 60000L) + )) + ) + ) + }) + } + + res.setContentType("application/vnd.git-lfs+json") + + val out = res.getWriter + out.print(write(batchResponse)) + out.flush() + } + } } class GitBucketRepositoryResolver(parent: FileResolver[HttpServletRequest]) extends RepositoryResolver[HttpServletRequest] { @@ -232,3 +280,41 @@ class CommitLogHook(owner: String, repository: String, pusher: String, baseUrl: } } + +object GitLfs { + + case class BatchRequest( + operation: String, + transfers: Seq[String], + objects: Seq[BatchRequestObject] + ) + + case class BatchRequestObject( + oid: String, + size: Long + ) + + case class BatchUploadResponse( + transfer: String, + objects: Seq[BatchResponseObject] + ) + + case class BatchResponseObject( + oid: String, + size: Long, + authenticated: Boolean, + actions: Actions + ) + + case class Actions( + download: Option[Action] = None, + upload: Option[Action] = None + ) + + case class Action( + href: String, + header: Map[String, String] = Map.empty, + expires_at: Date + ) + +} \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 066f99f52..9634066c2 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -46,17 +46,6 @@ /git/* - - GitLfsBatchServlet - gitbucket.core.servlet.GitLfsBatchServlet - true - - - - GitLfsBatchServlet - /git/root/git-lfs-test.git/info/lfs/* - - From 9cded1b4deea7689bffb4d8fcdc256058facc8d0 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 07:09:56 +0900 Subject: [PATCH 06/13] (refs #1101)Add original GitLFS Transfer API implementation --- .../core/controller/ControllerBase.scala | 2 +- .../RepositoryViewerController.scala | 4 +- .../core/servlet/GitLfsTransferServlet.scala | 78 +++++++++++++++++++ .../core/servlet/GitRepositoryServlet.scala | 16 +++- .../gitbucket/core/util/ControlUtil.scala | 14 ++++ src/main/webapp/WEB-INF/web.xml | 11 +++ 6 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala diff --git a/src/main/scala/gitbucket/core/controller/ControllerBase.scala b/src/main/scala/gitbucket/core/controller/ControllerBase.scala index eeedd1c09..5db473f77 100644 --- a/src/main/scala/gitbucket/core/controller/ControllerBase.scala +++ b/src/main/scala/gitbucket/core/controller/ControllerBase.scala @@ -57,7 +57,7 @@ abstract class ControllerBase extends ScalatraFilter // Redirect to dashboard httpResponse.sendRedirect(baseUrl + "/") } - } else if(path.startsWith("/git/")){ + } else if(path.startsWith("/git/") || path.startsWith("/git-lfs/")){ // Git repository chain.doFilter(request, response) } else { diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index e336a8ed4..8d0730388 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -315,9 +315,9 @@ trait RepositoryViewerControllerBase extends ControllerBase { }.toMap response.setContentLength(attrs("size").toInt) - val hash = attrs("oid").split(":")(1) + val oid = attrs("oid").split(":")(1) - using(new FileInputStream(Directory.LfsHome + "/" + hash.substring(0, 2) + "/" + hash.substring(2, 4) + "/" + hash)){ in => + using(new FileInputStream(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid)){ in => IOUtils.copy(in, response.getOutputStream) } } else { diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala new file mode 100644 index 000000000..afac4304a --- /dev/null +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -0,0 +1,78 @@ +package gitbucket.core.servlet + +import java.io.{File, FileInputStream, FileOutputStream} +import java.text.MessageFormat +import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} + +import gitbucket.core.util.Directory +import org.apache.commons.io.{FileUtils, IOUtils} +import org.json4s.jackson.Serialization._ +import org.apache.http.HttpStatus +import gitbucket.core.util.ControlUtil._ + +/** + * Provides GitLFS Transfer API + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md + */ +class GitLfsTransferServlet extends HttpServlet { + + private implicit val jsonFormats = gitbucket.core.api.JsonFormat.jsonFormats + private val LongObjectIdLength = 32 + private val LongObjectIdStringLength = LongObjectIdLength * 2 + + override protected def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { + for { + oid <- getObjectId(req, res) + } yield { + val file = new File(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid) + if(file.exists()){ + res.setStatus(HttpStatus.SC_OK) + res.setContentType("application/octet-stream") + res.setContentLength(file.length.toInt) + using(new FileInputStream(file), res.getOutputStream){ (in, out) => + IOUtils.copy(in, out) + out.flush() + } + } else { + sendError(res, HttpStatus.SC_NOT_FOUND, + MessageFormat.format("Object ''{0}'' not found", oid)) + } + } + } + + override protected def doPut(req: HttpServletRequest, res: HttpServletResponse): Unit = { + for { + oid <- getObjectId(req, res) + } yield { + val file = new File(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid) + FileUtils.forceMkdir(file.getParentFile) + using(req.getInputStream, new FileOutputStream(file)){ (in, out) => + IOUtils.copy(in, out) + } + res.setStatus(HttpStatus.SC_OK) + } + } + + private def getObjectId(req: HttpServletRequest, rsp: HttpServletResponse): Option[String] = { + val info: String = req.getPathInfo + val length: Int = 1 + LongObjectIdStringLength + if (info.length != length) { + sendError(rsp, HttpStatus.SC_UNPROCESSABLE_ENTITY, + MessageFormat.format("Invalid pathInfo ''{0}'' does not match ''/'{'SHA-256'}'''", info)) + None + } else { + Some(info.substring(1, length)) + } + } + + private def sendError(res: HttpServletResponse, status: Int, message: String): Unit = { + res.setStatus(status) + using(res.getWriter()){ out => + out.write(write(GitLfs.Error(message))) + out.flush() + } + } + +} + + diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index 879ab5c40..c0cf9dbd4 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -61,6 +61,10 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { } } + /** + * Provides GitLFS Batch API + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + */ protected def serviceGitLfsBatchAPI(req: HttpServletRequest, res: HttpServletResponse): Unit = { val batchRequest = read[GitLfs.BatchRequest](req.getInputStream) val settings = loadSystemSettings() @@ -96,10 +100,10 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { } res.setContentType("application/vnd.git-lfs+json") - - val out = res.getWriter - out.print(write(batchResponse)) - out.flush() + using(res.getWriter){ out => + out.print(write(batchResponse)) + out.flush() + } } } } @@ -317,4 +321,8 @@ object GitLfs { expires_at: Date ) + case class Error( + message: String + ) + } \ No newline at end of file diff --git a/src/main/scala/gitbucket/core/util/ControlUtil.scala b/src/main/scala/gitbucket/core/util/ControlUtil.scala index a323c4285..74569ac47 100644 --- a/src/main/scala/gitbucket/core/util/ControlUtil.scala +++ b/src/main/scala/gitbucket/core/util/ControlUtil.scala @@ -20,6 +20,20 @@ object ControlUtil { } } + def using[A <% { def close(): Unit }, B <% { def close(): Unit }, C](resource1: A, resource2: B)(f: (A, B) => C): C = + try f(resource1, resource2) finally { + if(resource1 != null){ + ignoring(classOf[Throwable]) { + resource1.close() + } + } + if(resource2 != null){ + ignoring(classOf[Throwable]) { + resource2.close() + } + } + } + def using[T](git: Git)(f: Git => T): T = try f(git) finally git.getRepository.close() diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 9634066c2..36b904eb2 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -46,6 +46,17 @@ /git/* + + GitLfsTransferServlet + gitbucket.core.servlet.GitLfsTransferServlet + + + + GitLfsTransferServlet + /git-lfs/* + + + From 3b99e619db03c658809561693c4695c36c414ffa Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 07:34:04 +0900 Subject: [PATCH 07/13] (refs #1101)Remove LFS setting --- .../core/controller/RepositoryViewerController.scala | 4 ++-- .../core/controller/SystemSettingsController.scala | 5 +---- .../gitbucket/core/service/SystemSettingsService.scala | 10 ++-------- .../gitbucket/core/servlet/GitLfsTransferServlet.scala | 6 +++--- .../gitbucket/core/servlet/GitRepositoryServlet.scala | 8 ++++---- src/main/scala/gitbucket/core/util/FileUtil.scala | 4 ++++ src/main/twirl/gitbucket/core/admin/system.scala.html | 2 ++ 7 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index 8d0730388..a0c4f675d 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -300,7 +300,7 @@ trait RepositoryViewerControllerBase extends ControllerBase { JGitUtil.getObjectLoaderFromId(git, objectId){ loader => contentType = FileUtil.getMimeType(path) - if(loader.isLarge || context.settings.lfs.serverUrl.isEmpty){ + if(loader.isLarge){ response.setContentLength(loader.getSize.toInt) loader.copyTo(response.outputStream) } else { @@ -317,7 +317,7 @@ trait RepositoryViewerControllerBase extends ControllerBase { response.setContentLength(attrs("size").toInt) val oid = attrs("oid").split(":")(1) - using(new FileInputStream(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid)){ in => + using(new FileInputStream(FileUtil.getLfsFilePath(oid))){ in => IOUtils.copy(in, response.getOutputStream) } } else { diff --git a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala index 96a33e78c..bfca8582d 100644 --- a/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala +++ b/src/main/scala/gitbucket/core/controller/SystemSettingsController.scala @@ -58,10 +58,7 @@ trait SystemSettingsControllerBase extends AccountManagementControllerBase { "tls" -> trim(label("Enable TLS", optional(boolean()))), "ssl" -> trim(label("Enable SSL", optional(boolean()))), "keystore" -> trim(label("Keystore", optional(text()))) - )(Ldap.apply)), - "lfs" -> mapping( - "serverUrl" -> trim(label("LDAP host", optional(text()))) - )(Lfs.apply) + )(Ldap.apply)) )(SystemSettings.apply).verifying { settings => Vector( if(settings.ssh && settings.baseUrl.isEmpty){ diff --git a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala index 00199a472..123c1210f 100644 --- a/src/main/scala/gitbucket/core/service/SystemSettingsService.scala +++ b/src/main/scala/gitbucket/core/service/SystemSettingsService.scala @@ -53,7 +53,6 @@ trait SystemSettingsService { ldap.keystore.foreach(x => props.setProperty(LdapKeystore, x)) } } - settings.lfs.serverUrl.foreach { x => props.setProperty(LfsServerUrl, x) } using(new java.io.FileOutputStream(GitBucketConf)){ out => props.store(out, null) } @@ -110,10 +109,7 @@ trait SystemSettingsService { getOptionValue(props, LdapKeystore, None))) } else { None - }, - Lfs( - getOptionValue(props, LfsServerUrl, None) - ) + } ) } } @@ -138,8 +134,7 @@ object SystemSettingsService { useSMTP: Boolean, smtp: Option[Smtp], ldapAuthentication: Boolean, - ldap: Option[Ldap], - lfs: Lfs){ + ldap: Option[Ldap]){ def baseUrl(request: HttpServletRequest): String = baseUrl.fold(request.baseUrl)(_.stripSuffix("/")) def sshAddress:Option[SshAddress] = @@ -220,7 +215,6 @@ object SystemSettingsService { private val LdapTls = "ldap.tls" private val LdapSsl = "ldap.ssl" private val LdapKeystore = "ldap.keystore" - private val LfsServerUrl = "lfs.server_url" private def getValue[A: ClassTag](props: java.util.Properties, key: String, default: A): A = defining(props.getProperty(key)){ value => diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala index afac4304a..e84a53dcd 100644 --- a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -4,7 +4,7 @@ import java.io.{File, FileInputStream, FileOutputStream} import java.text.MessageFormat import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import gitbucket.core.util.Directory +import gitbucket.core.util.{Directory, FileUtil} import org.apache.commons.io.{FileUtils, IOUtils} import org.json4s.jackson.Serialization._ import org.apache.http.HttpStatus @@ -24,7 +24,7 @@ class GitLfsTransferServlet extends HttpServlet { for { oid <- getObjectId(req, res) } yield { - val file = new File(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid) + val file = new File(FileUtil.getLfsFilePath(oid)) if(file.exists()){ res.setStatus(HttpStatus.SC_OK) res.setContentType("application/octet-stream") @@ -44,7 +44,7 @@ class GitLfsTransferServlet extends HttpServlet { for { oid <- getObjectId(req, res) } yield { - val file = new File(Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid) + val file = new File(FileUtil.getLfsFilePath(oid)) FileUtils.forceMkdir(file.getParentFile) using(req.getInputStream, new FileOutputStream(file)){ (in, out) => IOUtils.copy(in, out) diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index c0cf9dbd4..6114b30df 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -69,18 +69,18 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { val batchRequest = read[GitLfs.BatchRequest](req.getInputStream) val settings = loadSystemSettings() - settings.lfs.serverUrl match { + settings.baseUrl match { case None => throw new IllegalStateException("lfs.server_url is not configured.") - case Some(serverUrl) => + case Some(baseUrl) => val batchResponse = batchRequest.operation match { case "upload" => GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, GitLfs.Actions( upload = Some(GitLfs.Action( - href = serverUrl + "/" + requestObject.oid, + href = baseUrl + "/git-lfs/" + requestObject.oid, expires_at = new Date(System.currentTimeMillis + 60000L) )) ) @@ -91,7 +91,7 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, GitLfs.Actions( download = Some(GitLfs.Action( - href = serverUrl + "/" + requestObject.oid, + href = baseUrl + "/git-lfs/" + requestObject.oid, expires_at = new Date(System.currentTimeMillis + 60000L) )) ) diff --git a/src/main/scala/gitbucket/core/util/FileUtil.scala b/src/main/scala/gitbucket/core/util/FileUtil.scala index f753a60f7..9723ad3b1 100644 --- a/src/main/scala/gitbucket/core/util/FileUtil.scala +++ b/src/main/scala/gitbucket/core/util/FileUtil.scala @@ -62,4 +62,8 @@ object FileUtil { "image/jpeg", "image/png", "text/plain") + + def getLfsFilePath(oid: String): String = + Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid + } diff --git a/src/main/twirl/gitbucket/core/admin/system.scala.html b/src/main/twirl/gitbucket/core/admin/system.scala.html index fd63210b1..549f32eba 100644 --- a/src/main/twirl/gitbucket/core/admin/system.scala.html +++ b/src/main/twirl/gitbucket/core/admin/system.scala.html @@ -314,6 +314,7 @@ + @*
+ *@
From 4dfc9fc456900755301c5f770421f103c8449b13 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 07:47:23 +0900 Subject: [PATCH 08/13] (refs #1101)No need transaction for /git-lfs --- .../scala/gitbucket/core/servlet/TransactionFilter.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala index c25a2feaa..a6d5f7ad1 100644 --- a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala +++ b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala @@ -21,8 +21,10 @@ class TransactionFilter extends Filter { def destroy(): Unit = {} def doFilter(req: ServletRequest, res: ServletResponse, chain: FilterChain): Unit = { - if(req.asInstanceOf[HttpServletRequest].getServletPath().startsWith("/assets/")){ - // assets don't need transaction + val servletPath = req.asInstanceOf[HttpServletRequest].getServletPath() + println(servletPath) + if(servletPath.startsWith("/assets/") || servletPath.startsWith("/git-lfs")){ + // assets and git-lfs don't need transaction chain.doFilter(req, res) } else { Database() withTransaction { session => From 8d7ec16ed01a82cdc54f365bc5d9ab7dbb5aaf3f Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 07:48:18 +0900 Subject: [PATCH 09/13] (refs #1101)Remove debug code --- src/main/scala/gitbucket/core/servlet/TransactionFilter.scala | 1 - .../scala/gitbucket/core/view/AvatarImageProviderSpec.scala | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala index a6d5f7ad1..5ace04b62 100644 --- a/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala +++ b/src/main/scala/gitbucket/core/servlet/TransactionFilter.scala @@ -22,7 +22,6 @@ class TransactionFilter extends Filter { def doFilter(req: ServletRequest, res: ServletResponse, chain: FilterChain): Unit = { val servletPath = req.asInstanceOf[HttpServletRequest].getServletPath() - println(servletPath) if(servletPath.startsWith("/assets/") || servletPath.startsWith("/git-lfs")){ // assets and git-lfs don't need transaction chain.doFilter(req, res) diff --git a/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala b/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala index 5120a5604..99feeee99 100644 --- a/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala +++ b/src/test/scala/gitbucket/core/view/AvatarImageProviderSpec.scala @@ -112,8 +112,7 @@ class AvatarImageProviderSpec extends FunSpec with MockitoSugar { useSMTP = false, smtp = None, ldapAuthentication = false, - ldap = None, - lfs = Lfs(None)) + ldap = None) /** * Adapter to test AvatarImageProviderImpl. From 2297ef0bec5d39a83c7565513f5ec3386e4471d0 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 14:25:41 +0900 Subject: [PATCH 10/13] Fix typo --- src/main/scala/gitbucket/core/service/AccessTokenService.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gitbucket/core/service/AccessTokenService.scala b/src/main/scala/gitbucket/core/service/AccessTokenService.scala index 0a8109dc8..a2345be58 100644 --- a/src/main/scala/gitbucket/core/service/AccessTokenService.scala +++ b/src/main/scala/gitbucket/core/service/AccessTokenService.scala @@ -20,7 +20,7 @@ trait AccessTokenService { def tokenToHash(token: String): String = StringUtil.sha1(token) /** - * @retuen (TokenId, Token) + * @return (TokenId, Token) */ def generateAccessToken(userName: String, note: String)(implicit s: Session): (Int, String) = { var token: String = null From d460185317eb66f729a2fd3215d098b8cf8ea2d9 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 16:12:44 +0900 Subject: [PATCH 11/13] (refs #1101)Authentication for Transfer API by one-time token --- .../core/servlet/GitLfsTransferServlet.scala | 16 ++++++++++--- .../core/servlet/GitRepositoryServlet.scala | 7 ++++-- .../gitbucket/core/util/StringUtil.scala | 23 +++++++++++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala index e84a53dcd..426b85f77 100644 --- a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -4,7 +4,7 @@ import java.io.{File, FileInputStream, FileOutputStream} import java.text.MessageFormat import javax.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import gitbucket.core.util.{Directory, FileUtil} +import gitbucket.core.util.{Directory, FileUtil, StringUtil} import org.apache.commons.io.{FileUtils, IOUtils} import org.json4s.jackson.Serialization._ import org.apache.http.HttpStatus @@ -22,7 +22,7 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - oid <- getObjectId(req, res) + oid <- getObjectId(req, res) if checkToken(req, oid) } yield { val file = new File(FileUtil.getLfsFilePath(oid)) if(file.exists()){ @@ -42,7 +42,7 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doPut(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - oid <- getObjectId(req, res) + oid <- getObjectId(req, res) if checkToken(req, oid) } yield { val file = new File(FileUtil.getLfsFilePath(oid)) FileUtils.forceMkdir(file.getParentFile) @@ -53,6 +53,16 @@ class GitLfsTransferServlet extends HttpServlet { } } + private def checkToken(req: HttpServletRequest, oid: String): Boolean = { + val token = req.getHeader("Authorization") + if(token != null){ + val Array(expireAt, targetOid) = StringUtil.decodeBlowfish(token).split(" ") + oid == targetOid && expireAt.toLong > System.currentTimeMillis + } else { + false + } + } + private def getObjectId(req: HttpServletRequest, rsp: HttpServletResponse): Option[String] = { val info: String = req.getPathInfo val length: Int = 1 + LongObjectIdStringLength diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index 6114b30df..f7d232cc9 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -74,6 +74,7 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { throw new IllegalStateException("lfs.server_url is not configured.") case Some(baseUrl) => + val timeout = System.currentTimeMillis + (60000 * 10) // 10 min. val batchResponse = batchRequest.operation match { case "upload" => GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => @@ -81,7 +82,8 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { GitLfs.Actions( upload = Some(GitLfs.Action( href = baseUrl + "/git-lfs/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000L) + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) )) ) ) @@ -92,7 +94,8 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { GitLfs.Actions( download = Some(GitLfs.Action( href = baseUrl + "/git-lfs/" + requestObject.oid, - expires_at = new Date(System.currentTimeMillis + 60000L) + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) )) ) ) diff --git a/src/main/scala/gitbucket/core/util/StringUtil.scala b/src/main/scala/gitbucket/core/util/StringUtil.scala index 76cc38969..f255b1dbc 100644 --- a/src/main/scala/gitbucket/core/util/StringUtil.scala +++ b/src/main/scala/gitbucket/core/util/StringUtil.scala @@ -1,14 +1,23 @@ package gitbucket.core.util import java.net.{URLDecoder, URLEncoder} + import org.mozilla.universalchardet.UniversalDetector import ControlUtil._ import org.apache.commons.io.input.BOMInputStream import org.apache.commons.io.IOUtils +import org.apache.commons.codec.binary.{Base64, StringUtils} + import scala.util.control.Exception._ object StringUtil { + private lazy val BlowfishKey = { + // last 4 numbers in current timestamp + val time = System.currentTimeMillis.toString + time.substring(time.length - 4) + } + def sha1(value: String): String = defining(java.security.MessageDigest.getInstance("SHA-1")){ md => md.update(value.getBytes) @@ -21,6 +30,20 @@ object StringUtil { md.digest.map(b => "%02x".format(b)).mkString } + def encodeBlowfish(value: String): String = { + val spec = new javax.crypto.spec.SecretKeySpec(BlowfishKey.getBytes(), "Blowfish") + val cipher = javax.crypto.Cipher.getInstance("Blowfish") + cipher.init(javax.crypto.Cipher.ENCRYPT_MODE, spec) + new String(Base64.encodeBase64(cipher.doFinal(value.getBytes("UTF-8"))), "UTF-8") + } + + def decodeBlowfish(value: String): String = { + val spec = new javax.crypto.spec.SecretKeySpec(BlowfishKey.getBytes(), "Blowfish") + val cipher = javax.crypto.Cipher.getInstance("Blowfish") + cipher.init(javax.crypto.Cipher.DECRYPT_MODE, spec) + new String(cipher.doFinal(Base64.decodeBase64(value)), "UTF-8") + } + def urlEncode(value: String): String = URLEncoder.encode(value, "UTF-8").replace("+", "%20") def urlDecode(value: String): String = URLDecoder.decode(value, "UTF-8") From eb50b74b4a91b1233ca7895f05f5dc883333ec89 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 20:31:22 +0900 Subject: [PATCH 12/13] (refs #1101)Store LFS files under GITBUCKET_HOME/repositories///lfs --- .../RepositoryViewerController.scala | 9 ++- .../core/servlet/GitLfsTransferServlet.scala | 21 ++---- .../core/servlet/GitRepositoryServlet.scala | 73 ++++++++++--------- .../scala/gitbucket/core/util/Directory.scala | 8 +- .../scala/gitbucket/core/util/FileUtil.scala | 3 +- 5 files changed, 59 insertions(+), 55 deletions(-) diff --git a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala index a0c4f675d..3d59d71aa 100644 --- a/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala +++ b/src/main/scala/gitbucket/core/controller/RepositoryViewerController.scala @@ -257,7 +257,7 @@ trait RepositoryViewerControllerBase extends ControllerBase { val revCommit = JGitUtil.getRevCommitFromId(git, git.getRepository.resolve(id)) getPathObjectId(git, path, revCommit).map { objectId => - responseRawFile(git, objectId, path) + responseRawFile(git, objectId, path, repository) } getOrElse NotFound() } }) @@ -273,7 +273,7 @@ trait RepositoryViewerControllerBase extends ControllerBase { getPathObjectId(git, path, revCommit).map { objectId => if(raw){ // Download (This route is left for backword compatibility) - responseRawFile(git, objectId, path) + responseRawFile(git, objectId, path, repository) } else { html.blob(id, repository, path.split("/").toList, JGitUtil.getContentInfo(git, path, objectId), @@ -296,7 +296,8 @@ trait RepositoryViewerControllerBase extends ControllerBase { }.getOrElse(false) } - private def responseRawFile(git: Git, objectId: ObjectId, path: String): Unit = { + private def responseRawFile(git: Git, objectId: ObjectId, path: String, + repository: RepositoryService.RepositoryInfo): Unit = { JGitUtil.getObjectLoaderFromId(git, objectId){ loader => contentType = FileUtil.getMimeType(path) @@ -317,7 +318,7 @@ trait RepositoryViewerControllerBase extends ControllerBase { response.setContentLength(attrs("size").toInt) val oid = attrs("oid").split(":")(1) - using(new FileInputStream(FileUtil.getLfsFilePath(oid))){ in => + using(new FileInputStream(FileUtil.getLfsFilePath(repository.owner, repository.name, oid))){ in => IOUtils.copy(in, response.getOutputStream) } } else { diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala index 426b85f77..497121d56 100644 --- a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -22,9 +22,9 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - oid <- getObjectId(req, res) if checkToken(req, oid) + (owner, name, oid) <- getPathInfo(req, res) if checkToken(req, oid) } yield { - val file = new File(FileUtil.getLfsFilePath(oid)) + val file = new File(FileUtil.getLfsFilePath(owner, name, oid)) if(file.exists()){ res.setStatus(HttpStatus.SC_OK) res.setContentType("application/octet-stream") @@ -42,9 +42,9 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doPut(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - oid <- getObjectId(req, res) if checkToken(req, oid) + (owner, name, oid) <- getPathInfo(req, res) if checkToken(req, oid) } yield { - val file = new File(FileUtil.getLfsFilePath(oid)) + val file = new File(FileUtil.getLfsFilePath(owner, name, oid)) FileUtils.forceMkdir(file.getParentFile) using(req.getInputStream, new FileOutputStream(file)){ (in, out) => IOUtils.copy(in, out) @@ -63,15 +63,10 @@ class GitLfsTransferServlet extends HttpServlet { } } - private def getObjectId(req: HttpServletRequest, rsp: HttpServletResponse): Option[String] = { - val info: String = req.getPathInfo - val length: Int = 1 + LongObjectIdStringLength - if (info.length != length) { - sendError(rsp, HttpStatus.SC_UNPROCESSABLE_ENTITY, - MessageFormat.format("Invalid pathInfo ''{0}'' does not match ''/'{'SHA-256'}'''", info)) - None - } else { - Some(info.substring(1, length)) + private def getPathInfo(req: HttpServletRequest, res: HttpServletResponse): Option[(String, String, String)] = { + req.getRequestURI.substring(1).split("/") match { + case Array(_, owner, name, oid) => Some((owner, name, oid)) + case _ => None } } diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index f7d232cc9..0e3ff4d02 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -70,43 +70,48 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { val settings = loadSystemSettings() settings.baseUrl match { - case None => + case None => { throw new IllegalStateException("lfs.server_url is not configured.") + } + case Some(baseUrl) => { + req.getRequestURI.substring(1).replace(".git/", "/").split("/") match { + case Array(_, owner, name, _*) => { + val timeout = System.currentTimeMillis + (60000 * 10) // 10 min. + val batchResponse = batchRequest.operation match { + case "upload" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + upload = Some(GitLfs.Action( + href = baseUrl + "/git-lfs/" + owner + "/" + name + "/" + requestObject.oid, + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) + )) + ) + ) + }) + case "download" => + GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => + GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, + GitLfs.Actions( + download = Some(GitLfs.Action( + href = baseUrl + "/git-lfs/" + owner + "/" + name + "/" + requestObject.oid, + header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), + expires_at = new Date(timeout) + )) + ) + ) + }) + } - case Some(baseUrl) => - val timeout = System.currentTimeMillis + (60000 * 10) // 10 min. - val batchResponse = batchRequest.operation match { - case "upload" => - GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, - GitLfs.Actions( - upload = Some(GitLfs.Action( - href = baseUrl + "/git-lfs/" + requestObject.oid, - header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), - expires_at = new Date(timeout) - )) - ) - ) - }) - case "download" => - GitLfs.BatchUploadResponse("basic", batchRequest.objects.map { requestObject => - GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, - GitLfs.Actions( - download = Some(GitLfs.Action( - href = baseUrl + "/git-lfs/" + requestObject.oid, - header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), - expires_at = new Date(timeout) - )) - ) - ) - }) - } - - res.setContentType("application/vnd.git-lfs+json") - using(res.getWriter){ out => - out.print(write(batchResponse)) - out.flush() + res.setContentType("application/vnd.git-lfs+json") + using(res.getWriter){ out => + out.print(write(batchResponse)) + out.flush() + } + } } + } } } } diff --git a/src/main/scala/gitbucket/core/util/Directory.scala b/src/main/scala/gitbucket/core/util/Directory.scala index 671344350..e73bca8ef 100644 --- a/src/main/scala/gitbucket/core/util/Directory.scala +++ b/src/main/scala/gitbucket/core/util/Directory.scala @@ -36,8 +36,6 @@ object Directory { val TemporaryHome = s"${GitBucketHome}/tmp" - val LfsHome = s"${GitBucketHome}/lfs" - /** * Substance directory of the repository. */ @@ -50,6 +48,12 @@ object Directory { def getAttachedDir(owner: String, repository: String): File = new File(s"${RepositoryHome}/${owner}/${repository}/comments") + /** + * Directory for files which are attached to issue. + */ + def getLfsDir(owner: String, repository: String): File = + new File(s"${RepositoryHome}/${owner}/${repository}/lfs") + /** * Directory for uploaded files by the specified user. */ diff --git a/src/main/scala/gitbucket/core/util/FileUtil.scala b/src/main/scala/gitbucket/core/util/FileUtil.scala index 9723ad3b1..9f7a3220d 100644 --- a/src/main/scala/gitbucket/core/util/FileUtil.scala +++ b/src/main/scala/gitbucket/core/util/FileUtil.scala @@ -63,7 +63,6 @@ object FileUtil { "image/png", "text/plain") - def getLfsFilePath(oid: String): String = - Directory.LfsHome + "/" + oid.substring(0, 2) + "/" + oid.substring(2, 4) + "/" + oid + def getLfsFilePath(owner: String, name: String, oid: String): String = Directory.getLfsDir(owner, name) + "/" + oid } From b95d912542eeeec470f7a76ed419b109241e9062 Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 4 Jan 2017 20:46:50 +0900 Subject: [PATCH 13/13] (refs #1101)Fix variable names --- .../gitbucket/core/servlet/GitLfsTransferServlet.scala | 10 +++++----- .../gitbucket/core/servlet/GitRepositoryServlet.scala | 6 +++--- src/main/scala/gitbucket/core/util/FileUtil.scala | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala index 497121d56..acdb2f60b 100644 --- a/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitLfsTransferServlet.scala @@ -22,9 +22,9 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doGet(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - (owner, name, oid) <- getPathInfo(req, res) if checkToken(req, oid) + (owner, repository, oid) <- getPathInfo(req, res) if checkToken(req, oid) } yield { - val file = new File(FileUtil.getLfsFilePath(owner, name, oid)) + val file = new File(FileUtil.getLfsFilePath(owner, repository, oid)) if(file.exists()){ res.setStatus(HttpStatus.SC_OK) res.setContentType("application/octet-stream") @@ -42,9 +42,9 @@ class GitLfsTransferServlet extends HttpServlet { override protected def doPut(req: HttpServletRequest, res: HttpServletResponse): Unit = { for { - (owner, name, oid) <- getPathInfo(req, res) if checkToken(req, oid) + (owner, repository, oid) <- getPathInfo(req, res) if checkToken(req, oid) } yield { - val file = new File(FileUtil.getLfsFilePath(owner, name, oid)) + val file = new File(FileUtil.getLfsFilePath(owner, repository, oid)) FileUtils.forceMkdir(file.getParentFile) using(req.getInputStream, new FileOutputStream(file)){ (in, out) => IOUtils.copy(in, out) @@ -65,7 +65,7 @@ class GitLfsTransferServlet extends HttpServlet { private def getPathInfo(req: HttpServletRequest, res: HttpServletResponse): Option[(String, String, String)] = { req.getRequestURI.substring(1).split("/") match { - case Array(_, owner, name, oid) => Some((owner, name, oid)) + case Array(_, owner, repository, oid) => Some((owner, repository, oid)) case _ => None } } diff --git a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala index 0e3ff4d02..d7b359a18 100644 --- a/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala +++ b/src/main/scala/gitbucket/core/servlet/GitRepositoryServlet.scala @@ -75,7 +75,7 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { } case Some(baseUrl) => { req.getRequestURI.substring(1).replace(".git/", "/").split("/") match { - case Array(_, owner, name, _*) => { + case Array(_, owner, repository, _*) => { val timeout = System.currentTimeMillis + (60000 * 10) // 10 min. val batchResponse = batchRequest.operation match { case "upload" => @@ -83,7 +83,7 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, GitLfs.Actions( upload = Some(GitLfs.Action( - href = baseUrl + "/git-lfs/" + owner + "/" + name + "/" + requestObject.oid, + href = baseUrl + "/git-lfs/" + owner + "/" + repository + "/" + requestObject.oid, header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), expires_at = new Date(timeout) )) @@ -95,7 +95,7 @@ class GitRepositoryServlet extends GitServlet with SystemSettingsService { GitLfs.BatchResponseObject(requestObject.oid, requestObject.size, true, GitLfs.Actions( download = Some(GitLfs.Action( - href = baseUrl + "/git-lfs/" + owner + "/" + name + "/" + requestObject.oid, + href = baseUrl + "/git-lfs/" + owner + "/" + repository + "/" + requestObject.oid, header = Map("Authorization" -> StringUtil.encodeBlowfish(timeout + " " + requestObject.oid)), expires_at = new Date(timeout) )) diff --git a/src/main/scala/gitbucket/core/util/FileUtil.scala b/src/main/scala/gitbucket/core/util/FileUtil.scala index 9f7a3220d..4836c23b9 100644 --- a/src/main/scala/gitbucket/core/util/FileUtil.scala +++ b/src/main/scala/gitbucket/core/util/FileUtil.scala @@ -63,6 +63,7 @@ object FileUtil { "image/png", "text/plain") - def getLfsFilePath(owner: String, name: String, oid: String): String = Directory.getLfsDir(owner, name) + "/" + oid + def getLfsFilePath(owner: String, repository: String, oid: String): String = + Directory.getLfsDir(owner, repository) + "/" + oid }