From a1b8d1cd845fb3812a119c339384b6d83d97dfd0 Mon Sep 17 00:00:00 2001 From: takezoe Date: Thu, 18 Jul 2013 19:08:03 +0900 Subject: [PATCH 1/4] Add Guava to use CacheBuilder. --- project/build.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project/build.scala b/project/build.scala index 774c0dd63..8450203b5 100644 --- a/project/build.scala +++ b/project/build.scala @@ -33,6 +33,8 @@ object MyBuild extends Build { "org.apache.commons" % "commons-compress" % "1.5", "com.typesafe.slick" %% "slick" % "1.0.1", "com.h2database" % "h2" % "1.3.171", + "com.google.guava" % "guava" % "14.0.1", + "com.google.code.findbugs" % "jsr305" % "2.0.1", "ch.qos.logback" % "logback-classic" % "1.0.6" % "runtime", "org.eclipse.jetty" % "jetty-webapp" % "8.1.8.v20121106" % "container", "org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" % "container;provided;test" artifacts (Artifact("javax.servlet", "jar", "jar")) From 134624967b2ad0c07edefe8ba11588e19c1214c3 Mon Sep 17 00:00:00 2001 From: takezoe Date: Thu, 18 Jul 2013 19:09:21 +0900 Subject: [PATCH 2/4] Store search results into singleton cache. --- src/main/scala/app/IndexController.scala | 98 +++++++++++++++--------- 1 file changed, 61 insertions(+), 37 deletions(-) diff --git a/src/main/scala/app/IndexController.scala b/src/main/scala/app/IndexController.scala index 89a264d6b..12728b66e 100644 --- a/src/main/scala/app/IndexController.scala +++ b/src/main/scala/app/IndexController.scala @@ -8,13 +8,16 @@ import org.eclipse.jgit.treewalk.TreeWalk import org.eclipse.jgit.revwalk.RevWalk import scala.collection.mutable.ListBuffer import org.eclipse.jgit.lib.FileMode +import com.google.common.cache.{CacheLoader, CacheBuilder} +import java.util.concurrent.TimeUnit +import model.Issue class IndexController extends IndexControllerBase - with RepositoryService with AccountService with SystemSettingsService with ActivityService with IssuesService + with RepositoryService with AccountService with SystemSettingsService with ActivityService with ReferrerAuthenticator trait IndexControllerBase extends ControllerBase { self: RepositoryService - with SystemSettingsService with ActivityService with IssuesService + with SystemSettingsService with ActivityService with ReferrerAuthenticator => val searchForm = mapping( @@ -40,11 +43,11 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService } get("/:owner/:repository/search")(referrersOnly { repository => + import SearchCache._ val query = params("q").trim val target = params.getOrElse("type", "code") - val issues = if(query.isEmpty) Nil else searchIssuesByKeyword(repository.owner, repository.name, query) - val files = if(query.isEmpty) Nil else searchRepositoryFiles(repository.owner, repository.name, query) + val SearchResult(files, issues) = cache.get(repository.owner, repository.name, query) target.toLowerCase match { case "issue" => @@ -69,6 +72,59 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService } }) + private def getHighlightText(content: String, query: String): (String, Int) = { + val keywords = StringUtil.splitWords(query.toLowerCase) + val lowerText = content.toLowerCase + val indices = keywords.map(lowerText.indexOf _) + + if(!indices.exists(_ < 0)){ + val lineNumber = content.substring(0, indices.min).split("\n").size - 1 + val highlightText = StringUtil.escapeHtml(content.split("\n").drop(lineNumber).take(5).mkString("\n")) + .replaceAll("(?i)(" + keywords.map("\\Q" + _ + "\\E").mkString("|") + ")", + "$1") + (highlightText, lineNumber + 1) + } else { + (content.split("\n").take(5).mkString("\n"), 1) + } + } + +} + +case class IssueSearchResult( + issueId: Int, + title: String, + openedUserName: String, + registeredDate: java.util.Date, + commentCount: Int, + highlightText: String) + +case class FileSearchResult( + path: String, + lastModified: java.util.Date, + highlightText: String, + highlightLineNumber: Int) + +object SearchCache extends IssuesService { + + case class SearchResult( + files: List[(String, String)], + issues: List[(Issue, Int, String)] + ) + + val cache = CacheBuilder.newBuilder() + .maximumSize(100) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build( + new CacheLoader[(String, String, String), SearchResult]() { + override def load(key: (String, String, String)) = { + println("** Cache is reloaded! **") + val (owner, repository, query) = key + val issues = if(query.isEmpty) Nil else searchIssuesByKeyword(owner, repository, query) + val files = if(query.isEmpty) Nil else searchRepositoryFiles(owner, repository, query) + SearchResult(files, issues) + } + }) + private def searchRepositoryFiles(owner: String, repository: String, query: String): List[(String, String)] = { JGitUtil.withGit(getRepositoryDir(owner, repository)){ git => val revWalk = new RevWalk(git.getRepository) @@ -101,36 +157,4 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService list.toList } } - - private def getHighlightText(content: String, query: String): (String, Int) = { - val keywords = StringUtil.splitWords(query.toLowerCase) - val lowerText = content.toLowerCase - val indices = keywords.map(lowerText.indexOf _) - - if(!indices.exists(_ < 0)){ - val lineNumber = content.substring(0, indices.min).split("\n").size - 1 - val highlightText = StringUtil.escapeHtml(content.split("\n").drop(lineNumber).take(5).mkString("\n")) - .replaceAll("(?i)(" + keywords.map("\\Q" + _ + "\\E").mkString("|") + ")", - "$1") - (highlightText, lineNumber + 1) - } else { - (content.split("\n").take(5).mkString("\n"), 1) - } - } - -} - - -case class IssueSearchResult( - issueId: Int, - title: String, - openedUserName: String, - registeredDate: java.util.Date, - commentCount: Int, - highlightText: String) - -case class FileSearchResult( - path: String, - lastModified: java.util.Date, - highlightText: String, - highlightLineNumber: Int) \ No newline at end of file +} \ No newline at end of file From 7483ad1732ccc105c91d276bfbc18abd1cacb5b4 Mon Sep 17 00:00:00 2001 From: takezoe Date: Thu, 18 Jul 2013 20:02:12 +0900 Subject: [PATCH 3/4] Pagination for repository search results. --- src/main/scala/app/IndexController.scala | 25 +++++++++++---- src/main/twirl/helper/paginator.scala.html | 36 +++++++++++----------- src/main/twirl/search/code.scala.html | 10 ++++-- src/main/twirl/search/issues.scala.html | 12 ++++++-- 4 files changed, 54 insertions(+), 29 deletions(-) diff --git a/src/main/scala/app/IndexController.scala b/src/main/scala/app/IndexController.scala index 12728b66e..e2c06425f 100644 --- a/src/main/scala/app/IndexController.scala +++ b/src/main/scala/app/IndexController.scala @@ -43,9 +43,16 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService } get("/:owner/:repository/search")(referrersOnly { repository => - import SearchCache._ + import RepositorySearch._ val query = params("q").trim val target = params.getOrElse("type", "code") + val page = try { + val i = params.getOrElse("page", "1").toInt + if(i <= 0) 1 else i + } catch { + case e: NumberFormatException => 1 + } + val SearchResult(files, issues) = cache.get(repository.owner, repository.name, query) @@ -59,15 +66,19 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService issue.registeredDate, commentCount, getHighlightText(content, query)._1) - }, files.size, query, repository) + }, files.size, query, page, repository) case _ => JGitUtil.withGit(getRepositoryDir(repository.owner, repository.name)){ git => val commits = JGitUtil.getLatestCommitFromPaths(git, files.toList.map(_._1), "HEAD") search.html.code(files.toList.map { case (path, text) => val (highlightText, lineNumber) = getHighlightText(text, query) - FileSearchResult(path, commits(path).getCommitterIdent.getWhen, highlightText, lineNumber) - }, issues.size, query, repository) + FileSearchResult( + path, + commits(path).getCommitterIdent.getWhen, + highlightText, + lineNumber) + }, issues.size, query, page, repository) } } }) @@ -104,7 +115,10 @@ case class FileSearchResult( highlightText: String, highlightLineNumber: Int) -object SearchCache extends IssuesService { +object RepositorySearch extends IssuesService { + + val CodeLimit = 10 + val IssueLimit = 10 case class SearchResult( files: List[(String, String)], @@ -117,7 +131,6 @@ object SearchCache extends IssuesService { .build( new CacheLoader[(String, String, String), SearchResult]() { override def load(key: (String, String, String)) = { - println("** Cache is reloaded! **") val (owner, repository, query) = key val issues = if(query.isEmpty) Nil else searchIssuesByKeyword(owner, repository, query) val files = if(query.isEmpty) Nil else searchRepositoryFiles(owner, repository, query) diff --git a/src/main/twirl/helper/paginator.scala.html b/src/main/twirl/helper/paginator.scala.html index 4a37d8144..09250041d 100644 --- a/src/main/twirl/helper/paginator.scala.html +++ b/src/main/twirl/helper/paginator.scala.html @@ -1,32 +1,32 @@ @(page: Int, count: Int, limit: Int, width: Int, baseURL: String) -@defining(view.Pagination(page, count, service.IssuesService.IssueLimit, width)){ p => +@defining(view.Pagination(page, count, limit, width)){ p => @if(p.count > p.limit){ diff --git a/src/main/twirl/search/code.scala.html b/src/main/twirl/search/code.scala.html index 491fabeed..7dca4a488 100644 --- a/src/main/twirl/search/code.scala.html +++ b/src/main/twirl/search/code.scala.html @@ -1,4 +1,8 @@ -@(files: List[app.FileSearchResult], issueCount: Int, query: String, repository: service.RepositoryService.RepositoryInfo)(implicit context: app.Context) +@(files: List[app.FileSearchResult], + issueCount: Int, + query: String, + page: Int, + repository: service.RepositoryService.RepositoryInfo)(implicit context: app.Context) @import context._ @import view.helpers._ @html.main("Search Results", Some(repository)){ @@ -8,12 +12,14 @@ } else {

We've found @files.size code @plural(files.size, "result")

} - @files.map { file => + @files.drop((page - 1) * app.RepositorySearch.CodeLimit).take(app.RepositorySearch.CodeLimit).map { file =>
@file.path
Latest commit at @datetime(file.lastModified)
@Html(file.highlightText)
} + @helper.html.paginator(page, files.size, app.RepositorySearch.CodeLimit, 10, + s"${url(repository)}/search?q=${urlEncode(query)}&type=code") } } \ No newline at end of file diff --git a/src/main/twirl/search/issues.scala.html b/src/main/twirl/search/issues.scala.html index efdbc858a..95323da50 100644 --- a/src/main/twirl/search/issues.scala.html +++ b/src/main/twirl/search/issues.scala.html @@ -1,4 +1,8 @@ -@(issues: List[app.IssueSearchResult], fileCount: Int, query: String, repository: service.RepositoryService.RepositoryInfo)(implicit context: app.Context) +@(issues: List[app.IssueSearchResult], + fileCount: Int, + query: String, + page: Int, + repository: service.RepositoryService.RepositoryInfo)(implicit context: app.Context) @import context._ @import view.helpers._ @html.main("Search Results", Some(repository)){ @@ -8,10 +12,10 @@ } else {

We've found @issues.size code @plural(issues.size, "result")

} - @issues.map { issue => + @issues.drop((page - 1) * app.RepositorySearch.IssueLimit).take(app.RepositorySearch.IssueLimit).map { issue =>
#@issue.issueId
-

@issue.title

+

@issue.title

@Html(issue.highlightText)
Opened by @issue.openedUserName @@ -22,5 +26,7 @@
} + @helper.html.paginator(page, issues.size, app.RepositorySearch.IssueLimit, 10, + s"${url(repository)}/search?q=${urlEncode(query)}&type=issue") } } \ No newline at end of file From 133af935480d06a924db6cb842a88a47f41104ab Mon Sep 17 00:00:00 2001 From: takezoe Date: Fri, 19 Jul 2013 18:03:48 +0900 Subject: [PATCH 4/4] Don't use cache library immediately. --- project/build.scala | 2 - src/main/scala/app/IndexController.scala | 105 +++++++++++------------ 2 files changed, 48 insertions(+), 59 deletions(-) diff --git a/project/build.scala b/project/build.scala index 8450203b5..774c0dd63 100644 --- a/project/build.scala +++ b/project/build.scala @@ -33,8 +33,6 @@ object MyBuild extends Build { "org.apache.commons" % "commons-compress" % "1.5", "com.typesafe.slick" %% "slick" % "1.0.1", "com.h2database" % "h2" % "1.3.171", - "com.google.guava" % "guava" % "14.0.1", - "com.google.code.findbugs" % "jsr305" % "2.0.1", "ch.qos.logback" % "logback-classic" % "1.0.6" % "runtime", "org.eclipse.jetty" % "jetty-webapp" % "8.1.8.v20121106" % "container", "org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" % "container;provided;test" artifacts (Artifact("javax.servlet", "jar", "jar")) diff --git a/src/main/scala/app/IndexController.scala b/src/main/scala/app/IndexController.scala index e2c06425f..b865c3667 100644 --- a/src/main/scala/app/IndexController.scala +++ b/src/main/scala/app/IndexController.scala @@ -8,16 +8,14 @@ import org.eclipse.jgit.treewalk.TreeWalk import org.eclipse.jgit.revwalk.RevWalk import scala.collection.mutable.ListBuffer import org.eclipse.jgit.lib.FileMode -import com.google.common.cache.{CacheLoader, CacheBuilder} -import java.util.concurrent.TimeUnit import model.Issue class IndexController extends IndexControllerBase - with RepositoryService with AccountService with SystemSettingsService with ActivityService + with RepositoryService with AccountService with SystemSettingsService with ActivityService with IssuesService with ReferrerAuthenticator trait IndexControllerBase extends ControllerBase { self: RepositoryService - with SystemSettingsService with ActivityService + with SystemSettingsService with ActivityService with IssuesService with ReferrerAuthenticator => val searchForm = mapping( @@ -54,7 +52,7 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService } - val SearchResult(files, issues) = cache.get(repository.owner, repository.name, query) + val SearchResult(files, issues) = searchRepository(repository.owner, repository.name, query) target.toLowerCase match { case "issue" => @@ -83,6 +81,51 @@ trait IndexControllerBase extends ControllerBase { self: RepositoryService } }) + case class SearchResult( + files: List[(String, String)], + issues: List[(Issue, Int, String)] + ) + + def searchRepository(owner: String, repository: String, query: String): SearchResult = { + val issues = if(query.isEmpty) Nil else searchIssuesByKeyword(owner, repository, query) + val files = if(query.isEmpty) Nil else searchRepositoryFiles(owner, repository, query) + SearchResult(files, issues) + } + + private def searchRepositoryFiles(owner: String, repository: String, query: String): List[(String, String)] = { + JGitUtil.withGit(getRepositoryDir(owner, repository)){ git => + val revWalk = new RevWalk(git.getRepository) + val objectId = git.getRepository.resolve("HEAD") + val revCommit = revWalk.parseCommit(objectId) + val treeWalk = new TreeWalk(git.getRepository) + treeWalk.setRecursive(true) + treeWalk.addTree(revCommit.getTree) + + val keywords = StringUtil.splitWords(query.toLowerCase) + val list = new ListBuffer[(String, String)] + + while (treeWalk.next()) { + if(treeWalk.getFileMode(0) != FileMode.TREE){ + JGitUtil.getContent(git, treeWalk.getObjectId(0), false).foreach { bytes => + if(FileUtil.isText(bytes)){ + val text = new String(bytes, "UTF-8") + val lowerText = text.toLowerCase + val indices = keywords.map(lowerText.indexOf _) + if(!indices.exists(_ < 0)){ + list.append((treeWalk.getPathString, text)) + } + } + } + } + } + treeWalk.release + revWalk.release + + list.toList + } + } + + private def getHighlightText(content: String, query: String): (String, Int) = { val keywords = StringUtil.splitWords(query.toLowerCase) val lowerText = content.toLowerCase @@ -116,58 +159,6 @@ case class FileSearchResult( highlightLineNumber: Int) object RepositorySearch extends IssuesService { - val CodeLimit = 10 val IssueLimit = 10 - - case class SearchResult( - files: List[(String, String)], - issues: List[(Issue, Int, String)] - ) - - val cache = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterWrite(10, TimeUnit.MINUTES) - .build( - new CacheLoader[(String, String, String), SearchResult]() { - override def load(key: (String, String, String)) = { - val (owner, repository, query) = key - val issues = if(query.isEmpty) Nil else searchIssuesByKeyword(owner, repository, query) - val files = if(query.isEmpty) Nil else searchRepositoryFiles(owner, repository, query) - SearchResult(files, issues) - } - }) - - private def searchRepositoryFiles(owner: String, repository: String, query: String): List[(String, String)] = { - JGitUtil.withGit(getRepositoryDir(owner, repository)){ git => - val revWalk = new RevWalk(git.getRepository) - val objectId = git.getRepository.resolve("HEAD") - val revCommit = revWalk.parseCommit(objectId) - val treeWalk = new TreeWalk(git.getRepository) - treeWalk.setRecursive(true) - treeWalk.addTree(revCommit.getTree) - - val keywords = StringUtil.splitWords(query.toLowerCase) - val list = new ListBuffer[(String, String)] - - while (treeWalk.next()) { - if(treeWalk.getFileMode(0) != FileMode.TREE){ - JGitUtil.getContent(git, treeWalk.getObjectId(0), false).foreach { bytes => - if(FileUtil.isText(bytes)){ - val text = new String(bytes, "UTF-8") - val lowerText = text.toLowerCase - val indices = keywords.map(lowerText.indexOf _) - if(!indices.exists(_ < 0)){ - list.append((treeWalk.getPathString, text)) - } - } - } - } - } - treeWalk.release - revWalk.release - - list.toList - } - } } \ No newline at end of file