diff --git a/.travis.yml b/.travis.yml index 6caeabf9c..b85cad5d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,3 +2,5 @@ language: scala sudo: false script: - sbt test +jdk: + - oraclejdk8 diff --git a/project/build.scala b/project/build.scala index 6249a92f9..f8e111efe 100644 --- a/project/build.scala +++ b/project/build.scala @@ -38,7 +38,8 @@ object MyBuild extends Build { scalaVersion := ScalaVersion, resolvers ++= Seq( Classpaths.typesafeReleases, - "amateras-repo" at "http://amateras.sourceforge.jp/mvn/" + "amateras-repo" at "http://amateras.sourceforge.jp/mvn/", + "amateras-snapshot-repo" at "http://amateras.sourceforge.jp/mvn-snapshot/" ), scalacOptions := Seq("-deprecation", "-language:postfixOps"), libraryDependencies ++= Seq( @@ -50,7 +51,7 @@ object MyBuild extends Build { "org.json4s" %% "json4s-jackson" % "3.2.11", "jp.sf.amateras" %% "scalatra-forms" % "0.1.0", "commons-io" % "commons-io" % "2.4", - "org.pegdown" % "pegdown" % "1.5.0", + "io.github.gitbucket" % "markedj" % "1.0.1", "org.apache.commons" % "commons-compress" % "1.9", "org.apache.commons" % "commons-email" % "1.3.3", "org.apache.httpcomponents" % "httpclient" % "4.3.6", diff --git a/src/main/scala/gitbucket/core/controller/IssuesController.scala b/src/main/scala/gitbucket/core/controller/IssuesController.scala index 8f2bfafb1..1bee1bf3d 100644 --- a/src/main/scala/gitbucket/core/controller/IssuesController.scala +++ b/src/main/scala/gitbucket/core/controller/IssuesController.scala @@ -233,7 +233,7 @@ trait IssuesControllerBase extends ControllerBase { org.json4s.jackson.Serialization.write( Map("title" -> x.title, "content" -> Markdown.toHtml(x.content getOrElse "No description given.", - repository, false, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName)) + repository, false, true, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName)) )) } } else Unauthorized diff --git a/src/main/scala/gitbucket/core/view/Markdown.scala b/src/main/scala/gitbucket/core/view/Markdown.scala index 3a80b8f36..2e126eb6c 100644 --- a/src/main/scala/gitbucket/core/view/Markdown.scala +++ b/src/main/scala/gitbucket/core/view/Markdown.scala @@ -1,18 +1,14 @@ package gitbucket.core.view import java.text.Normalizer -import java.util.Locale import java.util.regex.Pattern +import java.util.{Optional, Locale} import gitbucket.core.controller.Context -import gitbucket.core.service.{RepositoryService, RequestCache, WikiService} +import gitbucket.core.service.{RepositoryService, RequestCache} import gitbucket.core.util.StringUtil -import org.parboiled.common.StringUtils -import org.pegdown.LinkRenderer.Rendering -import org.pegdown._ -import org.pegdown.ast._ - -import scala.collection.JavaConverters._ +import io.github.gitbucket.markedj._ +import io.github.gitbucket.markedj.Utils._ object Markdown { @@ -24,7 +20,7 @@ object Markdown { * @param enableRefsLink if true then issue reference (e.g. #123) is rendered as link * @param enableAnchor if true then anchor for headline is generated * @param enableTaskList if true then task list syntax is available - * @param hasWritePermission + * @param hasWritePermission true if user has writable to ths given repository * @param pages the list of existing Wiki pages */ def toHtml(markdown: String, @@ -35,7 +31,6 @@ object Markdown { enableTaskList: Boolean = false, hasWritePermission: Boolean = false, pages: List[String] = Nil)(implicit context: Context): String = { - // escape issue id val s = if(enableRefsLink){ markdown.replaceAll("(?<=(\\W|^))#(\\d+)(?=(\\W|$))", "issue:$2") @@ -43,252 +38,145 @@ object Markdown { // escape task list val source = if(enableTaskList){ - GitBucketHtmlSerializer.escapeTaskList(s) + escapeTaskList(s) } else s - val rootNode = new PegDownProcessor( - Extensions.AUTOLINKS | Extensions.WIKILINKS | Extensions.FENCED_CODE_BLOCKS | - Extensions.TABLES | Extensions.HARDWRAPS | Extensions.SUPPRESS_ALL_HTML | Extensions.STRIKETHROUGH - ).parseMarkdown(source.toCharArray) - - new GitBucketHtmlSerializer( - markdown, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList, - hasWritePermission, pages - ).toHtml(rootNode) + val options = new Options() + val renderer = new GitBucketMarkedRenderer(options, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList, hasWritePermission, pages) + Marked.marked(source, options, renderer) } -} -class GitBucketLinkRender( - context: Context, - repository: RepositoryService.RepositoryInfo, - enableWikiLink: Boolean, - pages: List[String]) extends LinkRenderer with WikiService { + /** + * Extends markedj Renderer for GitBucket + */ + class GitBucketMarkedRenderer(options: Options, repository: RepositoryService.RepositoryInfo, + enableWikiLink: Boolean, enableRefsLink: Boolean, enableAnchor: Boolean, enableTaskList: Boolean, hasWritePermission: Boolean, + pages: List[String]) + (implicit val context: Context) extends Renderer(options) with LinkConverter with RequestCache { - override def render(node: WikiLinkNode): Rendering = { - if(enableWikiLink){ - try { - val text = node.getText - val (label, page) = if(text.contains('|')){ - val i = text.indexOf('|') - (text.substring(0, i), text.substring(i + 1)) + override def heading(text: String, level: Int, raw: String): String = { + val id = generateAnchorName(text) + val out = new StringBuilder() + + out.append("") + + if(enableAnchor){ + out.append("") + out.append("") + } + + out.append(text) + out.append("\n") + out.toString() + } + + override def code(code: String, lang: Optional[String], escaped: Boolean): String = { + "
" +
+        (if(escaped) code else escape(code, true)) + "
" + } + + override def list(body: String, ordered: Boolean): String = { + var listType: String = null + if (ordered) { + listType = "ol" + } + else { + listType = "ul" + } + if(body.contains("""class="task-list-item-checkbox"""")){ + return "<" + listType + " class=\"task-list\">\n" + body + "\n" + } else { + return "<" + listType + ">\n" + body + "\n" + } + } + + override def listitem(text: String): String = { + if(text.contains("""class="task-list-item-checkbox" """)){ + return "
  • " + text + "
  • \n" + } else { + return "
  • " + text + "
  • \n" + } + } + + override def text(text: String): String = { + // convert commit id and username to link. + val t1 = if(enableRefsLink) convertRefsLinks(text, repository, "issue:") else text + + // convert task list to checkbox. + val t2 = if(enableTaskList) convertCheckBox(t1, hasWritePermission) else t1 + + t2 + } + + override def link(href: String, title: Optional[String], text: String): String = { + super.link(fixUrl(href, true), title, text) + } + + override def image(href: String, title: Optional[String], text: String): String = { + super.image(fixUrl(href, true), title, text) + } + + override def nolink(text: String): String = { + if(enableWikiLink && text.startsWith("[[") && text.endsWith("]]")){ + val link = text.replaceAll("(^\\[\\[|\\]\\]$)", "") + + val (label, page) = if(link.contains('|')){ + val i = link.indexOf('|') + (link.substring(0, i), link.substring(i + 1)) } else { - (text, text) + (link, link) } val url = repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/" + StringUtil.urlEncode(page) - if(pages.contains(page)){ - new Rendering(url, label) + "" + escape(label) + "" } else { - new Rendering(url, label).withAttribute("class", "absent") + "" + escape(label) + "" } - } catch { - case e: java.io.UnsupportedEncodingException => throw new IllegalStateException - } - } else { - super.render(node) - } - } -} - -class GitBucketVerbatimSerializer extends VerbatimSerializer { - def serialize(node: VerbatimNode, printer: Printer): Unit = { - printer.println.print("") - var text: String = node.getText - while (text.charAt(0) == '\n') { - printer.print("
    ") - text = text.substring(1) - } - printer.printEncoded(text) - printer.print("") - } -} - -class GitBucketHtmlSerializer( - markdown: String, - repository: RepositoryService.RepositoryInfo, - enableWikiLink: Boolean, - enableRefsLink: Boolean, - enableAnchor: Boolean, - enableTaskList: Boolean, - hasWritePermission: Boolean, - pages: List[String] - )(implicit val context: Context) extends ToHtmlSerializer( - new GitBucketLinkRender(context, repository, enableWikiLink, pages), - Map[String, VerbatimSerializer](VerbatimSerializer.DEFAULT -> new GitBucketVerbatimSerializer).asJava - ) with LinkConverter with RequestCache { - - override protected def printImageTag(rendering: LinkRenderer.Rendering): Unit = { - printer.print("") - .print("\"").printEncoded(rendering.text).print("\"/") - } - - override protected def printLink(rendering: LinkRenderer.Rendering): Unit = { - printer.print('<').print('a') - printAttribute("href", fixUrl(rendering.href)) - for (attr <- rendering.attributes.asScala) { - printAttribute(attr.name, attr.value) - } - printer.print('>').print(rendering.text).print("") - } - - private def fixUrl(url: String, isImage: Boolean = false): String = { - if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){ - url - } else if(url.startsWith("#")){ - ("#" + GitBucketHtmlSerializer.generateAnchorName(url.substring(1))) - } else if(!enableWikiLink){ - if(context.currentPath.contains("/blob/")){ - url + (if(isImage) "?raw=true" else "") - } else if(context.currentPath.contains("/tree/")){ - val paths = context.currentPath.split("/") - val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") } else { - val paths = context.currentPath.split("/") - val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + escape(text) } - } else { - repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url - } - } - - private def printAttribute(name: String, value: String): Unit = { - printer.print(' ').print(name).print('=').print('"').print(value).print('"') - } - - private def printHeaderTag(node: HeaderNode): Unit = { - val tag = s"h${node.getLevel}" - val child = node.getChildren.asScala.headOption - val anchorName = child match { - case Some(x: AnchorLinkNode) => x.getName - case Some(x: TextNode) => x.getText - case _ => GitBucketHtmlSerializer.generateAnchorName(extractText(node)) // TODO } - printer.print(s"""<$tag class="markdown-head">""") - if(enableAnchor){ - printer.print(s"""""") - printer.print(s"""""") + private def fixUrl(url: String, isImage: Boolean = false): String = { + if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){ + url + } else if(url.startsWith("#")){ + ("#" + generateAnchorName(url.substring(1))) + } else if(!enableWikiLink){ + if(context.currentPath.contains("/blob/")){ + url + (if(isImage) "?raw=true" else "") + } else if(context.currentPath.contains("/tree/")){ + val paths = context.currentPath.split("/") + val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + } else { + val paths = context.currentPath.split("/") + val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "") + } + } else { + repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url + } } - child match { - case Some(x: AnchorLinkNode) => printer.print(x.getText) - case _ => visitChildren(node) - } - printer.print(s"") - } - private def extractText(node: Node): String = { - val sb = new StringBuilder() - node.getChildren.asScala.map { - case x: TextNode => sb.append(x.getText) - case x: Node => sb.append(extractText(x)) - } - sb.toString() - } - - override def visit(node: HeaderNode): Unit = { - printHeaderTag(node) - } - - override def visit(node: TextNode): Unit = { - // convert commit id and username to link. - val t = if(enableRefsLink) convertRefsLinks(node.getText, repository, "issue:") else node.getText - - // convert task list to checkbox. - val text = if(enableTaskList) GitBucketHtmlSerializer.convertCheckBox(t, hasWritePermission) else t - - if (abbreviations.isEmpty) { - printer.print(text) - } else { - printWithAbbreviations(text) - } - } - - override def visit(node: VerbatimNode) { - val printer = new Printer() - val serializer = verbatimSerializers.get(VerbatimSerializer.DEFAULT) - serializer.serialize(node, printer) - val html = printer.getString - - // convert commit id and username to link. - val t = if(enableRefsLink) convertRefsLinks(html, repository, "issue:", escapeHtml = false) else html - - this.printer.print(t) - } - - override def visit(node: BulletListNode): Unit = { - if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) { - printer.println().print("""") - } else { - printIndentedTag(node, "ul") - } - } - - override def visit(node: ListItemNode): Unit = { - if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) { - printer.println() - printer.print("""
  • """) - visitChildren(node) - printer.print("
  • ") - } else { - printer.println() - printTag(node, "li") - } - } - - override def visit(node: ExpLinkNode) { - printLink(linkRenderer.render(node, printLinkChildrenToString(node))) - } - - def printLinkChildrenToString(node: SuperNode) = { - val priorPrinter = printer - printer = new Printer() - visitLinkChildren(node) - val result = printer.getString() - printer = priorPrinter - result - } - - def visitLinkChildren(node: SuperNode) { - import scala.collection.JavaConversions._ - node.getChildren.foreach(child => child match { - case node: ExpImageNode => visitLinkChild(node) - case node: SuperNode => visitLinkChildren(node) - case _ => child.accept(this) - }) - } - - def visitLinkChild(node: ExpImageNode) { - printer.print("\"").printEncoded(printChildrenToString(node)).print("\"/") - } -} - -object GitBucketHtmlSerializer { - - private val Whitespace = "[\\s]".r - - def generateAnchorName(text: String): String = { - val noWhitespace = Whitespace.replaceAllIn(text, "-") - val normalized = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD) - val noSpecialChars = StringUtil.urlEncode(normalized) - noSpecialChars.toLowerCase(Locale.ENGLISH) } def escapeTaskList(text: String): String = { Pattern.compile("""^( *)- \[([x| ])\] """, Pattern.MULTILINE).matcher(text).replaceAll("$1* task:$2: ") } + def generateAnchorName(text: String): String = { + val normalized = Normalizer.normalize(text.replaceAll("<.*>", "").replaceAll("[\\s]", "-"), Normalizer.Form.NFD) + val encoded = StringUtil.urlEncode(normalized) + encoded.toLowerCase(Locale.ENGLISH) + } + def convertCheckBox(text: String, hasWritePermission: Boolean): String = { val disabled = if (hasWritePermission) "" else "disabled" text.replaceAll("task:x:", """") - .replaceAll("task: :", """") + .replaceAll("task: :", """") } + } + diff --git a/src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala b/src/test/scala/gitbucket/core/view/MarkdownSpec.scala similarity index 95% rename from src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala rename to src/test/scala/gitbucket/core/view/MarkdownSpec.scala index 82fb3d958..a2ec2e9c4 100644 --- a/src/test/scala/gitbucket/core/view/GitBucketHtmlSerializerSpec.scala +++ b/src/test/scala/gitbucket/core/view/MarkdownSpec.scala @@ -2,9 +2,9 @@ package gitbucket.core.view import org.specs2.mutable._ -class GitBucketHtmlSerializerSpec extends Specification { +class MarkdownSpec extends Specification { - import GitBucketHtmlSerializer._ + import Markdown._ "generateAnchorName" should { "convert whitespace characters to hyphens" in {