Merge pull request #912 from takezoe/markedj

Switch markdown processor to markedj from pegdown

.travis.yml

@@ -2,3 +2,5 @@ language: scala
 sudo: false
 script:
   - sbt test
+jdk:
+  - oraclejdk8

project/build.scala

@@ -38,7 +38,8 @@ object MyBuild extends Build {
     scalaVersion := ScalaVersion,
     resolvers ++= Seq(
       Classpaths.typesafeReleases,
-      "amateras-repo" at "http://amateras.sourceforge.jp/mvn/"
+      "amateras-repo" at "http://amateras.sourceforge.jp/mvn/",
+      "amateras-snapshot-repo" at "http://amateras.sourceforge.jp/mvn-snapshot/"
     ),
     scalacOptions := Seq("-deprecation", "-language:postfixOps"),
     libraryDependencies ++= Seq(
@@ -50,7 +51,7 @@ object MyBuild extends Build {
       "org.json4s" %% "json4s-jackson" % "3.2.11",
       "jp.sf.amateras" %% "scalatra-forms" % "0.1.0",
       "commons-io" % "commons-io" % "2.4",
-      "org.pegdown" % "pegdown" % "1.5.0",
+      "io.github.gitbucket" % "markedj" % "1.0.1",
       "org.apache.commons" % "commons-compress" % "1.9",
       "org.apache.commons" % "commons-email" % "1.3.3",
       "org.apache.httpcomponents" % "httpclient" % "4.3.6",

src/main/scala/gitbucket/core/controller/IssuesController.scala

@@ -233,7 +233,7 @@ trait IssuesControllerBase extends ControllerBase {
           org.json4s.jackson.Serialization.write(
               Map("title"   -> x.title,
                   "content" -> Markdown.toHtml(x.content getOrElse "No description given.",
-                      repository, false, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName))
+                      repository, false, true, true, true, isEditable(x.userName, x.repositoryName, x.openedUserName))
               ))
         }
       } else Unauthorized

src/main/scala/gitbucket/core/view/Markdown.scala

@@ -1,18 +1,14 @@
 package gitbucket.core.view
 
 import java.text.Normalizer
-import java.util.Locale
 import java.util.regex.Pattern
+import java.util.{Optional, Locale}
 
 import gitbucket.core.controller.Context
-import gitbucket.core.service.{RepositoryService, RequestCache, WikiService}
+import gitbucket.core.service.{RepositoryService, RequestCache}
 import gitbucket.core.util.StringUtil
-import org.parboiled.common.StringUtils
-import org.pegdown.LinkRenderer.Rendering
-import org.pegdown._
-import org.pegdown.ast._
-
-import scala.collection.JavaConverters._
+import io.github.gitbucket.markedj._
+import io.github.gitbucket.markedj.Utils._
 
 object Markdown {
 
@@ -24,7 +20,7 @@ object Markdown {
    * @param enableRefsLink if true then issue reference (e.g. #123) is rendered as link
    * @param enableAnchor if true then anchor for headline is generated
    * @param enableTaskList if true then task list syntax is available
-   * @param hasWritePermission
+   * @param hasWritePermission true if user has writable to ths given repository
    * @param pages the list of existing Wiki pages
    */
   def toHtml(markdown: String,
@@ -35,7 +31,6 @@ object Markdown {
              enableTaskList: Boolean = false,
              hasWritePermission: Boolean = false,
              pages: List[String] = Nil)(implicit context: Context): String = {
-
     // escape issue id
     val s = if(enableRefsLink){
       markdown.replaceAll("(?<=(\\W|^))#(\\d+)(?=(\\W|$))", "issue:$2")
@@ -43,252 +38,145 @@ object Markdown {
 
     // escape task list
     val source = if(enableTaskList){
-      GitBucketHtmlSerializer.escapeTaskList(s)
+      escapeTaskList(s)
     } else s
 
-    val rootNode = new PegDownProcessor(
-      Extensions.AUTOLINKS | Extensions.WIKILINKS | Extensions.FENCED_CODE_BLOCKS |
-        Extensions.TABLES | Extensions.HARDWRAPS | Extensions.SUPPRESS_ALL_HTML | Extensions.STRIKETHROUGH
-    ).parseMarkdown(source.toCharArray)
-
-    new GitBucketHtmlSerializer(
-      markdown, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList,
-      hasWritePermission, pages
-    ).toHtml(rootNode)
+    val options = new Options()
+    val renderer = new GitBucketMarkedRenderer(options, repository, enableWikiLink, enableRefsLink, enableAnchor, enableTaskList, hasWritePermission, pages)
+    Marked.marked(source, options, renderer)
   }
-}
 
-class GitBucketLinkRender(
-    context: Context,
-    repository: RepositoryService.RepositoryInfo,
-    enableWikiLink: Boolean,
-    pages: List[String]) extends LinkRenderer with WikiService {
+  /**
+   * Extends markedj Renderer for GitBucket
+   */
+  class GitBucketMarkedRenderer(options: Options, repository: RepositoryService.RepositoryInfo,
+                                enableWikiLink: Boolean, enableRefsLink: Boolean, enableAnchor: Boolean, enableTaskList: Boolean, hasWritePermission: Boolean,
+                                pages: List[String])
+                               (implicit val context: Context) extends Renderer(options) with LinkConverter with RequestCache {
 
-  override def render(node: WikiLinkNode): Rendering = {
-    if(enableWikiLink){
-      try {
-        val text = node.getText
-        val (label, page) = if(text.contains('|')){
-          val i = text.indexOf('|')
-          (text.substring(0, i), text.substring(i + 1))
+    override  def heading(text: String, level: Int, raw: String): String = {
+      val id = generateAnchorName(text)
+      val out = new StringBuilder()
+
+      out.append("<h" + level + " id=\"" + options.getHeaderPrefix + id + "\" class=\"markdown-head\">")
+
+      if(enableAnchor){
+        out.append("<a class=\"markdown-anchor-link\" href=\"#" + id + "\"></a>")
+        out.append("<a class=\"markdown-anchor\" name=\"" + id + "\"></a>")
+      }
+
+      out.append(text)
+      out.append("</h" + level + ">\n")
+      out.toString()
+    }
+
+    override def code(code: String, lang: Optional[String], escaped: Boolean): String = {
+      "<pre class=\"prettyprint" + (if(lang.isPresent) s" ${options.getLangPrefix}${lang.get}" else "" )+ "\">" +
+        (if(escaped) code else escape(code, true)) + "</pre>"
+    }
+
+    override def list(body: String, ordered: Boolean): String = {
+      var listType: String = null
+      if (ordered) {
+        listType = "ol"
+      }
+      else {
+        listType = "ul"
+      }
+      if(body.contains("""class="task-list-item-checkbox"""")){
+        return "<" + listType + " class=\"task-list\">\n" + body + "</" + listType + ">\n"
+      } else {
+        return "<" + listType + ">\n" + body + "</" + listType + ">\n"
+      }
+    }
+
+    override def listitem(text: String): String = {
+      if(text.contains("""class="task-list-item-checkbox" """)){
+        return "<li class=\"task-list-item\">" + text + "</li>\n"
+      } else {
+        return "<li>" + text + "</li>\n"
+      }
+    }
+
+    override def text(text: String): String = {
+      // convert commit id and username to link.
+      val t1 = if(enableRefsLink) convertRefsLinks(text, repository, "issue:") else text
+
+      // convert task list to checkbox.
+      val t2 = if(enableTaskList) convertCheckBox(t1, hasWritePermission) else t1
+
+      t2
+    }
+
+    override def link(href: String, title: Optional[String], text: String): String = {
+      super.link(fixUrl(href, true), title, text)
+    }
+
+    override def image(href: String, title: Optional[String], text: String): String = {
+      super.image(fixUrl(href, true), title, text)
+    }
+
+    override def nolink(text: String): String = {
+      if(enableWikiLink && text.startsWith("[[") && text.endsWith("]]")){
+        val link = text.replaceAll("(^\\[\\[|\\]\\]$)", "")
+
+        val (label, page) = if(link.contains('|')){
+          val i = link.indexOf('|')
+          (link.substring(0, i), link.substring(i + 1))
         } else {
-          (text, text)
+          (link, link)
         }
 
         val url = repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/" + StringUtil.urlEncode(page)
-
         if(pages.contains(page)){
-          new Rendering(url, label)
+          "<a href=\"" + url + "\">" + escape(label) + "</a>"
         } else {
-          new Rendering(url, label).withAttribute("class", "absent")
+          "<a href=\"" + url + "\" class=\"absent\">" + escape(label) + "</a>"
         }
-      } catch {
-        case e: java.io.UnsupportedEncodingException => throw new IllegalStateException
-      }
-    } else {
-      super.render(node)
-    }
-  }
-}
-
-class GitBucketVerbatimSerializer extends VerbatimSerializer {
-  def serialize(node: VerbatimNode, printer: Printer): Unit = {
-    printer.println.print("<pre")
-    if (!StringUtils.isEmpty(node.getType)) {
-      printer.print(" class=").print('"').print("prettyprint ").print(node.getType).print('"')
-    }
-    printer.print(">")
-    var text: String = node.getText
-    while (text.charAt(0) == '\n') {
-      printer.print("<br/>")
-      text = text.substring(1)
-    }
-    printer.printEncoded(text)
-    printer.print("</pre>")
-  }
-}
-
-class GitBucketHtmlSerializer(
-    markdown: String,
-    repository: RepositoryService.RepositoryInfo,
-    enableWikiLink: Boolean,
-    enableRefsLink: Boolean,
-    enableAnchor: Boolean,
-    enableTaskList: Boolean,
-    hasWritePermission: Boolean,
-    pages: List[String]
-  )(implicit val context: Context) extends ToHtmlSerializer(
-    new GitBucketLinkRender(context, repository, enableWikiLink, pages),
-    Map[String, VerbatimSerializer](VerbatimSerializer.DEFAULT -> new GitBucketVerbatimSerializer).asJava
-  ) with LinkConverter with RequestCache {
-
-  override protected def printImageTag(rendering: LinkRenderer.Rendering): Unit = {
-    printer.print("<a target=\"_blank\" href=\"").print(fixUrl(rendering.href, true)).print("\">")
-      .print("<img src=\"").print(fixUrl(rendering.href, true)).print("\"  alt=\"").printEncoded(rendering.text).print("\"/></a>")
-  }
-
-  override protected def printLink(rendering: LinkRenderer.Rendering): Unit = {
-    printer.print('<').print('a')
-    printAttribute("href", fixUrl(rendering.href))
-    for (attr <- rendering.attributes.asScala) {
-      printAttribute(attr.name, attr.value)
-    }
-    printer.print('>').print(rendering.text).print("</a>")
-  }
-
-  private def fixUrl(url: String, isImage: Boolean = false): String = {
-    if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){
-      url
-    } else if(url.startsWith("#")){
-      ("#" + GitBucketHtmlSerializer.generateAnchorName(url.substring(1)))
-    } else if(!enableWikiLink){
-      if(context.currentPath.contains("/blob/")){
-        url + (if(isImage) "?raw=true" else "")
-      } else if(context.currentPath.contains("/tree/")){
-        val paths = context.currentPath.split("/")
-        val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch
-        repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "")
       } else {
-        val paths = context.currentPath.split("/")
-        val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch
-        repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "")
+        escape(text)
       }
-    } else {
-      repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url
-    }
-  }
-
-  private def printAttribute(name: String, value: String): Unit = {
-    printer.print(' ').print(name).print('=').print('"').print(value).print('"')
-  }
-
-  private def printHeaderTag(node: HeaderNode): Unit = {
-    val tag = s"h${node.getLevel}"
-    val child = node.getChildren.asScala.headOption
-    val anchorName = child match {
-      case Some(x: AnchorLinkNode) => x.getName
-      case Some(x: TextNode)       => x.getText
-      case _ => GitBucketHtmlSerializer.generateAnchorName(extractText(node)) // TODO
     }
 
-    printer.print(s"""<$tag class="markdown-head">""")
-    if(enableAnchor){
-      printer.print(s"""<a class="markdown-anchor-link" href="#$anchorName"></a>""")
-      printer.print(s"""<a class="markdown-anchor" name="$anchorName"></a>""")
+    private def fixUrl(url: String, isImage: Boolean = false): String = {
+      if(url.startsWith("http://") || url.startsWith("https://") || url.startsWith("/")){
+        url
+      } else if(url.startsWith("#")){
+        ("#" + generateAnchorName(url.substring(1)))
+      } else if(!enableWikiLink){
+        if(context.currentPath.contains("/blob/")){
+          url + (if(isImage) "?raw=true" else "")
+        } else if(context.currentPath.contains("/tree/")){
+          val paths = context.currentPath.split("/")
+          val branch = if(paths.length > 3) paths.drop(4).mkString("/") else repository.repository.defaultBranch
+          repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "")
+        } else {
+          val paths = context.currentPath.split("/")
+          val branch = if(paths.length > 3) paths.last else repository.repository.defaultBranch
+          repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/blob/" + branch + "/" + url + (if(isImage) "?raw=true" else "")
+        }
+      } else {
+        repository.httpUrl.replaceFirst("/git/", "/").stripSuffix(".git") + "/wiki/_blob/" + url
+      }
     }
-    child match {
-      case Some(x: AnchorLinkNode) => printer.print(x.getText)
-      case _ => visitChildren(node)
-    }
-    printer.print(s"</$tag>")
-  }
 
-  private def extractText(node: Node): String = {
-    val sb = new StringBuilder()
-    node.getChildren.asScala.map {
-      case x: TextNode => sb.append(x.getText)
-      case x: Node => sb.append(extractText(x))
-    }
-    sb.toString()
-  }
-
-  override def visit(node: HeaderNode): Unit = {
-    printHeaderTag(node)
-  }
-
-  override def visit(node: TextNode): Unit =  {
-    // convert commit id and username to link.
-    val t = if(enableRefsLink) convertRefsLinks(node.getText, repository, "issue:") else node.getText
-
-    // convert task list to checkbox.
-    val text = if(enableTaskList) GitBucketHtmlSerializer.convertCheckBox(t, hasWritePermission) else t
-
-    if (abbreviations.isEmpty) {
-      printer.print(text)
-    } else {
-      printWithAbbreviations(text)
-    }
-  }
-
-  override def visit(node: VerbatimNode) {
-    val printer = new Printer()
-    val serializer = verbatimSerializers.get(VerbatimSerializer.DEFAULT)
-    serializer.serialize(node, printer)
-    val html = printer.getString
-
-    // convert commit id and username to link.
-    val t = if(enableRefsLink) convertRefsLinks(html, repository, "issue:", escapeHtml = false) else html
-
-    this.printer.print(t)
-  }
-
-  override def visit(node: BulletListNode): Unit = {
-    if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) {
-      printer.println().print("""<ul class="task-list">""").indent(+2)
-      visitChildren(node)
-      printer.indent(-2).println().print("</ul>")
-    } else {
-      printIndentedTag(node, "ul")
-    }
-  }
-
-  override def visit(node: ListItemNode): Unit = {
-    if (printChildrenToString(node).contains("""class="task-list-item-checkbox" """)) {
-      printer.println()
-      printer.print("""<li class="task-list-item">""")
-      visitChildren(node)
-      printer.print("</li>")
-    } else {
-      printer.println()
-      printTag(node, "li")
-    }
-  }
-
-  override def visit(node: ExpLinkNode) {
-    printLink(linkRenderer.render(node, printLinkChildrenToString(node)))
-  }
-
-  def printLinkChildrenToString(node: SuperNode) = {
-    val priorPrinter = printer
-    printer = new Printer()
-    visitLinkChildren(node)
-    val result = printer.getString()
-    printer = priorPrinter
-    result
-  }
-
-  def visitLinkChildren(node: SuperNode) {
-    import scala.collection.JavaConversions._
-    node.getChildren.foreach(child => child match {
-      case node: ExpImageNode => visitLinkChild(node)
-      case node: SuperNode => visitLinkChildren(node)
-      case _ => child.accept(this)
-    })
-  }
-
-  def visitLinkChild(node: ExpImageNode) {
-    printer.print("<img src=\"").print(fixUrl(node.url, true)).print("\"  alt=\"").printEncoded(printChildrenToString(node)).print("\"/>")
-  }
-}
-
-object GitBucketHtmlSerializer {
-
-  private val Whitespace = "[\\s]".r
-
-  def generateAnchorName(text: String): String = {
-    val noWhitespace = Whitespace.replaceAllIn(text, "-")
-    val normalized = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD)
-    val noSpecialChars = StringUtil.urlEncode(normalized)
-    noSpecialChars.toLowerCase(Locale.ENGLISH)
   }
 
   def escapeTaskList(text: String): String = {
     Pattern.compile("""^( *)- \[([x| ])\] """, Pattern.MULTILINE).matcher(text).replaceAll("$1* task:$2: ")
   }
 
+  def generateAnchorName(text: String): String = {
+    val normalized = Normalizer.normalize(text.replaceAll("<.*>", "").replaceAll("[\\s]", "-"), Normalizer.Form.NFD)
+    val encoded    = StringUtil.urlEncode(normalized)
+    encoded.toLowerCase(Locale.ENGLISH)
+  }
+
   def convertCheckBox(text: String, hasWritePermission: Boolean): String = {
     val disabled = if (hasWritePermission) "" else "disabled"
     text.replaceAll("task:x:", """<input type="checkbox" class="task-list-item-checkbox" checked="checked" """ + disabled + "/>")
-        .replaceAll("task: :", """<input type="checkbox" class="task-list-item-checkbox" """ + disabled + "/>")
+      .replaceAll("task: :", """<input type="checkbox" class="task-list-item-checkbox" """ + disabled + "/>")
   }
+
 }
+

src/test/scala/gitbucket/core/view/MarkdownSpec.scala

@@ -2,9 +2,9 @@ package gitbucket.core.view
 
 import org.specs2.mutable._
 
-class GitBucketHtmlSerializerSpec extends Specification {
+class MarkdownSpec extends Specification {
 
-  import GitBucketHtmlSerializer._
+  import Markdown._
 
   "generateAnchorName" should {
     "convert whitespace characters to hyphens" in {