Use more accurate language detection for syntax highlighting (#1891)

Updated spotter to version 4 in order to get prism syntax mode for detected coding languages.
Expose syntax modes of coding languages as headers on content endpoint and as fields on diff dto.
Remove leading line break on search result fragments.
Use mark instead of span or strong for highlighted search results.
Add option to use syntax highlighting in TextHitField component.

Co-authored-by: Matthias Thieroff <matthias.thieroff@cloudogu.com>

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java

@@ -54,6 +54,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.Locale;
 
 public class ContentResource {
 
@@ -211,6 +212,11 @@ public class ContentResource {
     contentType.getLanguage().ifPresent(
       language -> responseBuilder.header(ProgrammingLanguages.HEADER, language)
     );
+
+    contentType.getSyntaxModes().forEach((mode, lang) -> {
+      String modeName = mode.substring(0, 1).toUpperCase(Locale.ENGLISH) + mode.substring(1);
+      responseBuilder.header(ProgrammingLanguages.HEADER_SYNTAX_MODE_PREFIX + modeName, lang);
+    });
   }
 
   private byte[] getHead(String revision, String path, RepositoryService repositoryService) throws IOException {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java

@@ -32,6 +32,7 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 @EqualsAndHashCode(callSuper = false)
@@ -63,6 +64,7 @@ public class DiffResultDto extends HalRepresentation {
     private String oldMode;
     private String type;
     private String language;
+    private Map<String, String> syntaxModes;
     private List<HunkDto> hunks;
 
   }

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java

@@ -26,6 +26,7 @@ package sonia.scm.api.v2.resources;
 
 import com.google.inject.Inject;
 import de.otto.edison.hal.Links;
+import sonia.scm.io.ContentType;
 import sonia.scm.io.ContentTypeResolver;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.DiffFile;
@@ -155,8 +156,10 @@ class DiffResultToDiffResultDtoMapper {
     dto.setOldPath(oldPath);
     dto.setOldRevision(file.getOldRevision());
 
-    Optional<String> language = contentTypeResolver.resolve(path).getLanguage();
+    ContentType contentType = contentTypeResolver.resolve(path);
+    Optional<String> language = contentType.getLanguage();
     language.ifPresent(dto::setLanguage);
+    dto.setSyntaxModes(contentType.getSyntaxModes());
 
     List<DiffResultDto.HunkDto> hunks = new ArrayList<>();
     for (Hunk hunk : file) {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java

@@ -28,6 +28,8 @@ final class ProgrammingLanguages {
 
   static final String HEADER = "X-Programming-Language";
 
+  static final String HEADER_SYNTAX_MODE_PREFIX = "X-Syntax-Mode-";
+
   private ProgrammingLanguages() {
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java

@@ -24,15 +24,18 @@
 
 package sonia.scm.io;
 
+import com.cloudogu.spotter.Language;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Collections;
+import java.util.Map;
 import java.util.Optional;
 
 public class DefaultContentType implements ContentType {
 
-  private static final String DEFAULT_LANG_MODE = "text";
+  private final com.cloudogu.spotter.ContentType contentType;
 
-  private final com.github.sdorra.spotter.ContentType contentType;
-
-  DefaultContentType(com.github.sdorra.spotter.ContentType contentType) {
+  DefaultContentType(com.cloudogu.spotter.ContentType contentType) {
     this.contentType = contentType;
   }
 
@@ -58,9 +61,23 @@ public class DefaultContentType implements ContentType {
 
   @Override
   public Optional<String> getLanguage() {
-    return contentType.getLanguage().map(language -> {
-      Optional<String> aceMode = language.getAceMode();
-      return aceMode.orElseGet(() -> language.getCodemirrorMode().orElse(DEFAULT_LANG_MODE));
-    });
+    return contentType.getLanguage().map(Language::getName);
+  }
+
+  @Override
+  public Map<String, String> getSyntaxModes() {
+    Optional<Language> language = contentType.getLanguage();
+    if (language.isPresent()) {
+      return syntaxMode(language.get());
+    }
+    return Collections.emptyMap();
+  }
+
+  static Map<String, String> syntaxMode(Language language) {
+    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+    language.getAceMode().ifPresent(mode -> builder.put("ace", mode));
+    language.getCodemirrorMode().ifPresent(mode -> builder.put("codemirror", mode));
+    language.getPrismMode().ifPresent(mode -> builder.put("prism", mode));
+    return builder.build();
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java

@@ -24,17 +24,32 @@
 
 package sonia.scm.io;
 
-import com.github.sdorra.spotter.ContentTypeDetector;
-import com.github.sdorra.spotter.Language;
+import com.cloudogu.spotter.ContentTypeDetector;
+import com.cloudogu.spotter.Language;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
 
 public final class DefaultContentTypeResolver implements ContentTypeResolver {
 
+  private static final Language[] BOOST = new Language[]{
+    // GCC Machine Description uses .md as extension, but markdown is much more likely
+    Language.MARKDOWN,
+    // XML uses .rs as extension, but rust is much more likely
+    Language.RUST,
+    // XML is also returned by content type boost strategy, but rust is really much more likely
+    Language.RUST,
+  };
+
   private static final ContentTypeDetector PATH_BASED = ContentTypeDetector.builder()
-    .defaultPathBased().boost(Language.MARKDOWN)
+    .defaultPathBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   private static final ContentTypeDetector PATH_AND_CONTENT_BASED = ContentTypeDetector.builder()
-    .defaultPathAndContentBased().boost(Language.MARKDOWN)
+    .defaultPathAndContentBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   @Override
@@ -46,4 +61,13 @@ public final class DefaultContentTypeResolver implements ContentTypeResolver {
   public DefaultContentType resolve(String path, byte[] contentPrefix) {
     return new DefaultContentType(PATH_AND_CONTENT_BASED.detect(path, contentPrefix));
   }
+
+  @Override
+  public Map<String, String> findSyntaxModesByLanguage(String language) {
+    Optional<Language> byName = Language.getByName(language);
+    if (byName.isPresent()) {
+      return DefaultContentType.syntaxMode(byName.get());
+    }
+    return Collections.emptyMap();
+  }
 }

scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java

@@ -90,11 +90,21 @@ public final class LuceneHighlighter {
     int index = content.indexOf(raw);
 
     int start = content.lastIndexOf('\n', index);
-    if (start < 0) {
-      start = 0;
-    }
 
-    String snippet = content.substring(start, index) + fragment;
+    String snippet;
+    if (start == index) {
+      // fragment starts with a linebreak
+      snippet = fragment.substring(1);
+    } else {
+      if (start < 0) {
+        // no leading linebreak
+        start = 0;
+      } else if (start < content.length()) {
+        // skip linebreak
+        start++;
+      }
+      snippet = content.substring(start, index) + fragment;
+    }
 
     int end = content.indexOf('\n', index + raw.length());
     if (end < 0) {