Use more accurate language detection for syntax highlighting (#1891)

Updated spotter to version 4 in order to get prism syntax mode for detected coding languages.
Expose syntax modes of coding languages as headers on content endpoint and as fields on diff dto.
Remove leading line break on search result fragments.
Use mark instead of span or strong for highlighted search results.
Add option to use syntax highlighting in TextHitField component.

Co-authored-by: Matthias Thieroff <matthias.thieroff@cloudogu.com>

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java

@@ -54,6 +54,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.Locale;
 
 public class ContentResource {
 
@@ -211,6 +212,11 @@ public class ContentResource {
     contentType.getLanguage().ifPresent(
       language -> responseBuilder.header(ProgrammingLanguages.HEADER, language)
     );
+
+    contentType.getSyntaxModes().forEach((mode, lang) -> {
+      String modeName = mode.substring(0, 1).toUpperCase(Locale.ENGLISH) + mode.substring(1);
+      responseBuilder.header(ProgrammingLanguages.HEADER_SYNTAX_MODE_PREFIX + modeName, lang);
+    });
   }
 
   private byte[] getHead(String revision, String path, RepositoryService repositoryService) throws IOException {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java

@@ -32,6 +32,7 @@ import lombok.Data;
 import lombok.EqualsAndHashCode;
 
 import java.util.List;
+import java.util.Map;
 
 @Data
 @EqualsAndHashCode(callSuper = false)
@@ -63,6 +64,7 @@ public class DiffResultDto extends HalRepresentation {
     private String oldMode;
     private String type;
     private String language;
+    private Map<String, String> syntaxModes;
     private List<HunkDto> hunks;
 
   }

scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java

@@ -26,6 +26,7 @@ package sonia.scm.api.v2.resources;
 
 import com.google.inject.Inject;
 import de.otto.edison.hal.Links;
+import sonia.scm.io.ContentType;
 import sonia.scm.io.ContentTypeResolver;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.DiffFile;
@@ -155,8 +156,10 @@ class DiffResultToDiffResultDtoMapper {
     dto.setOldPath(oldPath);
     dto.setOldRevision(file.getOldRevision());
 
-    Optional<String> language = contentTypeResolver.resolve(path).getLanguage();
+    ContentType contentType = contentTypeResolver.resolve(path);
+    Optional<String> language = contentType.getLanguage();
     language.ifPresent(dto::setLanguage);
+    dto.setSyntaxModes(contentType.getSyntaxModes());
 
     List<DiffResultDto.HunkDto> hunks = new ArrayList<>();
     for (Hunk hunk : file) {

scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java

@@ -28,6 +28,8 @@ final class ProgrammingLanguages {
 
   static final String HEADER = "X-Programming-Language";
 
+  static final String HEADER_SYNTAX_MODE_PREFIX = "X-Syntax-Mode-";
+
   private ProgrammingLanguages() {
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java

@@ -24,15 +24,18 @@
 
 package sonia.scm.io;
 
+import com.cloudogu.spotter.Language;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Collections;
+import java.util.Map;
 import java.util.Optional;
 
 public class DefaultContentType implements ContentType {
 
-  private static final String DEFAULT_LANG_MODE = "text";
+  private final com.cloudogu.spotter.ContentType contentType;
 
-  private final com.github.sdorra.spotter.ContentType contentType;
-
-  DefaultContentType(com.github.sdorra.spotter.ContentType contentType) {
+  DefaultContentType(com.cloudogu.spotter.ContentType contentType) {
     this.contentType = contentType;
   }
 
@@ -58,9 +61,23 @@ public class DefaultContentType implements ContentType {
 
   @Override
   public Optional<String> getLanguage() {
-    return contentType.getLanguage().map(language -> {
-      Optional<String> aceMode = language.getAceMode();
-      return aceMode.orElseGet(() -> language.getCodemirrorMode().orElse(DEFAULT_LANG_MODE));
-    });
+    return contentType.getLanguage().map(Language::getName);
+  }
+
+  @Override
+  public Map<String, String> getSyntaxModes() {
+    Optional<Language> language = contentType.getLanguage();
+    if (language.isPresent()) {
+      return syntaxMode(language.get());
+    }
+    return Collections.emptyMap();
+  }
+
+  static Map<String, String> syntaxMode(Language language) {
+    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+    language.getAceMode().ifPresent(mode -> builder.put("ace", mode));
+    language.getCodemirrorMode().ifPresent(mode -> builder.put("codemirror", mode));
+    language.getPrismMode().ifPresent(mode -> builder.put("prism", mode));
+    return builder.build();
   }
 }

scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java

@@ -24,17 +24,32 @@
 
 package sonia.scm.io;
 
-import com.github.sdorra.spotter.ContentTypeDetector;
-import com.github.sdorra.spotter.Language;
+import com.cloudogu.spotter.ContentTypeDetector;
+import com.cloudogu.spotter.Language;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
 
 public final class DefaultContentTypeResolver implements ContentTypeResolver {
 
+  private static final Language[] BOOST = new Language[]{
+    // GCC Machine Description uses .md as extension, but markdown is much more likely
+    Language.MARKDOWN,
+    // XML uses .rs as extension, but rust is much more likely
+    Language.RUST,
+    // XML is also returned by content type boost strategy, but rust is really much more likely
+    Language.RUST,
+  };
+
   private static final ContentTypeDetector PATH_BASED = ContentTypeDetector.builder()
-    .defaultPathBased().boost(Language.MARKDOWN)
+    .defaultPathBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   private static final ContentTypeDetector PATH_AND_CONTENT_BASED = ContentTypeDetector.builder()
-    .defaultPathAndContentBased().boost(Language.MARKDOWN)
+    .defaultPathAndContentBased()
+    .boost(BOOST)
     .bestEffortMatch();
 
   @Override
@@ -46,4 +61,13 @@ public final class DefaultContentTypeResolver implements ContentTypeResolver {
   public DefaultContentType resolve(String path, byte[] contentPrefix) {
     return new DefaultContentType(PATH_AND_CONTENT_BASED.detect(path, contentPrefix));
   }
+
+  @Override
+  public Map<String, String> findSyntaxModesByLanguage(String language) {
+    Optional<Language> byName = Language.getByName(language);
+    if (byName.isPresent()) {
+      return DefaultContentType.syntaxMode(byName.get());
+    }
+    return Collections.emptyMap();
+  }
 }

scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java

@@ -90,11 +90,21 @@ public final class LuceneHighlighter {
     int index = content.indexOf(raw);
 
     int start = content.lastIndexOf('\n', index);
-    if (start < 0) {
-      start = 0;
-    }
 
-    String snippet = content.substring(start, index) + fragment;
+    String snippet;
+    if (start == index) {
+      // fragment starts with a linebreak
+      snippet = fragment.substring(1);
+    } else {
+      if (start < 0) {
+        // no leading linebreak
+        start = 0;
+      } else if (start < content.length()) {
+        // skip linebreak
+        start++;
+      }
+      snippet = content.substring(start, index) + fragment;
+    }
 
     int end = content.indexOf('\n', index + raw.length());
     if (end < 0) {

scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java

@@ -141,7 +141,7 @@ public class ContentResourceTest {
     Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
     assertEquals(200, response.getStatus());
 
-    assertEquals("golang", response.getHeaderString("X-Programming-Language"));
+    assertEquals("Go", response.getHeaderString("X-Programming-Language"));
     assertEquals("text/x-go", response.getHeaderString("Content-Type"));
   }
 
@@ -152,10 +152,22 @@ public class ContentResourceTest {
     Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "Dockerfile", null, null);
     assertEquals(200, response.getStatus());
 
-    assertEquals("dockerfile", response.getHeaderString("X-Programming-Language"));
+    assertEquals("Dockerfile", response.getHeaderString("X-Programming-Language"));
     assertEquals("text/plain", response.getHeaderString("Content-Type"));
   }
 
+  @Test
+  public void shouldRecognizeSyntaxModes() throws Exception {
+    mockContentFromResource("SomeGoCode.go");
+
+    Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null);
+    assertEquals(200, response.getStatus());
+
+    assertEquals("golang", response.getHeaderString("X-Syntax-Mode-Ace"));
+    assertEquals("go", response.getHeaderString("X-Syntax-Mode-Codemirror"));
+    assertEquals("go", response.getHeaderString("X-Syntax-Mode-Prism"));
+  }
+
   @Test
   public void shouldHandleRandomByteFile() throws Exception {
     mockContentFromResource("JustBytes");
@@ -190,6 +202,7 @@ public class ContentResourceTest {
     assertEquals("application/octet-stream", response.getHeaderString("Content-Type"));
   }
 
+  @SuppressWarnings("UnstableApiUsage")
   private void mockContentFromResource(String fileName) throws Exception {
     URL url = Resources.getResource(fileName);
     mockContent(fileName, Resources.toByteArray(url));

scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java

@@ -60,11 +60,16 @@ class DiffResultToDiffResultDtoMapperTest {
     DiffResultDto dto = mapper.mapForRevision(REPOSITORY, createResult(), "123");
 
     List<DiffResultDto.FileDto> files = dto.getFiles();
-    assertAddedFile(files.get(0), "A.java", "abc", "java");
-    assertModifiedFile(files.get(1), "B.ts", "abc", "def", "typescript");
-    assertDeletedFile(files.get(2), "C.go", "ghi", "golang");
-    assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "typescript");
-    assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "typescript");
+    assertAddedFile(files.get(0), "A.java", "abc", "Java");
+    assertModifiedFile(files.get(1), "B.ts", "abc", "def", "TypeScript");
+    DiffResultDto.FileDto cGo = files.get(2);
+    assertDeletedFile(cGo, "C.go", "ghi", "Go");
+    assertThat(cGo.getSyntaxModes())
+      .containsEntry("ace", "golang")
+      .containsEntry("codemirror", "go")
+      .containsEntry("prism", "go");
+    assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "TypeScript");
+    assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "TypeScript");
 
     DiffResultDto.HunkDto hunk = files.get(1).getHunks().get(0);
     assertHunk(hunk, "@@ -3,4 1,2 @@", 1, 2, 3, 4);

scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java

@@ -24,15 +24,15 @@
 
 package sonia.scm.io;
 
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Nested;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Map;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
 
 class DefaultContentTypeResolverTest {
 
@@ -84,42 +84,45 @@ class DefaultContentTypeResolverTest {
         "% Which does not start with markdown"
       );
       ContentType contentType = contentTypeResolver.resolve("somedoc.md", content.getBytes(StandardCharsets.UTF_8));
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveMarkdownWithoutContent() {
       ContentType contentType = contentTypeResolver.resolve("somedoc.md");
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveMarkdownEvenWithDotsInFilename() {
       ContentType contentType = contentTypeResolver.resolve("somedoc.1.1.md");
-      Assertions.assertThat(contentType.getLanguage()).contains("markdown");
+      assertThat(contentType.getLanguage()).contains("Markdown");
     }
 
     @Test
     void shouldResolveDockerfile() {
       ContentType contentType = contentTypeResolver.resolve("Dockerfile");
-      Assertions.assertThat(contentType.getLanguage()).contains("dockerfile");
+      assertThat(contentType.getLanguage()).contains("Dockerfile");
     }
 
+  }
+
+  @Nested
+  class GetSyntaxModesTests {
 
     @Test
-    void shouldReturnAceModeIfPresent() {
-      assertThat(contentTypeResolver.resolve("app.go").getLanguage()).contains("golang"); // codemirror is just go
-      assertThat(contentTypeResolver.resolve("App.java").getLanguage()).contains("java"); // codemirror is clike
+    void shouldReturnEmptyMapOfModesWithoutLanguage() {
+      Map<String, String> syntaxModes = contentTypeResolver.resolve("app.exe").getSyntaxModes();
+      assertThat(syntaxModes).isEmpty();
     }
 
     @Test
-    void shouldReturnCodemirrorIfAceModeIsMissing() {
-      assertThat(contentTypeResolver.resolve("index.ecr").getLanguage()).contains("htmlmixed");
-    }
-
-    @Test
-    void shouldReturnTextIfNoModeIsPresent() {
-      assertThat(contentTypeResolver.resolve("index.hxml").getLanguage()).contains("text");
+    void shouldReturnMapOfModes() {
+      Map<String, String> syntaxModes = contentTypeResolver.resolve("app.rs").getSyntaxModes();
+      assertThat(syntaxModes)
+        .containsEntry("ace", "rust")
+        .containsEntry("codemirror", "rust")
+        .containsEntry("prism", "rust");
     }
 
   }

scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java

@@ -47,8 +47,6 @@ import static org.mockito.Mockito.when;
 @ExtendWith(MockitoExtension.class)
 class LuceneHighlighterTest {
 
-
-
   @Test
   void shouldHighlightText() throws InvalidTokenOffsetsException, IOException {
     StandardAnalyzer analyzer = new StandardAnalyzer();
@@ -80,6 +78,15 @@ class LuceneHighlighterTest {
     );
   }
 
+  @Test
+  void shouldNotStartHighlightedFragmentWithLineBreak() throws IOException, InvalidTokenOffsetsException {
+    String[] snippets = highlightCode("GameOfLife.java", "die");
+
+    assertThat(snippets).hasSize(1).allSatisfy(
+      snippet -> assertThat(snippet).doesNotStartWith("\n")
+    );
+  }
+
   @Test
   void shouldHighlightCodeInTsx() throws IOException, InvalidTokenOffsetsException {
     String[] snippets = highlightCode("Button.tsx", "inherit");