From e2d63cc2a10b6f7101858d1a894af1ff4e7ed062 Mon Sep 17 00:00:00 2001 From: Sebastian Sdorra Date: Mon, 13 Dec 2021 17:03:08 +0100 Subject: [PATCH] Use more accurate language detection for syntax highlighting (#1891) Updated spotter to version 4 in order to get prism syntax mode for detected coding languages. Expose syntax modes of coding languages as headers on content endpoint and as fields on diff dto. Remove leading line break on search result fragments. Use mark instead of span or strong for highlighted search results. Add option to use syntax highlighting in TextHitField component. Co-authored-by: Matthias Thieroff --- gradle/changelog/search_highlighter.yaml | 2 + gradle/dependencies.gradle | 2 +- .../main/java/sonia/scm/io/ContentType.java | 12 + .../sonia/scm/io/ContentTypeResolver.java | 17 +- scm-ui/ui-api/src/contentType.ts | 6 + scm-ui/ui-components/src/SplitAndReplace.tsx | 5 +- .../src/SyntaxHighlighter.stories.tsx | 2 +- .../src/__resources__/ContentSearchHit.ts | 67 ++ .../src/__snapshots__/storyshots.test.ts.snap | 1006 +++++------------ scm-ui/ui-components/src/languages.test.ts | 6 +- scm-ui/ui-components/src/languages.ts | 11 +- .../src/repos/TokenizedDiffView.tsx | 22 +- .../src/repos/annotate/Annotate.stories.tsx | 2 +- .../src/repos/refractorAdapter.ts | 17 +- .../src/search/HighlightedFragment.tsx | 2 +- .../src/search/SyntaxHighlightedFragment.tsx | 130 +++ .../src/search/TextHitField.stories.tsx | 54 + .../ui-components/src/search/TextHitField.tsx | 52 +- scm-ui/ui-types/src/Diff.ts | 1 + .../content/SwitchableMarkdownViewer.tsx | 2 +- .../repos/sources/containers/AnnotateView.tsx | 4 +- .../repos/sources/containers/SourcesView.tsx | 8 +- .../src/repos/sources/utils/files.ts | 5 + .../scm/api/v2/resources/ContentResource.java | 6 + .../scm/api/v2/resources/DiffResultDto.java | 2 + .../DiffResultToDiffResultDtoMapper.java | 5 +- .../v2/resources/ProgrammingLanguages.java | 2 + .../java/sonia/scm/io/DefaultContentType.java | 33 +- .../scm/io/DefaultContentTypeResolver.java | 32 +- .../sonia/scm/search/LuceneHighlighter.java | 18 +- .../api/v2/resources/ContentResourceTest.java | 17 +- .../DiffResultToDiffResultDtoMapperTest.java | 15 +- .../io/DefaultContentTypeResolverTest.java | 35 +- .../scm/search/LuceneHighlighterTest.java | 11 +- 34 files changed, 809 insertions(+), 802 deletions(-) create mode 100644 scm-ui/ui-components/src/__resources__/ContentSearchHit.ts create mode 100644 scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx create mode 100644 scm-ui/ui-components/src/search/TextHitField.stories.tsx diff --git a/gradle/changelog/search_highlighter.yaml b/gradle/changelog/search_highlighter.yaml index 2f1d4906a0..9ef23ede19 100644 --- a/gradle/changelog/search_highlighter.yaml +++ b/gradle/changelog/search_highlighter.yaml @@ -1,2 +1,4 @@ - type: changed description: Keep whole lines for code highlighting in search ([#1871](https://github.com/scm-manager/scm-manager/pull/1871)) +- type: changed + description: Use more accurate language detection for syntax highlighting ([#1891](https://github.com/scm-manager/scm-manager/pull/1891)) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 055378c5e6..1653079bb5 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -127,7 +127,7 @@ ext { webResources: 'com.github.sdorra:web-resources:1.1.1', // content type detection - spotter: 'com.github.sdorra:spotter-core:3.0.1', + spotter: 'com.cloudogu.spotter:spotter-core:4.0.0', tika: 'org.apache.tika:tika-core:1.25', // restart on unix diff --git a/scm-core/src/main/java/sonia/scm/io/ContentType.java b/scm-core/src/main/java/sonia/scm/io/ContentType.java index db788c9f34..941a8f63f1 100644 --- a/scm-core/src/main/java/sonia/scm/io/ContentType.java +++ b/scm-core/src/main/java/sonia/scm/io/ContentType.java @@ -24,6 +24,8 @@ package sonia.scm.io; +import java.util.Collections; +import java.util.Map; import java.util.Optional; /** @@ -68,4 +70,14 @@ public interface ContentType { * @return programming language or empty */ Optional getLanguage(); + + /** + * Returns a map of syntax modes such as codemirror, ace or prism. + * + * @return map of syntax modes + * @since 2.28.0 + */ + default Map getSyntaxModes() { + return Collections.emptyMap(); + } } diff --git a/scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java b/scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java index 347d0a1ae1..c56315b01c 100644 --- a/scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java +++ b/scm-core/src/main/java/sonia/scm/io/ContentTypeResolver.java @@ -24,6 +24,9 @@ package sonia.scm.io; +import java.util.Collections; +import java.util.Map; + /** * ContentTypeResolver is able to detect the {@link ContentType} of files based on their path and (optinally) a few starting bytes. These files do not have to be real files on the file system, but can be hypothetical constructs ("What content type is most probable for a file named like this"). * @@ -35,7 +38,6 @@ public interface ContentTypeResolver { * Detects the {@link ContentType} of the given path, by only using path based strategies. * * @param path path of the file - * * @return {@link ContentType} of path */ ContentType resolve(String path); @@ -43,10 +45,19 @@ public interface ContentTypeResolver { /** * Detects the {@link ContentType} of the given path, by using path and content based strategies. * - * @param path path of the file + * @param path path of the file * @param contentPrefix first few bytes of the content - * * @return {@link ContentType} of path and content prefix */ ContentType resolve(String path, byte[] contentPrefix); + + /** + * Returns a map of syntax highlighting modes such as ace, codemirror or prism by language. + * @param language name of the coding language + * @return map of syntax highlighting modes + * @since 2.28.0 + */ + default Map findSyntaxModesByLanguage(String language) { + return Collections.emptyMap(); + } } diff --git a/scm-ui/ui-api/src/contentType.ts b/scm-ui/ui-api/src/contentType.ts index 96d75b9582..28cca7a65c 100644 --- a/scm-ui/ui-api/src/contentType.ts +++ b/scm-ui/ui-api/src/contentType.ts @@ -28,6 +28,9 @@ import { ApiResultWithFetching } from "./base"; export type ContentType = { type: string; language?: string; + aceMode?: string; + codemirrorMode?: string; + prismMode?: string; }; function getContentType(url: string): Promise { @@ -35,6 +38,9 @@ function getContentType(url: string): Promise { return { type: response.headers.get("Content-Type") || "application/octet-stream", language: response.headers.get("X-Programming-Language") || undefined, + aceMode: response.headers.get("X-Syntax-Mode-Ace") || undefined, + codemirrorMode: response.headers.get("X-Syntax-Mode-Codemirror") || undefined, + prismMode: response.headers.get("X-Syntax-Mode-Prism") || undefined, }; }); } diff --git a/scm-ui/ui-components/src/SplitAndReplace.tsx b/scm-ui/ui-components/src/SplitAndReplace.tsx index e1125b2265..485133cc0c 100644 --- a/scm-ui/ui-components/src/SplitAndReplace.tsx +++ b/scm-ui/ui-components/src/SplitAndReplace.tsx @@ -33,9 +33,10 @@ export type Replacement = { type Props = { text: string; replacements: Replacement[]; + textWrapper?: (s: string) => ReactNode; }; -const textWrapper = (s: string) => { +const defaultTextWrapper = (s: string) => { const first = s.startsWith(" ") ? <>  : ""; const last = s.endsWith(" ") ? <>  : ""; return ( @@ -47,7 +48,7 @@ const textWrapper = (s: string) => { ); }; -const SplitAndReplace: FC = ({ text, replacements }) => { +const SplitAndReplace: FC = ({ text, replacements, textWrapper = defaultTextWrapper }) => { const parts = textSplitAndReplace(text, replacements, textWrapper); if (parts.length === 0) { return <>{parts[0]}; diff --git a/scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx b/scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx index 846f95ecfb..aa4c49c501 100644 --- a/scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx +++ b/scm-ui/ui-components/src/SyntaxHighlighter.stories.tsx @@ -48,7 +48,7 @@ storiesOf("SyntaxHighlighter", module) )) .add("Go", () => ( - + )) .add("Javascript", () => ( diff --git a/scm-ui/ui-components/src/__resources__/ContentSearchHit.ts b/scm-ui/ui-components/src/__resources__/ContentSearchHit.ts new file mode 100644 index 0000000000..bfb07e3a66 --- /dev/null +++ b/scm-ui/ui-components/src/__resources__/ContentSearchHit.ts @@ -0,0 +1,67 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Hit } from "@scm-manager/ui-types"; + +export const javaHit: Hit = { + score: 2.5, + fields: { + content: { + highlighted: true, + fragments: [ + "import org.slf4j.LoggerFactory;\n\nimport java.util.Date;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * Jwt implementation of {@link <|[[--AccessTokenBuilder--]]|>}.\n * \n * @author Sebastian Sdorra\n * @since 2.0.0\n */\npublic final class <|[[--JwtAccessTokenBuilder--]]|> implements <|[[--AccessTokenBuilder--]]|> {\n\n /**\n * the logger for <|[[--JwtAccessTokenBuilder--]]|>\n */\n private static final Logger LOG = LoggerFactory.getLogger(<|[[--JwtAccessTokenBuilder.class--]]|>);\n \n private final KeyGenerator keyGenerator; \n private final SecureKeyResolver keyResolver; \n \n private String subject;\n private String issuer;\n", + " private final Map custom = Maps.newHashMap();\n \n <|[[--JwtAccessTokenBuilder--]]|>(KeyGenerator keyGenerator, SecureKeyResolver keyResolver) {\n this.keyGenerator = keyGenerator;\n this.keyResolver = keyResolver;\n }\n\n @Override\n public <|[[--JwtAccessTokenBuilder--]]|> subject(String subject) {\n", + ' public <|[[--JwtAccessTokenBuilder--]]|> custom(String key, Object value) {\n Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "null or empty value not allowed");\n Preconditions.checkArgument(value != null, "null or empty value not allowed");\n' + ] + } + }, + _links: {} +}; + +export const bashHit: Hit = { + score: 2.5, + fields: { + content: { + highlighted: true, + fragments: [ + '# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n# SOFTWARE.\n#\n\n<|[[--getent--]]|> group scm >/dev/null || groupadd -r scm\n<|[[--getent--]]|> passwd scm >/dev/null || \\\n useradd -r -g scm -M \\\n -s /sbin/nologin -d /var/lib/scm \\\n -c "user for the scm-server process" scm\nexit 0\n\n' + ] + } + }, + _links: {} +}; + +export const markdownHit: Hit = { + score: 2.5, + fields: { + content: { + highlighted: true, + fragments: [ + "---\ntitle: SCM-Manager v2 Test <|[[--Cases--]]|>\n---\n\nDescribes the expected behaviour for SCMM v2 REST Resources using manual tests.\n\nThe following states general test <|[[--cases--]]|> per HTTP Method and en expected return code as well as exemplary curl calls.\nResource-specifics are stated \n\n## Test <|[[--Cases--]]|>\n\n### GET\n\n- Collection Resource (e.g. `/users`)\n - Without parameters -> 200\n - Parameters\n - `?pageSize=1` -> Only one embedded element, pageTotal reflects the correct number of pages, `last` link points to last page.\n - `?pageSize=1&page=1` -> `next` link points to page 0 ; `prev` link points to page 2\n - `?sortBy=admin` -> Sorted by `admin` field of embedded objects\n - `?sortBy=admin&desc=true` -> Invert sorting\n- Individual Resource (e.g. `/users/scmadmin`)\n - Exists -> 200\n", + "\n### DELETE\n\n- existing -> 204\n- not existing -> 204\n- without permission -> 401\n\n## Exemplary calls & Resource specific test <|[[--cases--]]|>\n\nIn order to extend those tests to other Resources, have a look at the rest docs. Note that the Content Type is specific to each resource as well.\n" + ] + } + }, + _links: {} +}; diff --git a/scm-ui/ui-components/src/__snapshots__/storyshots.test.ts.snap b/scm-ui/ui-components/src/__snapshots__/storyshots.test.ts.snap index 9445f16f0e..7c7930b3cd 100644 --- a/scm-ui/ui-components/src/__snapshots__/storyshots.test.ts.snap +++ b/scm-ui/ui-components/src/__snapshots__/storyshots.test.ts.snap @@ -83959,20 +83959,10 @@ exports[`Storyshots SyntaxHighlighter Markdown 1`] = ` - package - - - main + package main @@ -84031,45 +84021,8 @@ exports[`Storyshots SyntaxHighlighter Markdown 1`] = ` - - - - - import - - - - - - ( - - - + import ( -
- - - - - "fmt" - - - + "fmt" -
- - - - - "net/http" - - - + "net/http" -
- - - - - ) - - - + ) -
- - - - - func - - - - - - handler - - - ( - - - w http - - - . - - - ResponseWriter - - - , - - - r - - - * - - - http - - - . - - - Request - - - ) - - - - - - { - - - + func handler(w http.ResponseWriter, r *http.Request) { -
- - fmt - - - . - - - Fprintf - - - ( - - - w - - - , - - - - - - "Hi there, I love %s!" - - - , - - - r - - - . - - - URL - - - . - - - Path - - - [ - - - 1 - - - : - - - ] - - - ) - - - + fmt.Fprintf(w, "Hi there, I love %s!", r.URL.Path[1:]) -
- - - - - } - - - + } -
- - - - - func - - - - - - main - - - ( - - - ) - - - - - - { - - - + func main() { -
- - http - - - . - - - HandleFunc - - - ( - - - "/" - - - , - - - handler - - - ) - - - + http.HandleFunc("/", handler) -
- - http - - - . - - - ListenAndServe - - - ( - - - ":8080" - - - , - - - - - - nil - - - ) - - - + http.ListenAndServe(":8080", nil) -
- - - } `; +exports[`Storyshots TextHitField Bash SyntaxHighlighting 1`] = ` +
+  ...
+
+  # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+
+  
+    getent
+  
+   group scm >/dev/null || groupadd -r scm
+
+  
+    getent
+  
+   passwd scm >/dev/null || \\
+    useradd -r -g scm -M \\
+    -s /sbin/nologin -d /var/lib/scm \\
+    -c "user for the scm-server process" scm
+exit 0
+
+
+  ...
+
+
+`; + +exports[`Storyshots TextHitField Default 1`] = ` +
+   ... 
+  import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Jwt implementation of {@link 
+  
+    AccessTokenBuilder
+  
+  }.
+ * 
+ * @author Sebastian Sdorra
+ * @since 2.0.0
+ */
+public final class 
+  
+    JwtAccessTokenBuilder
+  
+   implements 
+  
+    AccessTokenBuilder
+  
+   {
+
+  /**
+   * the logger for 
+  
+    JwtAccessTokenBuilder
+  
+  
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(
+  
+    JwtAccessTokenBuilder.class
+  
+  );
+  
+  private final KeyGenerator keyGenerator; 
+  private final SecureKeyResolver keyResolver; 
+  
+  private String subject;
+  private String issuer;
+
+   ... 
+    private final Map<String,Object> custom = Maps.newHashMap();
+  
+  
+  
+    JwtAccessTokenBuilder
+  
+  (KeyGenerator keyGenerator, SecureKeyResolver keyResolver)  {
+    this.keyGenerator = keyGenerator;
+    this.keyResolver = keyResolver;
+  }
+
+  @Override
+  public 
+  
+    JwtAccessTokenBuilder
+  
+   subject(String subject) {
+
+   ... 
+    public 
+  
+    JwtAccessTokenBuilder
+  
+   custom(String key, Object value) {
+    Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "null or empty value not allowed");
+    Preconditions.checkArgument(value != null, "null or empty value not allowed");
+
+   ... 
+
+`; + +exports[`Storyshots TextHitField Java SyntaxHighlighting 1`] = ` +
+  ...
+
+  import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Jwt implementation of {@link 
+  
+    AccessTokenBuilder
+  
+  }.
+ * 
+ * @author Sebastian Sdorra
+ * @since 2.0.0
+ */
+public final class 
+  
+    JwtAccessTokenBuilder
+  
+   implements 
+  
+    AccessTokenBuilder
+  
+   {
+
+  /**
+   * the logger for 
+  
+    JwtAccessTokenBuilder
+  
+  
+   */
+  private static final Logger LOG = LoggerFactory.getLogger(
+  
+    JwtAccessTokenBuilder.class
+  
+  );
+  
+  private final KeyGenerator keyGenerator; 
+  private final SecureKeyResolver keyResolver; 
+  
+  private String subject;
+  private String issuer;
+
+  ...
+
+    private final Map<String,Object> custom = Maps.newHashMap();
+  
+  
+  
+    JwtAccessTokenBuilder
+  
+  (KeyGenerator keyGenerator, SecureKeyResolver keyResolver)  {
+    this.keyGenerator = keyGenerator;
+    this.keyResolver = keyResolver;
+  }
+
+  @Override
+  public 
+  
+    JwtAccessTokenBuilder
+  
+   subject(String subject) {
+
+  ...
+
+    public 
+  
+    JwtAccessTokenBuilder
+  
+   custom(String key, Object value) {
+    Preconditions.checkArgument(!Strings.isNullOrEmpty(key), "null or empty value not allowed");
+    Preconditions.checkArgument(value != null, "null or empty value not allowed");
+
+  ...
+
+
+`; + +exports[`Storyshots TextHitField Markdown SyntaxHighlighting 1`] = ` +
+  ...
+
+  ---
+title: SCM-Manager v2 Test 
+  
+    Cases
+  
+  
+---
+
+Describes the expected behaviour for SCMM v2 REST Resources using manual tests.
+
+The following states general test 
+  
+    cases
+  
+   per HTTP Method and en expected return code as well as exemplary curl calls.
+Resource-specifics are stated 
+
+## Test 
+  
+    Cases
+  
+  
+
+### GET
+
+- Collection Resource (e.g. \`/users\`)
+    - Without parameters -> 200
+    - Parameters
+        - \`?pageSize=1\` -> Only one embedded element, pageTotal reflects the correct number of pages, \`last\` link points to last page.
+        - \`?pageSize=1&page=1\` -> \`next\` link points to page 0 ; \`prev\` link points to page 2
+        - \`?sortBy=admin\` -> Sorted by \`admin\` field of embedded objects
+        - \`?sortBy=admin&desc=true\` -> Invert sorting
+- Individual Resource (e.g. \`/users/scmadmin\`)
+    - Exists  -> 200
+
+  ...
+
+  
+### DELETE
+
+- existing -> 204
+- not existing -> 204
+- without permission -> 401
+
+## Exemplary calls & Resource specific test 
+  
+    cases
+  
+  
+
+In order to extend those tests to other Resources, have a look at the rest docs. Note that the Content Type is specific to each resource as well.
+
+  ...
+
+
+`; + +exports[`Storyshots TextHitField Unknown SyntaxHighlighting 1`] = ` +
+  ...
+
+  # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+
+  
+    getent
+  
+   group scm >/dev/null || groupadd -r scm
+
+  
+    getent
+  
+   passwd scm >/dev/null || \\
+    useradd -r -g scm -M \\
+    -s /sbin/nologin -d /var/lib/scm \\
+    -c "user for the scm-server process" scm
+exit 0
+
+
+  ...
+
+
+`; + exports[`Storyshots Toast Click to close 1`] = `null`; exports[`Storyshots Toast Danger 1`] = `null`; diff --git a/scm-ui/ui-components/src/languages.test.ts b/scm-ui/ui-components/src/languages.test.ts index 61a6214065..9003559b79 100644 --- a/scm-ui/ui-components/src/languages.test.ts +++ b/scm-ui/ui-components/src/languages.test.ts @@ -35,7 +35,7 @@ describe("syntax highlighter", () => { expect(java).toBe("java"); }); - it("should return text if language is undefied", () => { + it("should return text if language is undefined", () => { const lang = determineLanguage(); expect(lang).toBe("text"); }); @@ -45,8 +45,4 @@ describe("syntax highlighter", () => { expect(lang).toBe("text"); }); - it("should use alias go for golang", () => { - const go = determineLanguage("golang"); - expect(go).toBe("go"); - }); }); diff --git a/scm-ui/ui-components/src/languages.ts b/scm-ui/ui-components/src/languages.ts index 80f8e2a4a0..6d9c16c897 100644 --- a/scm-ui/ui-components/src/languages.ts +++ b/scm-ui/ui-components/src/languages.ts @@ -22,20 +22,11 @@ * SOFTWARE. */ -// this aliases are only to map from spotter detection to prismjs -const languageAliases: { [key: string]: string } = { - golang: "go", -}; - export const defaultLanguage = "text"; export const determineLanguage = (language?: string) => { if (!language) { return defaultLanguage; } - const lang = language.toLowerCase(); - if (languageAliases[lang]) { - return languageAliases[lang]; - } - return lang; + return language.toLowerCase(); }; diff --git a/scm-ui/ui-components/src/repos/TokenizedDiffView.tsx b/scm-ui/ui-components/src/repos/TokenizedDiffView.tsx index a62054b044..07a2f6ce30 100644 --- a/scm-ui/ui-components/src/repos/TokenizedDiffView.tsx +++ b/scm-ui/ui-components/src/repos/TokenizedDiffView.tsx @@ -26,35 +26,46 @@ import styled from "styled-components"; // @ts-ignore we have no typings for react-diff-view import { Diff, useTokenizeWorker } from "react-diff-view"; import { FileDiff } from "@scm-manager/ui-types"; -import { determineLanguage } from "../languages"; // @ts-ignore no types for css modules import theme from "../syntax-highlighting.module.css"; +import { determineLanguage } from "../languages"; const DiffView = styled(Diff)` /* align line numbers */ + & .diff-gutter { text-align: right; } + /* column sizing */ + > colgroup .diff-gutter-col { width: 3.25rem; } + /* prevent following content from moving down */ + > .diff-gutter:empty:hover::after { font-size: 0.7rem; } + /* smaller font size for code and ensure same monospace font throughout whole scmm */ + & .diff-line { font-size: 0.75rem; font-family: "Courier New", Monaco, Menlo, "Ubuntu Mono", "source-code-pro", monospace; } + /* comment padding for sidebyside view */ + &.split .diff-widget-content .is-indented-line { padding-left: 3.25rem; } + /* comment padding for combined view */ + &.unified .diff-widget-content .is-indented-line { padding-left: 6.5rem; } @@ -71,10 +82,17 @@ type Props = { className?: string; }; +const findSyntaxHighlightingLanguage = (file: FileDiff) => { + if (file.syntaxModes) { + return file.syntaxModes["prism"] || file.syntaxModes["codemirror"] || file.syntaxModes["ace"] || file.language; + } + return file.language; +}; + const TokenizedDiffView: FC = ({ file, viewType, className, children }) => { const { tokens } = useTokenizeWorker(tokenize, { hunks: file.hunks, - language: determineLanguage(file.language), + language: determineLanguage(findSyntaxHighlightingLanguage(file)) }); return ( diff --git a/scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx b/scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx index 9f1e71035d..17fe9050e8 100644 --- a/scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx +++ b/scm-ui/ui-components/src/repos/annotate/Annotate.stories.tsx @@ -67,7 +67,7 @@ const commitImplementMain = { }; const source: AnnotatedSource = { - language: "golang", + language: "go", lines: [ { lineNumber: 1, diff --git a/scm-ui/ui-components/src/repos/refractorAdapter.ts b/scm-ui/ui-components/src/repos/refractorAdapter.ts index 909572670a..adfea25850 100644 --- a/scm-ui/ui-components/src/repos/refractorAdapter.ts +++ b/scm-ui/ui-components/src/repos/refractorAdapter.ts @@ -46,10 +46,15 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => { import( /* webpackChunkName: "tokenizer-refractor-[request]" */ `refractor/lang/${lang}` - ).then((loadedLanguage) => { - refractor.register(loadedLanguage.default); - callback(); - }); + ) + .then(loadedLanguage => { + refractor.register(loadedLanguage.default); + callback(); + }) + .catch(e => { + // eslint-disable-next-line no-console + console.log(`failed to load refractor language ${lang}: ${e}`); + }); } }; @@ -58,7 +63,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => { const runHook = (name: string, env: RunHookEnv) => { originalRunHook.apply(name, env); if (env.classes) { - env.classes = env.classes.map((className) => theme[className] || className); + env.classes = env.classes.map(className => theme[className] || className); } }; // @ts-ignore hooks are not in the type definition @@ -67,7 +72,7 @@ const createAdapter = (theme: { [key: string]: string }): RefractorAdapter => { return { isLanguageRegistered, loadLanguage, - ...refractor, + ...refractor }; }; diff --git a/scm-ui/ui-components/src/search/HighlightedFragment.tsx b/scm-ui/ui-components/src/search/HighlightedFragment.tsx index 5444ff039c..6437bcd485 100644 --- a/scm-ui/ui-components/src/search/HighlightedFragment.tsx +++ b/scm-ui/ui-components/src/search/HighlightedFragment.tsx @@ -42,7 +42,7 @@ const HighlightedFragment: FC = ({ value }) => { if (start > 0) { result.push(content.substring(0, start)); } - result.push({content.substring(start + PRE_TAG.length, end)}); + result.push({content.substring(start + PRE_TAG.length, end)}); content = content.substring(end + POST_TAG.length); } else { result.push(content); diff --git a/scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx b/scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx new file mode 100644 index 0000000000..4a6c857b41 --- /dev/null +++ b/scm-ui/ui-components/src/search/SyntaxHighlightedFragment.tsx @@ -0,0 +1,130 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import React, { FC, ReactNode, useEffect, useMemo, useState } from "react"; +import createAdapter from "../repos/refractorAdapter"; +// @ts-ignore no types for css modules +import theme from "../syntax-highlighting.module.css"; +import SplitAndReplace, { Replacement } from "../SplitAndReplace"; +import { AST, RefractorNode } from "refractor"; +import { determineLanguage } from "../languages"; + +const PRE_TAG = "<|[[--"; +const POST_TAG = "--]]|>"; +const PRE_TAG_REGEX = /<\|\[\[--/g; +const POST_TAG_REGEX = /--]]\|>/g; + +const adapter = createAdapter(theme); + +function createReplacement(textToReplace: string): Replacement { + return { + textToReplace, + replacement: {textToReplace}, + replaceAll: true + }; +} + +function mapWithDepth(depth: number, replacements: Replacement[]) { + return function mapChildrenWithDepth(child: RefractorNode, i: number) { + return mapChild(child, i, depth, replacements); + }; +} + +function isAstElement(node: RefractorNode): node is AST.Element { + return (node as AST.Element).tagName !== undefined; +} + +function mapChild(child: RefractorNode, i: number, depth: number, replacements: Replacement[]): ReactNode { + if (isAstElement(child)) { + const className = + child.properties && Array.isArray(child.properties.className) + ? child.properties.className.join(" ") + : child.properties.className; + + return React.createElement( + child.tagName, + Object.assign({ key: `fract-${depth}-${i}` }, child.properties, { className }), + child.children && child.children.map(mapWithDepth(depth + 1, replacements)) + ); + } + + return ( + s} /> + ); +} + +type Props = { + value: string; + language: string; +}; + +const stripAndReplace = (value: string) => { + const strippedValue = value.replace(PRE_TAG_REGEX, "").replace(POST_TAG_REGEX, ""); + let content = value; + + const result: string[] = []; + while (content.length > 0) { + const start = content.indexOf(PRE_TAG); + const end = content.indexOf(POST_TAG); + if (start >= 0 && end > 0) { + const item = content.substring(start + PRE_TAG.length, end); + if (!result.includes(item)) { + result.push(item); + } + content = content.substring(end + POST_TAG.length); + } else { + break; + } + } + + result.sort((a, b) => b.length - a.length); + + return { + strippedValue, + replacements: result.map(createReplacement) + }; +}; + +const SyntaxHighlightedFragment: FC = ({ value, language }) => { + const [isLoading, setIsLoading] = useState(true); + const determinedLanguage = determineLanguage(language); + const { strippedValue, replacements } = useMemo(() => stripAndReplace(value), [value]); + + useEffect(() => { + adapter.loadLanguage(determinedLanguage, () => { + setIsLoading(false); + }); + }, [determinedLanguage]); + + if (isLoading) { + return s} />; + } + + const refractorNodes = adapter.highlight(strippedValue, determinedLanguage); + const highlightedFragment = refractorNodes.map(mapWithDepth(0, replacements)); + + return <>{highlightedFragment}; +}; + +export default SyntaxHighlightedFragment; diff --git a/scm-ui/ui-components/src/search/TextHitField.stories.tsx b/scm-ui/ui-components/src/search/TextHitField.stories.tsx new file mode 100644 index 0000000000..5a40127f59 --- /dev/null +++ b/scm-ui/ui-components/src/search/TextHitField.stories.tsx @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +import React from "react"; +import { storiesOf } from "@storybook/react"; +import { bashHit, javaHit, markdownHit } from "../__resources__/ContentSearchHit"; +import TextHitField from "./TextHitField"; + +storiesOf("TextHitField", module) + .add("Default", () => ( + 
+      
+
+ )) + .add("Java SyntaxHighlighting", () => ( + 
+      
+
+ )) + .add("Bash SyntaxHighlighting", () => ( + 
+      
+
+ )) + .add("Markdown SyntaxHighlighting", () => ( + 
+      
+
+ )) + .add("Unknown SyntaxHighlighting", () => ( + 
+      
+
+ )); diff --git a/scm-ui/ui-components/src/search/TextHitField.tsx b/scm-ui/ui-components/src/search/TextHitField.tsx index 3af01a1334..73fdca7bb8 100644 --- a/scm-ui/ui-components/src/search/TextHitField.tsx +++ b/scm-ui/ui-components/src/search/TextHitField.tsx @@ -26,35 +26,51 @@ import React, { FC } from "react"; import { HighlightedHitField, Hit } from "@scm-manager/ui-types"; import HighlightedFragment from "./HighlightedFragment"; import { isHighlightedHitField } from "./fields"; +import SyntaxHighlightedFragment from "./SyntaxHighlightedFragment"; + +type HighlightedTextFieldProps = { + field: HighlightedHitField; + syntaxHighlightingLanguage?: string; +}; + +const HighlightedTextField: FC = ({ field, syntaxHighlightingLanguage }) => { + const separator = syntaxHighlightingLanguage ? "...\n" : " ... "; + return ( + <> + {field.fragments.map((fragment, i) => ( + + {separator} + {syntaxHighlightingLanguage ? ( + + ) : ( + + )} + {i + 1 >= field.fragments.length ? separator : null} + + ))} + + ); +}; type Props = { hit: Hit; field: string; truncateValueAt?: number; + syntaxHighlightingLanguage?: string; }; -type HighlightedTextFieldProps = { - field: HighlightedHitField; -}; - -const HighlightedTextField: FC = ({ field }) => ( - <> - {field.fragments.map((fr, i) => ( - - {" ... "} - - {i + 1 >= field.fragments.length ? " ... " : null} - - ))} - -); - -const TextHitField: FC = ({ hit, field: fieldName, children, truncateValueAt = 0 }) => { +const TextHitField: FC = ({ + hit, + field: fieldName, + children, + syntaxHighlightingLanguage, + truncateValueAt = 0 +}) => { const field = hit.fields[fieldName]; if (!field) { return <>{children}; } else if (isHighlightedHitField(field)) { - return ; + return ; } else { let value = field.value; if (value === "") { diff --git a/scm-ui/ui-types/src/Diff.ts b/scm-ui/ui-types/src/Diff.ts index 98b7a48d25..6ae8eded45 100644 --- a/scm-ui/ui-types/src/Diff.ts +++ b/scm-ui/ui-types/src/Diff.ts @@ -43,6 +43,7 @@ export type FileDiff = { oldRevision?: string; type: FileChangeType; language?: string; + syntaxModes?: { [mode: string]: string }; // TODO does this property exists? isBinary?: boolean; _links?: Links; diff --git a/scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx b/scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx index d4c46b62c3..392eaed8b6 100644 --- a/scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx +++ b/scm-ui/ui-webapp/src/repos/sources/components/content/SwitchableMarkdownViewer.tsx @@ -79,7 +79,7 @@ const SwitchableMarkdownViewer: FC = ({ file, basePath }) => { {renderMarkdown ? ( ) : ( - + )}
); diff --git a/scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx b/scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx index 34cfd352b2..dd57c62d8e 100644 --- a/scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx +++ b/scm-ui/ui-webapp/src/repos/sources/containers/AnnotateView.tsx @@ -26,6 +26,7 @@ import React, { FC } from "react"; import { File, Link, Repository } from "@scm-manager/ui-types"; import { Annotate, ErrorNotification, Loading } from "@scm-manager/ui-components"; import { useAnnotations, useContentType } from "@scm-manager/ui-api"; +import { determineSyntaxHighlightingLanguage } from "../utils/files"; type Props = { file: File; @@ -52,7 +53,8 @@ const AnnotateView: FC = ({ file, repository, revision }) => { return ; } - return ; + const language = determineSyntaxHighlightingLanguage(contentType); + return ; }; export default AnnotateView; diff --git a/scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx b/scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx index ead349bf0a..5a696691e6 100644 --- a/scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx +++ b/scm-ui/ui-webapp/src/repos/sources/containers/SourcesView.tsx @@ -31,7 +31,8 @@ import { File, Link, Repository } from "@scm-manager/ui-types"; import { ErrorNotification, Loading, PdfViewer } from "@scm-manager/ui-components"; import SwitchableMarkdownViewer from "../components/content/SwitchableMarkdownViewer"; import styled from "styled-components"; -import { useContentType } from "@scm-manager/ui-api"; +import { ContentType, useContentType } from "@scm-manager/ui-api"; +import {determineSyntaxHighlightingLanguage} from "../utils/files"; const NoSpacingSyntaxHighlighterContainer = styled.div` & pre { @@ -46,6 +47,8 @@ type Props = { revision: string; }; + + const SourcesView: FC = ({ file, repository, revision }) => { const { data: contentTypeData, error, isLoading } = useContentType((file._links.self as Link).href); @@ -59,7 +62,8 @@ const SourcesView: FC = ({ file, repository, revision }) => { let sources; - const { type: contentType, language } = contentTypeData; + const language = determineSyntaxHighlightingLanguage(contentTypeData); + const contentType = contentTypeData.type; const basePath = `/repo/${repository.namespace}/${repository.name}/code/sources/${revision}/`; if (contentType.startsWith("image/")) { sources = ; diff --git a/scm-ui/ui-webapp/src/repos/sources/utils/files.ts b/scm-ui/ui-webapp/src/repos/sources/utils/files.ts index eb3ca0f2d8..e4b5053908 100644 --- a/scm-ui/ui-webapp/src/repos/sources/utils/files.ts +++ b/scm-ui/ui-webapp/src/repos/sources/utils/files.ts @@ -22,6 +22,7 @@ * SOFTWARE. */ import { File } from "@scm-manager/ui-types"; +import { ContentType } from "@scm-manager/ui-api"; export const isRootPath = (path: string) => { return path === "" || path === "/"; @@ -40,3 +41,7 @@ export const isEmptyDirectory = (file: File) => { } return (file._embedded?.children?.length || 0) === 0; }; + +export const determineSyntaxHighlightingLanguage = (data: ContentType) => { + return data.prismMode || data.codemirrorMode || data.aceMode || data.language; +}; diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java index 2a6e1dd2c5..27d7420384 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ContentResource.java @@ -54,6 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; +import java.util.Locale; public class ContentResource { @@ -211,6 +212,11 @@ public class ContentResource { contentType.getLanguage().ifPresent( language -> responseBuilder.header(ProgrammingLanguages.HEADER, language) ); + + contentType.getSyntaxModes().forEach((mode, lang) -> { + String modeName = mode.substring(0, 1).toUpperCase(Locale.ENGLISH) + mode.substring(1); + responseBuilder.header(ProgrammingLanguages.HEADER_SYNTAX_MODE_PREFIX + modeName, lang); + }); } private byte[] getHead(String revision, String path, RepositoryService repositoryService) throws IOException { diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java index 4b08251053..51d05ec081 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultDto.java @@ -32,6 +32,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import java.util.List; +import java.util.Map; @Data @EqualsAndHashCode(callSuper = false) @@ -63,6 +64,7 @@ public class DiffResultDto extends HalRepresentation { private String oldMode; private String type; private String language; + private Map syntaxModes; private List hunks; } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java index 0c916cf601..c528606449 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapper.java @@ -26,6 +26,7 @@ package sonia.scm.api.v2.resources; import com.google.inject.Inject; import de.otto.edison.hal.Links; +import sonia.scm.io.ContentType; import sonia.scm.io.ContentTypeResolver; import sonia.scm.repository.Repository; import sonia.scm.repository.api.DiffFile; @@ -155,8 +156,10 @@ class DiffResultToDiffResultDtoMapper { dto.setOldPath(oldPath); dto.setOldRevision(file.getOldRevision()); - Optional language = contentTypeResolver.resolve(path).getLanguage(); + ContentType contentType = contentTypeResolver.resolve(path); + Optional language = contentType.getLanguage(); language.ifPresent(dto::setLanguage); + dto.setSyntaxModes(contentType.getSyntaxModes()); List hunks = new ArrayList<>(); for (Hunk hunk : file) { diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java index e6d88bd216..33dc6f815e 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ProgrammingLanguages.java @@ -28,6 +28,8 @@ final class ProgrammingLanguages { static final String HEADER = "X-Programming-Language"; + static final String HEADER_SYNTAX_MODE_PREFIX = "X-Syntax-Mode-"; + private ProgrammingLanguages() { } } diff --git a/scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java b/scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java index 19efaa42d0..530f66ba3e 100644 --- a/scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java +++ b/scm-webapp/src/main/java/sonia/scm/io/DefaultContentType.java @@ -24,15 +24,18 @@ package sonia.scm.io; +import com.cloudogu.spotter.Language; +import com.google.common.collect.ImmutableMap; + +import java.util.Collections; +import java.util.Map; import java.util.Optional; public class DefaultContentType implements ContentType { - private static final String DEFAULT_LANG_MODE = "text"; + private final com.cloudogu.spotter.ContentType contentType; - private final com.github.sdorra.spotter.ContentType contentType; - - DefaultContentType(com.github.sdorra.spotter.ContentType contentType) { + DefaultContentType(com.cloudogu.spotter.ContentType contentType) { this.contentType = contentType; } @@ -58,9 +61,23 @@ public class DefaultContentType implements ContentType { @Override public Optional getLanguage() { - return contentType.getLanguage().map(language -> { - Optional aceMode = language.getAceMode(); - return aceMode.orElseGet(() -> language.getCodemirrorMode().orElse(DEFAULT_LANG_MODE)); - }); + return contentType.getLanguage().map(Language::getName); + } + + @Override + public Map getSyntaxModes() { + Optional language = contentType.getLanguage(); + if (language.isPresent()) { + return syntaxMode(language.get()); + } + return Collections.emptyMap(); + } + + static Map syntaxMode(Language language) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + language.getAceMode().ifPresent(mode -> builder.put("ace", mode)); + language.getCodemirrorMode().ifPresent(mode -> builder.put("codemirror", mode)); + language.getPrismMode().ifPresent(mode -> builder.put("prism", mode)); + return builder.build(); } } diff --git a/scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java b/scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java index 7b354b6fb2..033232c3b7 100644 --- a/scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java +++ b/scm-webapp/src/main/java/sonia/scm/io/DefaultContentTypeResolver.java @@ -24,17 +24,32 @@ package sonia.scm.io; -import com.github.sdorra.spotter.ContentTypeDetector; -import com.github.sdorra.spotter.Language; +import com.cloudogu.spotter.ContentTypeDetector; +import com.cloudogu.spotter.Language; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; public final class DefaultContentTypeResolver implements ContentTypeResolver { + private static final Language[] BOOST = new Language[]{ + // GCC Machine Description uses .md as extension, but markdown is much more likely + Language.MARKDOWN, + // XML uses .rs as extension, but rust is much more likely + Language.RUST, + // XML is also returned by content type boost strategy, but rust is really much more likely + Language.RUST, + }; + private static final ContentTypeDetector PATH_BASED = ContentTypeDetector.builder() - .defaultPathBased().boost(Language.MARKDOWN) + .defaultPathBased() + .boost(BOOST) .bestEffortMatch(); private static final ContentTypeDetector PATH_AND_CONTENT_BASED = ContentTypeDetector.builder() - .defaultPathAndContentBased().boost(Language.MARKDOWN) + .defaultPathAndContentBased() + .boost(BOOST) .bestEffortMatch(); @Override @@ -46,4 +61,13 @@ public final class DefaultContentTypeResolver implements ContentTypeResolver { public DefaultContentType resolve(String path, byte[] contentPrefix) { return new DefaultContentType(PATH_AND_CONTENT_BASED.detect(path, contentPrefix)); } + + @Override + public Map findSyntaxModesByLanguage(String language) { + Optional byName = Language.getByName(language); + if (byName.isPresent()) { + return DefaultContentType.syntaxMode(byName.get()); + } + return Collections.emptyMap(); + } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java index b4445cb211..cdcfe81617 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneHighlighter.java @@ -90,11 +90,21 @@ public final class LuceneHighlighter { int index = content.indexOf(raw); int start = content.lastIndexOf('\n', index); - if (start < 0) { - start = 0; - } - String snippet = content.substring(start, index) + fragment; + String snippet; + if (start == index) { + // fragment starts with a linebreak + snippet = fragment.substring(1); + } else { + if (start < 0) { + // no leading linebreak + start = 0; + } else if (start < content.length()) { + // skip linebreak + start++; + } + snippet = content.substring(start, index) + fragment; + } int end = content.indexOf('\n', index + raw.length()); if (end < 0) { diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java index d7a72c10cc..0380027c12 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ContentResourceTest.java @@ -141,7 +141,7 @@ public class ContentResourceTest { Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null); assertEquals(200, response.getStatus()); - assertEquals("golang", response.getHeaderString("X-Programming-Language")); + assertEquals("Go", response.getHeaderString("X-Programming-Language")); assertEquals("text/x-go", response.getHeaderString("Content-Type")); } @@ -152,10 +152,22 @@ public class ContentResourceTest { Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "Dockerfile", null, null); assertEquals(200, response.getStatus()); - assertEquals("dockerfile", response.getHeaderString("X-Programming-Language")); + assertEquals("Dockerfile", response.getHeaderString("X-Programming-Language")); assertEquals("text/plain", response.getHeaderString("Content-Type")); } + @Test + public void shouldRecognizeSyntaxModes() throws Exception { + mockContentFromResource("SomeGoCode.go"); + + Response response = contentResource.get(NAMESPACE, REPO_NAME, REV, "SomeGoCode.go", null, null); + assertEquals(200, response.getStatus()); + + assertEquals("golang", response.getHeaderString("X-Syntax-Mode-Ace")); + assertEquals("go", response.getHeaderString("X-Syntax-Mode-Codemirror")); + assertEquals("go", response.getHeaderString("X-Syntax-Mode-Prism")); + } + @Test public void shouldHandleRandomByteFile() throws Exception { mockContentFromResource("JustBytes"); @@ -190,6 +202,7 @@ public class ContentResourceTest { assertEquals("application/octet-stream", response.getHeaderString("Content-Type")); } + @SuppressWarnings("UnstableApiUsage") private void mockContentFromResource(String fileName) throws Exception { URL url = Resources.getResource(fileName); mockContent(fileName, Resources.toByteArray(url)); diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java index f3171f5f90..66ae5a9459 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/DiffResultToDiffResultDtoMapperTest.java @@ -60,11 +60,16 @@ class DiffResultToDiffResultDtoMapperTest { DiffResultDto dto = mapper.mapForRevision(REPOSITORY, createResult(), "123"); List files = dto.getFiles(); - assertAddedFile(files.get(0), "A.java", "abc", "java"); - assertModifiedFile(files.get(1), "B.ts", "abc", "def", "typescript"); - assertDeletedFile(files.get(2), "C.go", "ghi", "golang"); - assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "typescript"); - assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "typescript"); + assertAddedFile(files.get(0), "A.java", "abc", "Java"); + assertModifiedFile(files.get(1), "B.ts", "abc", "def", "TypeScript"); + DiffResultDto.FileDto cGo = files.get(2); + assertDeletedFile(cGo, "C.go", "ghi", "Go"); + assertThat(cGo.getSyntaxModes()) + .containsEntry("ace", "golang") + .containsEntry("codemirror", "go") + .containsEntry("prism", "go"); + assertRenamedFile(files.get(3), "typo.ts", "okay.ts", "def", "fixed", "TypeScript"); + assertCopiedFile(files.get(4), "good.ts", "better.ts", "def", "fixed", "TypeScript"); DiffResultDto.HunkDto hunk = files.get(1).getHunks().get(0); assertHunk(hunk, "@@ -3,4 1,2 @@", 1, 2, 3, 4); diff --git a/scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java b/scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java index a2af3f1d8f..5ae03229a4 100644 --- a/scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java +++ b/scm-webapp/src/test/java/sonia/scm/io/DefaultContentTypeResolverTest.java @@ -24,15 +24,15 @@ package sonia.scm.io; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.nio.charset.StandardCharsets; +import java.util.Map; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.Assertions.assertThat; class DefaultContentTypeResolverTest { @@ -84,42 +84,45 @@ class DefaultContentTypeResolverTest { "% Which does not start with markdown" ); ContentType contentType = contentTypeResolver.resolve("somedoc.md", content.getBytes(StandardCharsets.UTF_8)); - Assertions.assertThat(contentType.getLanguage()).contains("markdown"); + assertThat(contentType.getLanguage()).contains("Markdown"); } @Test void shouldResolveMarkdownWithoutContent() { ContentType contentType = contentTypeResolver.resolve("somedoc.md"); - Assertions.assertThat(contentType.getLanguage()).contains("markdown"); + assertThat(contentType.getLanguage()).contains("Markdown"); } @Test void shouldResolveMarkdownEvenWithDotsInFilename() { ContentType contentType = contentTypeResolver.resolve("somedoc.1.1.md"); - Assertions.assertThat(contentType.getLanguage()).contains("markdown"); + assertThat(contentType.getLanguage()).contains("Markdown"); } @Test void shouldResolveDockerfile() { ContentType contentType = contentTypeResolver.resolve("Dockerfile"); - Assertions.assertThat(contentType.getLanguage()).contains("dockerfile"); + assertThat(contentType.getLanguage()).contains("Dockerfile"); } + } + + @Nested + class GetSyntaxModesTests { @Test - void shouldReturnAceModeIfPresent() { - assertThat(contentTypeResolver.resolve("app.go").getLanguage()).contains("golang"); // codemirror is just go - assertThat(contentTypeResolver.resolve("App.java").getLanguage()).contains("java"); // codemirror is clike + void shouldReturnEmptyMapOfModesWithoutLanguage() { + Map syntaxModes = contentTypeResolver.resolve("app.exe").getSyntaxModes(); + assertThat(syntaxModes).isEmpty(); } @Test - void shouldReturnCodemirrorIfAceModeIsMissing() { - assertThat(contentTypeResolver.resolve("index.ecr").getLanguage()).contains("htmlmixed"); - } - - @Test - void shouldReturnTextIfNoModeIsPresent() { - assertThat(contentTypeResolver.resolve("index.hxml").getLanguage()).contains("text"); + void shouldReturnMapOfModes() { + Map syntaxModes = contentTypeResolver.resolve("app.rs").getSyntaxModes(); + assertThat(syntaxModes) + .containsEntry("ace", "rust") + .containsEntry("codemirror", "rust") + .containsEntry("prism", "rust"); } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java index ade813f995..b9f2576079 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneHighlighterTest.java @@ -47,8 +47,6 @@ import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class LuceneHighlighterTest { - - @Test void shouldHighlightText() throws InvalidTokenOffsetsException, IOException { StandardAnalyzer analyzer = new StandardAnalyzer(); @@ -80,6 +78,15 @@ class LuceneHighlighterTest { ); } + @Test + void shouldNotStartHighlightedFragmentWithLineBreak() throws IOException, InvalidTokenOffsetsException { + String[] snippets = highlightCode("GameOfLife.java", "die"); + + assertThat(snippets).hasSize(1).allSatisfy( + snippet -> assertThat(snippet).doesNotStartWith("\n") + ); + } + @Test void shouldHighlightCodeInTsx() throws IOException, InvalidTokenOffsetsException { String[] snippets = highlightCode("Button.tsx", "inherit");