Fix query for enum fields (#1800)

The enum fields were not searchable, because they were stored without analysation or transformation, but if an enum field was searched for within a query, the StandardAnalyzer was used. This means that the enum was stored in the index as an uppercase string, but the query searches for lowercase (the StandardAnalyzer uses a lowercase filter).
To fix this problem we are now using the KeywordAnalyzer for every non tokenized field. The StandardAnalyzer is only used for tokenized fields, which does not specify an other analyzer such code, path or id.
For enum fields we have introduced a new analyzer which uses an uppercase filter by default, this makes it possible to ignore case during search for enum fields.

scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java

@@ -25,16 +25,21 @@
 package sonia.scm.search;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.core.UpperCaseFilterFactory;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
 public class AnalyzerFactory {
 
   public Analyzer create(LuceneSearchableType type) {
-    Analyzer defaultAnalyzer = createDefaultAnalyzer();
+    Analyzer defaultAnalyzer = createNonTokenizedAnalyzer();
 
     Map<String, Analyzer> analyzerMap = new HashMap<>();
     for (LuceneSearchableField field : type.getAllFields()) {
@@ -44,13 +49,42 @@ public class AnalyzerFactory {
     return new PerFieldAnalyzerWrapper(defaultAnalyzer, analyzerMap);
   }
 
-  private Analyzer createDefaultAnalyzer() {
-    return new StandardAnalyzer();
+  private Analyzer createNonTokenizedAnalyzer() {
+    return new KeywordAnalyzer();
   }
 
   private void addFieldAnalyzer(Map<String, Analyzer> analyzerMap, LuceneSearchableField field) {
-    if (field.getAnalyzer() != Indexed.Analyzer.DEFAULT) {
-      analyzerMap.put(field.getName(), new NonNaturalLanguageAnalyzer());
+    Analyzer analyzer = createAnalyzer(field);
+    if (analyzer != null) {
+      analyzerMap.put(field.getName(), analyzer);
+    }
+  }
+
+  private Analyzer createAnalyzer(LuceneSearchableField field) {
+    if (field.isTokenized()) {
+      return createTokenizedAnalyzer(field.getAnalyzer());
+    } else if (field.getType().isEnum()) {
+      return createEnumAnalyzer();
+    } else {
+      return null;
+    }
+  }
+
+  private Analyzer createTokenizedAnalyzer(Indexed.Analyzer analyzer) {
+    if (analyzer == Indexed.Analyzer.DEFAULT) {
+      return new StandardAnalyzer();
+    }
+    return new NonNaturalLanguageAnalyzer();
+  }
+
+  private Analyzer createEnumAnalyzer() {
+    try {
+      return CustomAnalyzer.builder()
+        .withTokenizer(KeywordTokenizerFactory.class)
+        .addTokenFilter(UpperCaseFilterFactory.class)
+        .build();
+    } catch (IOException ex) {
+      throw new IllegalStateException("failed to create enum analyzer", ex);
     }
   }
 

scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java

@@ -48,7 +48,6 @@ import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
-import java.util.Locale;
 
 public class LuceneQueryBuilder<T> extends QueryBuilder<T> {
 
@@ -138,7 +137,7 @@ public class LuceneQueryBuilder<T> extends QueryBuilder<T> {
       throw new NoDefaultQueryFieldsFoundException(searchableType.getType());
     }
 
-    String queryString = queryParams.getQueryString().toLowerCase(Locale.ENGLISH);
+    String queryString = queryParams.getQueryString();
     boolean hasWildcard = containsWildcard(queryString);
 
     BooleanQuery.Builder builder = new BooleanQuery.Builder();

scm-webapp/src/main/java/sonia/scm/search/LuceneSearchableField.java

@@ -43,6 +43,7 @@ class LuceneSearchableField implements SearchableField {
   private final PointsConfig pointsConfig;
   private final Indexed.Analyzer analyzer;
   private final boolean searchable;
+  private final boolean tokenized;
 
   LuceneSearchableField(Field field, Indexed indexed) {
     this.name = name(field, indexed);
@@ -54,6 +55,7 @@ class LuceneSearchableField implements SearchableField {
     this.pointsConfig = IndexableFields.pointConfig(field);
     this.analyzer = indexed.analyzer();
     this.searchable = indexed.type().isSearchable();
+    this.tokenized = indexed.type().isTokenized() && String.class.isAssignableFrom(type);
   }
 
   Object value(Document document) {

scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java

@@ -28,6 +28,7 @@ import org.apache.lucene.index.IndexableField;
 
 import javax.annotation.Nonnull;
 import java.time.Instant;
+import java.util.Locale;
 
 final class ValueExtractors {
 
@@ -54,7 +55,13 @@ final class ValueExtractors {
 
   @SuppressWarnings({"unchecked", "rawtypes"})
   private static ValueExtractor enumExtractor(String name, Class type) {
-    return doc -> Enum.valueOf(type, doc.get(name));
+    return doc -> {
+      String value = doc.get(name);
+      if (value != null) {
+        return Enum.valueOf(type, value.toUpperCase(Locale.ENGLISH));
+      }
+      return null;
+    };
   }
 
   @Nonnull