Prepare search api for different types (#1732)

We introduced a new annotation '@IndexedType' which gets collected by the scm-annotation-processor. All classes which are annotated are index and searchable. This opens the search api for plugins.

scm-webapp/src/test/java/sonia/scm/api/v2/ContentSearchableTypeResolverTest.java

@@ -32,7 +32,7 @@ import java.nio.charset.StandardCharsets;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-class ContentTypeResolverTest {
+class ContentSearchableTypeResolverTest {
 
   @Test
   void shouldResolveMarkdown() {

scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java

@@ -135,11 +135,11 @@ class SearchResourceTest {
       JsonMockHttpResponse response = search("paging", 1, 20);
 
       JsonNode links = response.getContentAsJson().get("_links");
-      assertLink(links, "self", "/v2/search?q=paging&page=1&pageSize=20");
-      assertLink(links, "first", "/v2/search?q=paging&page=0&pageSize=20");
-      assertLink(links, "prev", "/v2/search?q=paging&page=0&pageSize=20");
-      assertLink(links, "next", "/v2/search?q=paging&page=2&pageSize=20");
-      assertLink(links, "last", "/v2/search?q=paging&page=4&pageSize=20");
+      assertLink(links, "self", "/v2/search/string?q=paging&page=1&pageSize=20");
+      assertLink(links, "first", "/v2/search/string?q=paging&page=0&pageSize=20");
+      assertLink(links, "prev", "/v2/search/string?q=paging&page=0&pageSize=20");
+      assertLink(links, "next", "/v2/search/string?q=paging&page=2&pageSize=20");
+      assertLink(links, "last", "/v2/search/string?q=paging&page=4&pageSize=20");
     }
 
     @Test
@@ -220,7 +220,7 @@ class SearchResourceTest {
       searchEngine.search(IndexNames.DEFAULT)
         .start(start)
         .limit(limit)
-        .execute(Repository.class, query)
+        .execute("string", query)
     ).thenReturn(result);
   }
 
@@ -233,7 +233,7 @@ class SearchResourceTest {
   }
 
   private JsonMockHttpResponse search(String query, Integer page, Integer pageSize) throws URISyntaxException, UnsupportedEncodingException {
-    String uri = "/v2/search?q=" + URLEncoder.encode(query, "UTF-8");
+    String uri = "/v2/search/string?q=" + URLEncoder.encode(query, "UTF-8");
     if (page != null) {
       uri += "&page=" + page;
     }

scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java

@@ -61,13 +61,16 @@ class DefaultIndexQueueTest {
   @BeforeEach
   void createQueue() throws IOException {
     directory = new ByteBuffersDirectory();
-    IndexOpener factory = mock(IndexOpener.class);
-    when(factory.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
+    IndexOpener opener = mock(IndexOpener.class);
+    when(opener.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> {
       IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
       config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
       return new IndexWriter(directory, config);
     });
-    SearchEngine engine = new LuceneSearchEngine(factory, new DocumentConverter(), queryBuilderFactory);
+
+    SearchableTypeResolver resolver = new SearchableTypeResolver(Account.class, IndexedNumber.class);
+    LuceneIndexFactory indexFactory = new LuceneIndexFactory(resolver, opener);
+    SearchEngine engine = new LuceneSearchEngine(indexFactory, queryBuilderFactory);
     queue = new DefaultIndexQueue(engine);
   }
 
@@ -111,6 +114,7 @@ class DefaultIndexQueueTest {
   }
 
   @Value
+  @IndexedType
   public static class Account {
     @Indexed
     String username;
@@ -121,6 +125,7 @@ class DefaultIndexQueueTest {
   }
 
   @Value
+  @IndexedType
   public static class IndexedNumber {
     @Indexed
     int value;

scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java

@@ -82,7 +82,7 @@ class LuceneIndexTest {
       index.store(ONE, null, new Storable("Awesome content which should be indexed"));
     }
 
-    assertHits(UID, "one/" + Storable.class.getName(), 1);
+    assertHits(UID, "one/storable", 1);
   }
 
   @Test
@@ -109,7 +109,7 @@ class LuceneIndexTest {
       index.store(ONE, null, new Storable("Some other text"));
     }
 
-    assertHits(TYPE, Storable.class.getName(), 1);
+    assertHits(TYPE, "storable", 1);
   }
 
   @Test
@@ -214,7 +214,8 @@ class LuceneIndexTest {
   }
 
   private LuceneIndex createIndex() throws IOException {
-    return new LuceneIndex(new DocumentConverter(), createWriter());
+    SearchableTypeResolver resolver = new SearchableTypeResolver(Storable.class, OtherStorable.class);
+    return new LuceneIndex(resolver, createWriter());
   }
 
   private IndexWriter createWriter() throws IOException {
@@ -224,12 +225,14 @@ class LuceneIndexTest {
   }
 
   @Value
+  @IndexedType
   private static class Storable {
     @Indexed
     String value;
   }
 
   @Value
+  @IndexedType
   private static class OtherStorable {
     @Indexed
     String value;

scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java

@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import lombok.Getter;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -153,7 +154,7 @@ class LuceneQueryBuilderTest {
     try (IndexWriter writer = writer()) {
       writer.addDocument(personDoc("Dent"));
     }
-    assertThrows(QueryParseException.class, () -> query(String.class, ":~:~"));
+    assertThrows(QueryParseException.class, () -> query(InetOrgPerson.class, ":~:~"));
   }
 
   @Test
@@ -247,8 +248,9 @@ class LuceneQueryBuilderTest {
     QueryResult result;
     try (DirectoryReader reader = DirectoryReader.open(directory)) {
       when(opener.openForRead("default")).thenReturn(reader);
+      SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class);
       LuceneQueryBuilder builder = new LuceneQueryBuilder(
-        opener, "default", new StandardAnalyzer()
+        opener, resolver, "default", new StandardAnalyzer()
       );
       result = builder.repository("cde").execute(Simple.class, "content:awesome");
     }
@@ -480,8 +482,9 @@ class LuceneQueryBuilderTest {
   private QueryResult query(Class<?> type, String queryString, Integer start, Integer limit) throws IOException {
     try (DirectoryReader reader = DirectoryReader.open(directory)) {
       lenient().when(opener.openForRead("default")).thenReturn(reader);
+      SearchableTypeResolver resolver = new SearchableTypeResolver(type);
       LuceneQueryBuilder builder = new LuceneQueryBuilder(
-        opener, "default", new StandardAnalyzer()
+        opener, resolver, "default", new StandardAnalyzer()
       );
       if (start != null) {
         builder.start(start);
@@ -502,14 +505,14 @@ class LuceneQueryBuilderTest {
   private Document simpleDoc(String content) {
     Document document = new Document();
     document.add(new TextField("content", content, Field.Store.YES));
-    document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
     return document;
   }
 
   private Document permissionDoc(String content, String permission) {
     Document document = new Document();
     document.add(new TextField("content", content, Field.Store.YES));
-    document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
     document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES));
     return document;
   }
@@ -517,7 +520,7 @@ class LuceneQueryBuilderTest {
   private Document repositoryDoc(String content, String repository) {
     Document document = new Document();
     document.add(new TextField("content", content, Field.Store.YES));
-    document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES));
     document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES));
     return document;
   }
@@ -529,14 +532,14 @@ class LuceneQueryBuilderTest {
     document.add(new TextField("displayName", displayName, Field.Store.YES));
     document.add(new TextField("carLicense", carLicense, Field.Store.YES));
     document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES));
-    document.add(new StringField(FieldNames.TYPE, InetOrgPerson.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES));
     return document;
   }
 
   private Document personDoc(String lastName) {
     Document document = new Document();
     document.add(new TextField("lastName", lastName, Field.Store.YES));
-    document.add(new StringField(FieldNames.TYPE, Person.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES));
     return document;
   }
 
@@ -549,10 +552,12 @@ class LuceneQueryBuilderTest {
     document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES));
     document.add(new LongPoint("instantValue", instantValue.toEpochMilli()));
     document.add(new StoredField("instantValue", instantValue.toEpochMilli()));
-    document.add(new StringField(FieldNames.TYPE, Types.class.getName(), Field.Store.YES));
+    document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES));
     return document;
   }
 
+  @Getter
+  @IndexedType
   static class Types {
 
     @Indexed
@@ -566,12 +571,16 @@ class LuceneQueryBuilderTest {
 
   }
 
+  @Getter
+  @IndexedType
   static class Person {
 
     @Indexed(defaultQuery = true)
     private String lastName;
   }
 
+  @Getter
+  @IndexedType
   static class InetOrgPerson extends Person {
 
     @Indexed(defaultQuery = true, boost = 2f)
@@ -584,6 +593,8 @@ class LuceneQueryBuilderTest {
     private String carLicense;
   }
 
+  @Getter
+  @IndexedType
   static class Simple {
     @Indexed(defaultQuery = true)
     private String content;

scm-webapp/src/test/java/sonia/scm/search/TypeConvertersTest.java

@@ -35,62 +35,61 @@ import org.apache.lucene.index.IndexableFieldType;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import javax.annotation.Nonnull;
 import java.time.Instant;
 import java.util.function.Consumer;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-class DocumentConverterTest {
-
-  private DocumentConverter documentConverter;
-
-  @BeforeEach
-  void prepare() {
-    documentConverter = new DocumentConverter();
-  }
+class TypeConvertersTest {
 
   @Test
   void shouldConvertPersonToDocument() {
     Person person = new Person("Arthur", "Dent");
 
-    Document document = documentConverter.convert(person);
+    Document document = convert(person);
 
     assertThat(document.getField("firstName").stringValue()).isEqualTo("Arthur");
     assertThat(document.getField("lastName").stringValue()).isEqualTo("Dent");
   }
 
+  @Nonnull
+  private Document convert(Object object) {
+    return TypeConverters.create(object.getClass()).convert(object);
+  }
+
   @Test
   void shouldUseNameFromAnnotation() {
-    Document document = documentConverter.convert(new ParamSample());
+    Document document = convert(new ParamSample());
 
     assertThat(document.getField("username").stringValue()).isEqualTo("dent");
   }
 
   @Test
   void shouldBeIndexedAsTextFieldByDefault() {
-    Document document = documentConverter.convert(new ParamSample());
+    Document document = convert(new ParamSample());
 
     assertThat(document.getField("username")).isInstanceOf(TextField.class);
   }
 
   @Test
   void shouldBeIndexedAsStringField() {
-    Document document = documentConverter.convert(new ParamSample());
+    Document document = convert(new ParamSample());
 
     assertThat(document.getField("searchable")).isInstanceOf(StringField.class);
   }
 
   @Test
   void shouldBeIndexedAsStoredField() {
-    Document document = documentConverter.convert(new ParamSample());
+    Document document = convert(new ParamSample());
 
     assertThat(document.getField("storedOnly")).isInstanceOf(StoredField.class);
   }
 
   @Test
   void shouldIgnoreNonIndexedFields() {
-    Document document = documentConverter.convert(new ParamSample());
+    Document document = convert(new ParamSample());
 
     assertThat(document.getField("notIndexed")).isNull();
   }
@@ -99,7 +98,7 @@ class DocumentConverterTest {
   void shouldSupportInheritance() {
     Account account = new Account("Arthur", "Dent", "arthur@hitchhiker.com");
 
-    Document document = documentConverter.convert(account);
+    Document document = convert(account);
 
     assertThat(document.getField("firstName")).isNotNull();
     assertThat(document.getField("lastName")).isNotNull();
@@ -109,18 +108,18 @@ class DocumentConverterTest {
   @Test
   void shouldFailWithoutGetter() {
     WithoutGetter withoutGetter = new WithoutGetter();
-    assertThrows(NonReadableFieldException.class, () -> documentConverter.convert(withoutGetter));
+    assertThrows(NonReadableFieldException.class, () -> convert(withoutGetter));
   }
 
   @Test
   void shouldFailOnUnsupportedFieldType() {
     UnsupportedFieldType unsupportedFieldType = new UnsupportedFieldType();
-    assertThrows(UnsupportedTypeOfFieldException.class, () -> documentConverter.convert(unsupportedFieldType));
+    assertThrows(UnsupportedTypeOfFieldException.class, () -> convert(unsupportedFieldType));
   }
 
   @Test
   void shouldStoreLongFieldsAsPointAndStoredByDefault() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     assertPointField(document, "longType",
       field -> assertThat(field.numericValue().longValue()).isEqualTo(42L)
@@ -129,7 +128,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldStoreLongFieldAsStored() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     IndexableField field = document.getField("storedOnlyLongType");
     assertThat(field).isInstanceOf(StoredField.class);
@@ -138,7 +137,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldStoreIntegerFieldsAsPointAndStoredByDefault() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     assertPointField(document, "intType",
       field -> assertThat(field.numericValue().intValue()).isEqualTo(42)
@@ -147,7 +146,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldStoreIntegerFieldAsStored() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     IndexableField field = document.getField("storedOnlyIntegerType");
     assertThat(field).isInstanceOf(StoredField.class);
@@ -156,7 +155,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldStoreBooleanFieldsAsStringField() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     IndexableField field = document.getField("boolType");
     assertThat(field).isInstanceOf(StringField.class);
@@ -166,7 +165,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldStoreBooleanFieldAsStored() {
-    Document document = documentConverter.convert(new SupportedTypes());
+    Document document = convert(new SupportedTypes());
 
     IndexableField field = document.getField("storedOnlyBoolType");
     assertThat(field).isInstanceOf(StoredField.class);
@@ -176,7 +175,7 @@ class DocumentConverterTest {
   @Test
   void shouldStoreInstantFieldsAsPointAndStoredByDefault() {
     Instant now = Instant.now();
-    Document document = documentConverter.convert(new DateTypes(now));
+    Document document = convert(new DateTypes(now));
 
     assertPointField(document, "instant",
       field -> assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli())
@@ -186,7 +185,7 @@ class DocumentConverterTest {
   @Test
   void shouldStoreInstantFieldAsStored() {
     Instant now = Instant.now();
-    Document document = documentConverter.convert(new DateTypes(now));
+    Document document = convert(new DateTypes(now));
 
     IndexableField field = document.getField("storedOnlyInstant");
     assertThat(field).isInstanceOf(StoredField.class);
@@ -195,7 +194,7 @@ class DocumentConverterTest {
 
   @Test
   void shouldCreateNoFieldForNullValues() {
-    Document document = documentConverter.convert(new Person("Trillian", null));
+    Document document = convert(new Person("Trillian", null));
 
     assertThat(document.getField("firstName")).isNotNull();
     assertThat(document.getField("lastName")).isNull();
@@ -210,6 +209,7 @@ class DocumentConverterTest {
   }
 
   @Getter
+  @IndexedType
   @AllArgsConstructor
   public static class Person {
     @Indexed
@@ -219,6 +219,7 @@ class DocumentConverterTest {
   }
 
   @Getter
+  @IndexedType
   public static class Account extends Person {
     @Indexed
     private String mail;
@@ -230,6 +231,7 @@ class DocumentConverterTest {
   }
 
   @Getter
+  @IndexedType
   public static class ParamSample {
     @Indexed(name = "username")
     private final String name = "dent";
@@ -243,18 +245,21 @@ class DocumentConverterTest {
     private final String notIndexed = "--";
   }
 
+  @IndexedType
   public static class WithoutGetter {
     @Indexed
     private final String value = "one";
   }
 
   @Getter
+  @IndexedType
   public static class UnsupportedFieldType {
     @Indexed
     private final Object value = "one";
   }
 
   @Getter
+  @IndexedType
   public static class SupportedTypes {
     @Indexed
     private final Long longType = 42L;
@@ -273,6 +278,7 @@ class DocumentConverterTest {
   }
 
   @Getter
+  @IndexedType
   private static class DateTypes {
     @Indexed
     private final Instant instant;