diff --git a/gradle/changelog/search.yaml b/gradle/changelog/search.yaml new file mode 100644 index 0000000000..20b7b8d661 --- /dev/null +++ b/gradle/changelog/search.yaml @@ -0,0 +1,4 @@ +- type: Added + description: API to index and query objects ([#1727](https://github.com/scm-manager/scm-manager/pull/1727)) +- type: Added + description: Quick search for repositories ([#1727](https://github.com/scm-manager/scm-manager/pull/1727)) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 9a61eb6229..9c96c249cb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -15,6 +15,7 @@ ext { jjwtVersion = '0.11.2' bouncycastleVersion = '1.67' jettyVersion = '9.4.35.v20201120' + luceneVersion = '8.9.0' junitJupiterVersion = '5.7.0' hamcrestVersion = '2.1' @@ -148,6 +149,12 @@ ext { jettyJmx: "org.eclipse.jetty:jetty-jmx:${jettyVersion}", jettyClient: "org.eclipse.jetty:jetty-client:${jettyVersion}", + // search + luceneCore: "org.apache.lucene:lucene-core:${luceneVersion}", + luceneQueryParser: "org.apache.lucene:lucene-queryparser:${luceneVersion}", + luceneHighlighter: "org.apache.lucene:lucene-highlighter:${luceneVersion}", + luceneAnalyzersCommon: "org.apache.lucene:lucene-analyzers-common:${luceneVersion}", + // tests junitJupiterApi: "org.junit.jupiter:junit-jupiter-api:${junitJupiterVersion}", junitJupiterParams: "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}", diff --git a/scm-core/src/main/java/sonia/scm/repository/Repository.java b/scm-core/src/main/java/sonia/scm/repository/Repository.java index 59cabbbe38..70fb56909c 100644 --- a/scm-core/src/main/java/sonia/scm/repository/Repository.java +++ b/scm-core/src/main/java/sonia/scm/repository/Repository.java @@ -31,6 +31,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import sonia.scm.BasicPropertiesAware; import sonia.scm.ModelObject; +import sonia.scm.search.Indexed; import sonia.scm.util.Util; import sonia.scm.util.ValidationUtil; @@ -65,21 +66,28 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per private static final long serialVersionUID = 3486560714961909711L; - private String contact; - private Long creationDate; + private String id; + + @Indexed(defaultQuery = true, boost = 1.25f) + private String namespace; + + @Indexed(defaultQuery = true, boost = 1.5f) + private String name; + @Indexed(type = Indexed.Type.SEARCHABLE) + private String type; + @Indexed(defaultQuery = true, highlighted = true) private String description; + private String contact; + @Indexed + private Long creationDate; + @Indexed + private Long lastModified; @XmlTransient private List healthCheckFailures; - private String id; - private Long lastModified; - private String namespace; - private String name; @XmlElement(name = "permission") private Set permissions = new HashSet<>(); - private String type; private boolean archived; - /** * Constructs a new {@link Repository}. * This constructor is used by JAXB. @@ -162,10 +170,9 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per * @return {@link List} of {@link HealthCheckFailure}s * @since 1.36 */ - @SuppressWarnings("unchecked") public List getHealthCheckFailures() { if (healthCheckFailures == null) { - healthCheckFailures = Collections.EMPTY_LIST; + healthCheckFailures = Collections.emptyList(); } return healthCheckFailures; diff --git a/scm-core/src/main/java/sonia/scm/search/Hit.java b/scm-core/src/main/java/sonia/scm/search/Hit.java new file mode 100644 index 0000000000..b19a69951c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Hit.java @@ -0,0 +1,95 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Value; + +import java.util.Map; + +/** + * Represents an object which matched the search query. + * + * @since 2.21.0 + */ +@Beta +@Value +public class Hit { + + /** + * Id of the matched object. + */ + String id; + + /** + * The score describes how good the match was. + */ + float score; + + /** + * Fields of the matched object. + * Key of the map is the name of the field and the value is either a {@link ValueField} or a {@link HighlightedField}. + */ + Map fields; + + /** + * Base class of hit field types. + */ + @Getter + @AllArgsConstructor(access = AccessLevel.PRIVATE) + public abstract static class Field { + boolean highlighted; + } + + /** + * A field holding a complete value. + */ + @Getter + public static class ValueField extends Field { + Object value; + + public ValueField(Object value) { + super(false); + this.value = value; + } + } + + /** + * A field which consists of fragments that contain a match of the search query. + */ + @Getter + public static class HighlightedField extends Field { + String[] fragments; + + public HighlightedField(String[] fragments) { + super(true); + this.fragments = fragments; + } + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Id.java b/scm-core/src/main/java/sonia/scm/search/Id.java new file mode 100644 index 0000000000..5d47c19252 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Id.java @@ -0,0 +1,158 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import sonia.scm.ModelObject; +import sonia.scm.repository.Repository; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Optional; + +/** + * Describes the id of an indexed object. + * + * @since 2.21.0 + */ +@Beta +@ToString +@EqualsAndHashCode +public final class Id { + + private final String value; + private final String repository; + + private Id(@Nonnull String value, @Nullable String repository) { + this.value = value; + this.repository = repository; + } + + /** + * Returns the string representation of the id without the repository part. + * + * @return string representation without repository. + */ + public String getValue() { + return value; + } + + /** + * Returns the repository id part of the id or an empty optional if the id does not belong to a repository. + * @return repository id or empty + */ + public Optional getRepository() { + return Optional.ofNullable(repository); + } + + /** + * Creates the id with the id of the given repository. + * @param repository repository + * @return id with repository id + */ + public Id withRepository(@Nonnull Repository repository) { + checkRepository(repository); + return withRepository(repository.getId()); + } + + /** + * Creates the id with the given repository id. + * @param repository repository id + * @return id with repository id + */ + public Id withRepository(@Nonnull String repository) { + checkRepository(repository); + return new Id(value, repository); + } + + /** + * Returns the string representation of the id including the repository. + * @return string representation + */ + public String asString() { + if (repository != null) { + return value + "/" + repository; + } + return value; + } + + /** + * Creates a new id. + * + * @param value primary value of the id + * @param others additional values which should be part of the id + * + * @return new id + */ + public static Id of(@Nonnull String value, String... others) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "primary value is required"); + String idValue = value; + if (others.length > 0) { + idValue += ":" + Joiner.on(':').join(others); + } + return new Id(idValue, null); + } + + /** + * Creates a new id for the given repository. + * + * @param repository repository + * @return id of repository + */ + public static Id of(@Nonnull Repository repository) { + checkRepository(repository); + String id = repository.getId(); + checkRepository(id); + return new Id(id, id); + } + + /** + * Creates a new id for the given model object. + * @param object model object + * @param others additional values which should be part of the id + * @return new id from model object + */ + public static Id of(@Nonnull ModelObject object, String... others) { + checkObject(object); + return of(object.getId(), others); + } + + private static void checkRepository(Repository repository) { + Preconditions.checkArgument(repository != null, "repository is required"); + } + + private static void checkRepository(String repository) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(repository), "repository id is required"); + } + + private static void checkObject(@Nonnull ModelObject object) { + Preconditions.checkArgument(object != null, "object is required"); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/Index.java b/scm-core/src/main/java/sonia/scm/search/Index.java new file mode 100644 index 0000000000..bebdfed559 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Index.java @@ -0,0 +1,75 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Can be used to index objects for full text searches. + * + * @since 2.21.0 + */ +@Beta +public interface Index extends AutoCloseable { + + /** + * Store the given object in the index. + * All fields of the object annotated with {@link Indexed} will be stored in the index. + * + * @param id identifier of the object in the index + * @param permission Shiro permission string representing the required permission to see this object as a result + * @param object object to index + * + * @see Indexed + */ + void store(Id id, String permission, Object object); + + /** + * Delete the object with the given id and type from index. + * + * @param id id of object + * @param type type of object + */ + void delete(Id id, Class type); + + /** + * Delete all objects which are related the given repository from index. + * + * @param repositoryId id of repository + */ + void deleteByRepository(String repositoryId); + + /** + * Delete all objects with the given type from index. + * @param type type of objects + */ + void deleteByType(Class type); + + /** + * Close index and commit changes. + */ + @Override + void close(); +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexLog.java b/scm-core/src/main/java/sonia/scm/search/IndexLog.java new file mode 100644 index 0000000000..76ea32d357 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexLog.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Data; +import sonia.scm.xml.XmlInstantAdapter; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.time.Instant; + +/** + * A marker keeping track of when and with which model version an object type was last indexed. + * @since 2.21 + */ +@Beta +@Data +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class IndexLog { + + private int version = 1; + @XmlJavaTypeAdapter(XmlInstantAdapter.class) + private Instant date = Instant.now(); + + public IndexLog() { + } + + public IndexLog(int version) { + this.version = version; + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java b/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java new file mode 100644 index 0000000000..9b3d7e66ef --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.util.Optional; + +/** + * Can be used to mark when a type of object was last indexed and with which version. + * This is useful to detect and mark if a bootstrap index was created for the kind of object + * or if the way how an object is indexed has changed. + * + * @since 2.21.0 + */ +@Beta +public interface IndexLogStore { + + /** + * Log index and version of a type which is now indexed. + * + * @param index name of index + * @param type type which was indexed + * @param version model version + */ + void log(String index, Class type, int version); + + /** + * Returns version and date of the indexed type or an empty object, + * if the object was not indexed at all. + * + * @param index name if index + * @param type type of object + * + * @return log entry or empty + */ + Optional get(String index, Class type); + +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexNames.java b/scm-core/src/main/java/sonia/scm/search/IndexNames.java new file mode 100644 index 0000000000..e264be3e07 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexNames.java @@ -0,0 +1,43 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Names of predefined indexes. + * @since 2.21.0 + */ +@Beta +public final class IndexNames { + + /** + * The default index. + */ + public static final String DEFAULT = "_default"; + + private IndexNames() { + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java new file mode 100644 index 0000000000..a528ee5628 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java @@ -0,0 +1,101 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.util.Locale; + +/** + * Options to configure how things are indexed and searched. + * + * @since 2.21.0 + */ +@Beta +public class IndexOptions { + + private final Type type; + private final Locale locale; + + private IndexOptions(Type type, Locale locale) { + this.type = type; + this.locale = locale; + } + + /** + * Returns the type of the index. + * @return type of index + */ + public Type getType() { + return type; + } + + /** + * Returns the locale of the index content. + * + * @return locale of index content + */ + public Locale getLocale() { + return locale; + } + + /** + * Returns the default index options which should match most of the use cases. + * + * @return default index options + */ + public static IndexOptions defaults() { + return new IndexOptions(Type.GENERIC, Locale.ENGLISH); + } + + /** + * Returns index options for a specific language. + * This options should be used if the content is written in a specific natural language. + * + * @param locale natural language of content + * + * @return options for content in natural language + */ + public static IndexOptions naturalLanguage(Locale locale) { + return new IndexOptions(Type.NATURAL_LANGUAGE, locale); + } + + /** + * Type of indexing. + */ + public enum Type { + + /** + * Not specified content. + */ + GENERIC, + + /** + * Content in natural language. + */ + NATURAL_LANGUAGE; + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexQueue.java b/scm-core/src/main/java/sonia/scm/search/IndexQueue.java new file mode 100644 index 0000000000..2a49c51e6c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexQueue.java @@ -0,0 +1,60 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Queue the work of indexing. + * An index can't be opened in parallel, so the queue coordinates the work of indexing in an asynchronous manner. + * {@link IndexQueue} should be used most of the time to index content. + * + * @since 2.21.0 + */ +@Beta +public interface IndexQueue { + + /** + * Returns an index which queues every change to the content. + * + * @param name name of index + * @param indexOptions options for the index + * + * @return index which queues changes + */ + Index getQueuedIndex(String name, IndexOptions indexOptions); + + /** + * Returns an index which with default options which queues every change to the content. + * @param name name of index + * + * @return index with default options which queues changes + * @see IndexOptions#defaults() + */ + default Index getQueuedIndex(String name) { + return getQueuedIndex(name, IndexOptions.defaults()); + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Indexed.java b/scm-core/src/main/java/sonia/scm/search/Indexed.java new file mode 100644 index 0000000000..f54e7c879a --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Indexed.java @@ -0,0 +1,144 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Mark a field which should be indexed. + * @since 2.21.0 + */ +@Beta +@Documented +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Indexed { + + /** + * Name of the field. + * If not set the name of the annotated field is used. + * + * @return name of field + */ + String name() default ""; + + /** + * Describes how the field is indexed. + * + * @return type of indexing + */ + Type type() default Type.TOKENIZED; + + /** + * {@code true} if this field should be part of default query for this type of object. + * + * @return {@code true} if field is part of default query + */ + boolean defaultQuery() default false; + + /** + * Boost the object if the searched query matches this field. + * Greater than one brings the object further up in the search results. + * Smaller than one devalues the object in the search results. + * + * @return boost score + */ + float boost() default 1f; + + /** + * {@code true} to search the field value for matches and returns fragments with those matches instead of the whole value. + * + * @return {@code true} to return matched fragments + */ + boolean highlighted() default false; + + /** + * Describes how the field is indexed. + */ + enum Type { + /** + * The value of the field is analyzed and split into tokens, which allows searches for parts of the value. + * Tokenization only works for string values. If a field with another type is marked as tokenized, + * the field is indexed as if it was marked as {@link #SEARCHABLE}. + */ + TOKENIZED(true, true, true), + + /** + * The value can only be searched as a whole. + * Numeric fields can also be search as part of a range, + * but strings are only found if the query contains the whole field value. + */ + SEARCHABLE(false, true, true), + + /** + * Value of the field cannot be searched for, but is returned in the result. + */ + STORED_ONLY(false, false, true); + + private final boolean tokenized; + private final boolean searchable; + private final boolean stored; + + Type(boolean tokenized, boolean searchable, boolean stored) { + this.tokenized = tokenized; + this.searchable = searchable; + this.stored = stored; + } + + /** + * Returns {@code true} if the field is tokenized. + * + * @return {@code true} if tokenized + * @see #TOKENIZED + */ + public boolean isTokenized() { + return tokenized; + } + + /** + * Returns {@code true} if the field is searchable. + * + * @return {@code true} if searchable + * @see #SEARCHABLE + */ + public boolean isSearchable() { + return searchable; + } + + /** + * Returns {@code true} if the field is stored. + * @return {@code true} if stored + */ + public boolean isStored() { + return stored; + } + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java b/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java new file mode 100644 index 0000000000..20c11cc5ab --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java @@ -0,0 +1,41 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Exception is thrown if a best guess query is executed + * and the search type does not contain a field which is marked for default query. + * + * @see Indexed#defaultQuery() + * @since 2.21.0 + */ +@Beta +public class NoDefaultQueryFieldsFoundException extends SearchEngineException { + public NoDefaultQueryFieldsFoundException(Class type) { + super("no default query fields defined for " + type); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java b/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java new file mode 100644 index 0000000000..7c1e81a2a0 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java @@ -0,0 +1,113 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Value; +import sonia.scm.repository.Repository; + +import java.util.Optional; + +/** + * Build and execute queries against an index. + * + * @since 2.21.0 + */ +@Beta +public abstract class QueryBuilder { + + private String repositoryId; + private int start = 0; + private int limit = 10; + + /** + * Return only results which are related to the given repository. + * @param repository repository + * @return {@code this} + */ + public QueryBuilder repository(Repository repository) { + return repository(repository.getId()); + } + + /** + * Return only results which are related to the repository with the given id. + * @param repositoryId id of the repository + * @return {@code this} + */ + public QueryBuilder repository(String repositoryId) { + this.repositoryId = repositoryId; + return this; + } + + /** + * The result should start at the given number. + * All matching objects before the given start are skipped. + * @param start start of result + * @return {@code this} + */ + public QueryBuilder start(int start) { + this.start = start; + return this; + } + + /** + * Defines how many hits are returned. + * @param limit limit of hits + * @return {@code this} + */ + public QueryBuilder limit(int limit) { + this.limit = limit; + return this; + } + + /** + * Executes the query and returns the matches. + * + * @param type type of objects which are searched + * @param queryString searched query + * @return result of query + */ + public QueryResult execute(Class type, String queryString){ + return execute(new QueryParams(type, repositoryId, queryString, start, limit)); + } + + protected abstract QueryResult execute(QueryParams queryParams); + + /** + * The searched query and all parameters, which belong to the query. + */ + @Value + static class QueryParams { + Class type; + String repositoryId; + String queryString; + int start; + int limit; + + public Optional getRepositoryId() { + return Optional.ofNullable(repositoryId); + } + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryParseException.java b/scm-core/src/main/java/sonia/scm/search/QueryParseException.java new file mode 100644 index 0000000000..11d943c898 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryParseException.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import sonia.scm.BadRequestException; + +import static sonia.scm.ContextEntry.ContextBuilder.only; + +/** + * Is thrown if a query with invalid syntax was executed. + * + * @since 2.21.0 + */ +@Beta +@SuppressWarnings("java:S110") // large inheritance is ok for exceptions +public class QueryParseException extends BadRequestException { + + private static final String CODE = "5VScek8Xp1"; + + public QueryParseException(String query, String message, Exception cause) { + super(only("query", query), message, cause); + } + + @Override + public String getCode() { + return CODE; + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryResult.java b/scm-core/src/main/java/sonia/scm/search/QueryResult.java new file mode 100644 index 0000000000..28409ab72b --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryResult.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Value; + +import java.util.List; + +/** + * Result of a query. + * @since 2.21.0 + */ +@Beta +@Value +public class QueryResult { + + /** + * Total count of hits, which are matched by the query. + */ + long totalHits; + + /** + * Searched type of object. + */ + Class type; + + /** + * List of hits found by the query. + * The list contains only those hits which are starting at start and they are limit by the given amount. + * @see QueryBuilder + */ + List hits; + +} diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java new file mode 100644 index 0000000000..245db00c05 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java @@ -0,0 +1,83 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * The {@link SearchEngine} is the main entry point for indexing and searching. + * Note that this is kind of a low level api for indexing. + * For non expert indexing the {@link IndexQueue} should be used. + * + * @see IndexQueue + * @since 2.21.0 + */ +@Beta +public interface SearchEngine { + + /** + * Returns the index with the given name and the given options. + * The index is created if it does not exist. + * Warning: Be careful, because an index can't be opened multiple times in parallel. + * If you are not sure how you should index your objects, use the {@link IndexQueue}. + * + * @param name name of the index + * @param options index options + * @return existing index or a new one if none exists + */ + Index getOrCreate(String name, IndexOptions options); + + /** + * Same as {@link #getOrCreate(String, IndexOptions)} with default options. + * + * @param name name of the index + * @return existing index or a new one if none exists + * @see IndexOptions#defaults() + */ + default Index getOrCreate(String name) { + return getOrCreate(name, IndexOptions.defaults()); + } + + /** + * Search the index. + * Returns a {@link QueryBuilder} which allows to query the index. + * + * @param name name of the index + * @param options options for searching the index + * @return query builder + */ + QueryBuilder search(String name, IndexOptions options); + + /** + * Same as {@link #search(String, IndexOptions)} with default options. + * + * @param name name of the index + * @return query builder + * @see IndexOptions#defaults() + */ + default QueryBuilder search(String name) { + return search(name, IndexOptions.defaults()); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java b/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java new file mode 100644 index 0000000000..ccc0afed5f --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Generic exception which could by thrown by any part of the search engine. + * + * @since 2.21.0 + */ +@Beta +public class SearchEngineException extends RuntimeException { + + public SearchEngineException(String message) { + super(message); + } + + public SearchEngineException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-core/src/main/java/sonia/scm/web/VndMediaType.java b/scm-core/src/main/java/sonia/scm/web/VndMediaType.java index 0fe8252b21..dbb310ad7d 100644 --- a/scm-core/src/main/java/sonia/scm/web/VndMediaType.java +++ b/scm-core/src/main/java/sonia/scm/web/VndMediaType.java @@ -95,6 +95,8 @@ public class VndMediaType { public static final String NOTIFICATION_COLLECTION = PREFIX + "notificationCollection" + SUFFIX; + public static final String QUERY_RESULT = PREFIX + "queryResult" + SUFFIX; + private VndMediaType() { } diff --git a/scm-core/src/test/java/sonia/scm/search/IdTest.java b/scm-core/src/test/java/sonia/scm/search/IdTest.java new file mode 100644 index 0000000000..ae38ec2682 --- /dev/null +++ b/scm-core/src/test/java/sonia/scm/search/IdTest.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.Test; +import sonia.scm.ModelObject; +import sonia.scm.repository.Repository; +import sonia.scm.user.User; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class IdTest { + + @Test + void shouldCreateIdFromPrimary() { + Id id = Id.of("one"); + assertThat(id.getValue()).isEqualTo("one"); + } + + @Test + void shouldCreateIdWithoutRepository() { + Id id = Id.of("one"); + assertThat(id.getRepository()).isEmpty(); + } + + @Test + void shouldFailWithoutPrimaryValue() { + assertThrows(IllegalArgumentException.class, () -> Id.of((String) null)); + } + + @Test + void shouldFailWithEmptyPrimaryValue() { + assertThrows(IllegalArgumentException.class, () -> Id.of("")); + } + + @Test + void shouldCreateCombinedValue() { + Id id = Id.of("one", "two", "three"); + assertThat(id.getValue()).isEqualTo("one:two:three"); + } + + @Test + void shouldAddRepositoryId() { + Id id = Id.of("one").withRepository("4211"); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldAddRepository() { + Repository repository = new Repository(); + repository.setId("4211"); + Id id = Id.of("one").withRepository(repository); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldCreateIdFromRepository() { + Repository repository = new Repository(); + repository.setId("4211"); + Id id = Id.of(repository); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldFailWithoutRepository() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((Repository) null)); + } + + @Test + void shouldFailWithoutRepositoryId() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null)); + } + + @Test + void shouldFailWithEmptyRepositoryId() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null)); + } + + @Test + void shouldCreateIdFromModelObject() { + Id id = Id.of(new User("trillian")); + assertThat(id.getValue()).isEqualTo("trillian"); + } + + @Test + void shouldFailWithoutModelObject() { + assertThrows(IllegalArgumentException.class, () -> Id.of((ModelObject) null)); + } + + @Test + void shouldReturnSimpleIdAsString() { + Id id = Id.of("one", "two"); + assertThat(id.asString()).isEqualTo("one:two"); + } + + @Test + void shouldReturnIdWithRepositoryAsString() { + Id id = Id.of("one", "two").withRepository("4211"); + assertThat(id.asString()).isEqualTo("one:two/4211"); + } + +} diff --git a/scm-ui/ui-api/src/index.ts b/scm-ui/ui-api/src/index.ts index 3a007af22c..5cb88941a6 100644 --- a/scm-ui/ui-api/src/index.ts +++ b/scm-ui/ui-api/src/index.ts @@ -56,6 +56,7 @@ export * from "./fileContent"; export * from "./history"; export * from "./contentType"; export * from "./annotations"; +export * from "./search"; export { default as ApiProvider } from "./ApiProvider"; export * from "./ApiProvider"; diff --git a/scm-ui/ui-api/src/search.ts b/scm-ui/ui-api/src/search.ts new file mode 100644 index 0000000000..1773fa348b --- /dev/null +++ b/scm-ui/ui-api/src/search.ts @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { ApiResult, useRequiredIndexLink } from "./base"; +import { QueryResult } from "@scm-manager/ui-types"; +import { apiClient } from "./apiclient"; +import { createQueryString } from "./utils"; +import { useQuery } from "react-query"; + +export type SearchOptions = { + page?: number; + pageSize?: number; +}; + +const defaultSearchOptions: SearchOptions = {}; + +export const useSearch = (query: string, options = defaultSearchOptions): ApiResult => { + const link = useRequiredIndexLink("search"); + + const queryParams: Record = {}; + queryParams.q = query; + if (options.page) { + queryParams.page = options.page.toString(); + } + if (options.pageSize) { + queryParams.pageSize = options.pageSize.toString(); + } + return useQuery( + ["search", query], + () => apiClient.get(`${link}?${createQueryString(queryParams)}`).then((response) => response.json()), + { + enabled: query.length > 1, + } + ); +}; diff --git a/scm-ui/ui-components/src/forms/FilterInput.tsx b/scm-ui/ui-components/src/forms/FilterInput.tsx index 66d4b27fac..6d9d4f4475 100644 --- a/scm-ui/ui-components/src/forms/FilterInput.tsx +++ b/scm-ui/ui-components/src/forms/FilterInput.tsx @@ -81,7 +81,7 @@ const FilterInput: FC = ({ filter, value, testId, placeholder, autoFocus, autoFocus={autoFocus || false} /> - + diff --git a/scm-ui/ui-types/src/Search.ts b/scm-ui/ui-types/src/Search.ts new file mode 100644 index 0000000000..da9bdced1b --- /dev/null +++ b/scm-ui/ui-types/src/Search.ts @@ -0,0 +1,50 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { HalRepresentation, PagedCollection } from "./hal"; + +export type ValueField = { + highlighted: false; + value: unknown; +}; + +export type HighligthedField = { + highlighted: true; + fragments: string[]; +}; + +export type Field = ValueField | HighligthedField; + +export type Hit = HalRepresentation & { + score: number; + fields: { [name: string]: Field }; +}; + +export type HitEmbedded = { + hits: Hit[]; +}; + +export type QueryResult = PagedCollection & { + type: string; +}; diff --git a/scm-ui/ui-types/src/index.ts b/scm-ui/ui-types/src/index.ts index 84a7e872b2..8d6d1e13bf 100644 --- a/scm-ui/ui-types/src/index.ts +++ b/scm-ui/ui-types/src/index.ts @@ -70,3 +70,4 @@ export * from "./Notifications"; export * from "./ApiKeys"; export * from "./PublicKeys"; export * from "./GlobalPermissions"; +export * from "./Search"; diff --git a/scm-ui/ui-webapp/public/locales/de/commons.json b/scm-ui/ui-webapp/public/locales/de/commons.json index 58eed0b2dc..71c8f5ed1d 100644 --- a/scm-ui/ui-webapp/public/locales/de/commons.json +++ b/scm-ui/ui-webapp/public/locales/de/commons.json @@ -150,5 +150,12 @@ "d_plural": "{{count}} Tage", "w": "{{count}} Woche", "w_plural": "{{count}} Wochen" + }, + "search": { + "quickSearch": { + "resultHeading": "Top-Ergebnisse Repositories", + "parseError": "Der Suchstring is ungültig", + "noResults": "Es konnten keine Repositories gefunden werden" + } } } diff --git a/scm-ui/ui-webapp/public/locales/de/groups.json b/scm-ui/ui-webapp/public/locales/de/groups.json index a351a3eda1..0e32d34da5 100644 --- a/scm-ui/ui-webapp/public/locales/de/groups.json +++ b/scm-ui/ui-webapp/public/locales/de/groups.json @@ -26,7 +26,7 @@ } }, "overview": { - "searchGroup": "Gruppe suchen" + "filterGroup": "Gruppen filtern" }, "add-group": { "title": "Gruppe erstellen", diff --git a/scm-ui/ui-webapp/public/locales/de/repos.json b/scm-ui/ui-webapp/public/locales/de/repos.json index ef26670e0b..f2d3641264 100644 --- a/scm-ui/ui-webapp/public/locales/de/repos.json +++ b/scm-ui/ui-webapp/public/locales/de/repos.json @@ -59,7 +59,7 @@ "noRepositories": "Keine Repositories gefunden.", "invalidNamespace": "Keine Repositories gefunden. Möglicherweise existiert der ausgewählte Namespace nicht.", "createButton": "Repository hinzufügen", - "searchRepository": "Repository suchen", + "filterRepositories": "Repositories filtern", "allNamespaces": "Alle Namespaces" }, "create": { diff --git a/scm-ui/ui-webapp/public/locales/de/users.json b/scm-ui/ui-webapp/public/locales/de/users.json index 3936edb43b..aa7185f2c2 100644 --- a/scm-ui/ui-webapp/public/locales/de/users.json +++ b/scm-ui/ui-webapp/public/locales/de/users.json @@ -31,7 +31,7 @@ "createButton": "Benutzer erstellen" }, "overview": { - "searchUser": "Benutzer suchen" + "filterUser": "Benutzer filtern" }, "singleUser": { "errorTitle": "Fehler", diff --git a/scm-ui/ui-webapp/public/locales/en/commons.json b/scm-ui/ui-webapp/public/locales/en/commons.json index 47fbba6a71..3af4588929 100644 --- a/scm-ui/ui-webapp/public/locales/en/commons.json +++ b/scm-ui/ui-webapp/public/locales/en/commons.json @@ -151,5 +151,12 @@ "d_plural": "{{count}} days", "w": "{{count}} week", "w_plural": "{{count}} weeks" + }, + "search": { + "quickSearch": { + "resultHeading": "Top repository results", + "parseError": "Failed to parse query", + "noResults": "Could not find matching repository" + } } } diff --git a/scm-ui/ui-webapp/public/locales/en/groups.json b/scm-ui/ui-webapp/public/locales/en/groups.json index 51b421deb4..f190558845 100644 --- a/scm-ui/ui-webapp/public/locales/en/groups.json +++ b/scm-ui/ui-webapp/public/locales/en/groups.json @@ -26,7 +26,7 @@ } }, "overview": { - "searchGroup": "Search group" + "filterGroup": "Filter groups" }, "add-group": { "title": "Create Group", diff --git a/scm-ui/ui-webapp/public/locales/en/repos.json b/scm-ui/ui-webapp/public/locales/en/repos.json index dd4e3ce9f7..ffa98e167b 100644 --- a/scm-ui/ui-webapp/public/locales/en/repos.json +++ b/scm-ui/ui-webapp/public/locales/en/repos.json @@ -59,7 +59,7 @@ "noRepositories": "No repositories found.", "invalidNamespace": "No repositories found. It's likely that the selected namespace does not exist.", "createButton": "Add Repository", - "searchRepository": "Search repository", + "filterRepositories": "Filter repositories", "allNamespaces": "All namespaces" }, "create": { diff --git a/scm-ui/ui-webapp/public/locales/en/users.json b/scm-ui/ui-webapp/public/locales/en/users.json index ca13a6a1c9..6d2cd20f54 100644 --- a/scm-ui/ui-webapp/public/locales/en/users.json +++ b/scm-ui/ui-webapp/public/locales/en/users.json @@ -45,7 +45,7 @@ } }, "overview": { - "searchUser": "Search user" + "filterUser": "Filter users" }, "createUser": { "title": "Create User", diff --git a/scm-ui/ui-webapp/src/containers/NavigationBar.tsx b/scm-ui/ui-webapp/src/containers/NavigationBar.tsx index e5ff8cabae..11c2a96f28 100644 --- a/scm-ui/ui-webapp/src/containers/NavigationBar.tsx +++ b/scm-ui/ui-webapp/src/containers/NavigationBar.tsx @@ -27,6 +27,7 @@ import classNames from "classnames"; import styled from "styled-components"; import { devices, Logo, PrimaryNavigation } from "@scm-manager/ui-components"; import Notifications from "./Notifications"; +import OmniSearch from "./OmniSearch"; import LogoutButton from "./LogoutButton"; import LoginButton from "./LoginButton"; @@ -44,6 +45,7 @@ const StyledNavBar = styled.nav` position: absolute; top: 0; left: 52px; + flex-direction: row-reverse; } } @@ -142,6 +144,7 @@ const NavigationBar: FC = ({ links }) => {
+
diff --git a/scm-ui/ui-webapp/src/containers/Notifications.tsx b/scm-ui/ui-webapp/src/containers/Notifications.tsx index 4b6e28ed00..b4d239cb3f 100644 --- a/scm-ui/ui-webapp/src/containers/Notifications.tsx +++ b/scm-ui/ui-webapp/src/containers/Notifications.tsx @@ -60,13 +60,18 @@ const DropDownMenu = styled.div` min-width: 35rem; @media screen and (max-width: ${devices.mobile.width}px) { - min-width: 25rem; + min-width: 20rem; } @media screen and (max-width: ${devices.desktop.width}px) { margin-right: 1rem; } + @media screen and (min-width: ${devices.desktop.width}px) { + right: 0; + left: auto; + } + &:before { position: absolute; content: ""; @@ -338,15 +343,12 @@ const Notifications: FC = ({ className }) => { return () => window.removeEventListener("click", close); }, []); - const isMobileView = window.matchMedia(`(max-width: ${devices.desktop.width - 1}px)`).matches; - return ( <>
{ + const namespace = (hit.fields["namespace"] as ValueField).value as string; + const name = (hit.fields["name"] as ValueField).value as string; + return `${namespace}/${name}`; +}; + +type HitsProps = { + hits: Hit[]; + index: number; + clear: () => void; +}; + +const QuickSearchNotification: FC = ({ children }) =>
{children}
; + +const EmptyHits = () => { + const [t] = useTranslation("commons"); + return ( + + {t("search.quickSearch.noResults")} + + ); +}; + +type ErrorProps = { + error: Error; +}; + +const ParseErrorNotification: FC = () => { + const [t] = useTranslation("commons"); + // TODO add link to query syntax page/modal + return ( + + {t("search.quickSearch.parseError")} + + ); +}; + +const isBackendError = (error: Error | BackendError): error is BackendError => { + return (error as BackendError).errorCode !== undefined; +}; + +const SearchErrorNotification: FC = ({ error }) => { + // 5VScek8Xp1 is the id of sonia.scm.search.QueryParseException + if (isBackendError(error) && error.errorCode === "5VScek8Xp1") { + return ; + } + return ( + + + + ); +}; + +const ResultHeading = styled.div` + border-bottom: 1px solid lightgray; + margin: 0 0.5rem; + padding: 0.375rem 0.5rem; +`; + +const Hits: FC = ({ hits, index, clear }) => { + const id = useCallback(namespaceAndName, [hits]); + const [t] = useTranslation("commons"); + + if (hits.length === 0) { + return ; + } + + return ( +
+ {t("search.quickSearch.resultHeading")} + {hits.map((hit, idx) => ( +
e.preventDefault()} onClick={clear}> + + {id(hit)} + +
+ ))} +
+ ); +}; + +const useKeyBoardNavigation = (clear: () => void, hits?: Array) => { + const [index, setIndex] = useState(-1); + const history = useHistory(); + useEffect(() => { + setIndex(-1); + }, [hits]); + + const onKeyDown = (e: KeyboardEvent) => { + // We use e.which, because ie 11 does not support e.code + // https://caniuse.com/keyboardevent-code + switch (e.which) { + case 40: // e.code: ArrowDown + if (hits) { + setIndex((idx) => { + if (idx + 1 < hits.length) { + return idx + 1; + } + return idx; + }); + } + break; + case 38: // e.code: ArrowUp + if (hits) { + setIndex((idx) => { + if (idx > 0) { + return idx - 1; + } + return idx; + }); + } + break; + case 13: // e.code: Enter + if (hits && index >= 0) { + const hit = hits[index]; + history.push(`/repo/${namespaceAndName(hit)}`); + clear(); + } + break; + case 27: // e.code: Escape + clear(); + break; + } + }; + + return { + onKeyDown, + index, + }; +}; + +const useDebounce = (value: string, delay: number) => { + const [debouncedValue, setDebouncedValue] = useState(value); + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedValue(value); + }, delay); + return () => { + clearTimeout(handler); + }; + }, [value, delay]); + return debouncedValue; +}; + +const useShowResultsOnFocus = () => { + const [showResults, setShowResults] = useState(false); + return { + showResults, + onClick: (e: MouseEvent) => e.stopPropagation(), + onFocus: () => setShowResults(true), + onBlur: () => setShowResults(false) + }; +}; + +const OmniSearch: FC = () => { + const [query, setQuery] = useState(""); + const debouncedQuery = useDebounce(query, 250); + const { data, isLoading, error } = useSearch(debouncedQuery, { pageSize: 5 }); + const { showResults, ...handlers } = useShowResultsOnFocus(); + + const clearQuery = () => { + setQuery(""); + }; + const { onKeyDown, index } = useKeyBoardNavigation(clearQuery, data?._embedded.hits); + + return ( + +
+
+
+ setQuery(e.target.value)} + onKeyDown={onKeyDown} + value={query} + {...handlers} + /> + {isLoading ? null : ( + + + + )} +
+
+ {error ? : null} + {!error && data ? : null} +
+
+
+ + ); +}; + +const OmniSearchGuard: FC = ({ links }) => { + if (!links.search) { + return null; + } + return ; +}; + +export default OmniSearchGuard; diff --git a/scm-ui/ui-webapp/src/groups/containers/Groups.tsx b/scm-ui/ui-webapp/src/groups/containers/Groups.tsx index 160f1279e3..e42f0e028d 100644 --- a/scm-ui/ui-webapp/src/groups/containers/Groups.tsx +++ b/scm-ui/ui-webapp/src/groups/containers/Groups.tsx @@ -72,7 +72,7 @@ const Groups: FC = () => { showCreateButton={canCreateGroups} link="groups" label={t("create-group-button.label")} - searchPlaceholder={t("overview.searchGroup")} + searchPlaceholder={t("overview.filterGroup")} /> diff --git a/scm-ui/ui-webapp/src/repos/containers/Overview.tsx b/scm-ui/ui-webapp/src/repos/containers/Overview.tsx index bd6bfdf5d0..9fff58af5b 100644 --- a/scm-ui/ui-webapp/src/repos/containers/Overview.tsx +++ b/scm-ui/ui-webapp/src/repos/containers/Overview.tsx @@ -153,7 +153,7 @@ const Overview: FC = () => { createLink="/repos/create/" label={t("overview.createButton")} testId="repository-overview" - searchPlaceholder={t("overview.searchRepository")} + searchPlaceholder={t("overview.filterRepositories")} /> ) : null} diff --git a/scm-ui/ui-webapp/src/users/containers/Users.tsx b/scm-ui/ui-webapp/src/users/containers/Users.tsx index bf6edbbf77..b15a174945 100644 --- a/scm-ui/ui-webapp/src/users/containers/Users.tsx +++ b/scm-ui/ui-webapp/src/users/containers/Users.tsx @@ -79,7 +79,7 @@ const Users: FC = () => { showCreateButton={canAddUsers} link="users" label={t("users.createButton")} - searchPlaceholder={t("overview.searchUser")} + searchPlaceholder={t("overview.filterUser")} /> diff --git a/scm-webapp/build.gradle b/scm-webapp/build.gradle index 7ee90c3c31..474cf784e0 100644 --- a/scm-webapp/build.gradle +++ b/scm-webapp/build.gradle @@ -120,6 +120,11 @@ dependencies { // metrics implementation libraries.micrometerExtra + implementation libraries.luceneCore + implementation libraries.luceneQueryParser + implementation libraries.luceneHighlighter + implementation libraries.luceneAnalyzersCommon + // lombok compileOnly libraries.lombok testCompileOnly libraries.lombok diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java index a58430efa8..bc18b97df9 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.api.v2.resources; import de.otto.edison.hal.Embedded; @@ -31,6 +31,7 @@ import lombok.Getter; import lombok.Setter; @Getter @Setter +@SuppressWarnings("squid:S2160") // we do not need equals for dto class CollectionDto extends HalRepresentation { private int page; diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java new file mode 100644 index 0000000000..186967388a --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.HalRepresentation; +import de.otto.edison.hal.Links; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import sonia.scm.search.Hit; + +import java.util.Map; + +@Getter +@Setter +@NoArgsConstructor +@SuppressWarnings("java:S2160") // we do not need this for dto +public class HitDto extends HalRepresentation { + + private float score; + private Map fields; + + public HitDto(Links links, Embedded embedded) { + super(links, embedded); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java index 826b7558a9..a01cce0902 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java @@ -131,6 +131,8 @@ public class IndexDtoGenerator extends HalAppenderMapper { builder.single(link("namespaceStrategies", resourceLinks.namespaceStrategies().self())); builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self())); builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}"))); + + builder.single(link("search", resourceLinks.search().search())); } else { builder.single(link("login", resourceLinks.authentication().jsonLogin())); } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java index 06dbcff502..7d973d0d1a 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java @@ -91,5 +91,7 @@ public class MapperModule extends AbstractModule { bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class)); bind(RepositoryImportDtoToRepositoryImportParametersMapper.class).to(Mappers.getMapperClass(RepositoryImportDtoToRepositoryImportParametersMapper.class)); + + bind(QueryResultMapper.class).to(Mappers.getMapperClass(QueryResultMapper.class)); } } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java new file mode 100644 index 0000000000..f99865d9b5 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java @@ -0,0 +1,42 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.Links; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@SuppressWarnings("squid:S2160") // we do not need equals for dto +public class QueryResultDto extends CollectionDto { + + private Class type; + + QueryResultDto(Links links, Embedded embedded) { + super(links, embedded); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java new file mode 100644 index 0000000000..fa3a1e088c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java @@ -0,0 +1,120 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import com.damnhandy.uri.template.UriTemplate; +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.Links; +import de.otto.edison.hal.paging.NumberedPaging; +import de.otto.edison.hal.paging.PagingRel; +import org.mapstruct.AfterMapping; +import org.mapstruct.Context; +import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; +import org.mapstruct.ObjectFactory; +import sonia.scm.search.Hit; +import sonia.scm.search.QueryResult; +import sonia.scm.web.EdisonHalAppender; + +import javax.annotation.Nonnull; +import java.util.EnumSet; +import java.util.List; +import java.util.stream.Collectors; + +import static com.damnhandy.uri.template.UriTemplate.fromTemplate; +import static de.otto.edison.hal.Links.linkingTo; +import static de.otto.edison.hal.paging.NumberedPaging.zeroBasedNumberedPaging; + +@Mapper +public abstract class QueryResultMapper extends HalAppenderMapper { + + public abstract QueryResultDto map(@Context SearchParameters params, QueryResult result); + + @AfterMapping + void setPageValues(@MappingTarget QueryResultDto dto, QueryResult result, @Context SearchParameters params) { + int totalHits = (int) result.getTotalHits(); + dto.setPageTotal(computePageTotal(totalHits, params.getPageSize())); + dto.setPage(params.getPage()); + } + + @Nonnull + @ObjectFactory + QueryResultDto createDto(@Context SearchParameters params, QueryResult result) { + int totalHits = (int) result.getTotalHits(); + Links.Builder links = links(params, totalHits); + Embedded.Builder embedded = hits(result); + applyEnrichers(new EdisonHalAppender(links, embedded), result); + return new QueryResultDto(links.build(), embedded.build()); + } + + @Nonnull + private QueryResultDto createDto(SearchParameters params, QueryResult result, int totalHits) { + Links.Builder links = links(params, totalHits); + Embedded.Builder embedded = hits(result); + applyEnrichers(new EdisonHalAppender(links, embedded), result); + return new QueryResultDto(links.build(), embedded.build()); + } + + private Links.Builder links(SearchParameters params, int totalHits) { + NumberedPaging paging = zeroBasedNumberedPaging(params.getPage(), params.getPageSize(), totalHits); + + UriTemplate uriTemplate = fromTemplate(params.getSelfLink() + "{?q,page,pageSize}"); + uriTemplate.set("q", params.getQuery()); + + return linkingTo() + .with(paging.links( + uriTemplate, + EnumSet.allOf(PagingRel.class)) + ); + } + + @Nonnull + private Embedded.Builder hits(QueryResult result) { + List hits = result.getHits() + .stream() + .map(hit -> map(result, hit)) + .collect(Collectors.toList()); + return Embedded.embeddedBuilder().with("hits", hits); + } + + @ObjectFactory + protected HitDto createHitDto(@Context QueryResult queryResult, Hit hit) { + Links.Builder links = linkingTo(); + Embedded.Builder embedded = Embedded.embeddedBuilder(); + + applyEnrichers(new EdisonHalAppender(links, embedded), hit, queryResult); + return new HitDto(links.build(), embedded.build()); + } + + private int computePageTotal(int totalHits, int pageSize) { + if (totalHits % pageSize > 0) { + return totalHits / pageSize + 1; + } else { + return totalHits / pageSize; + } + } + + protected abstract HitDto map(@Context QueryResult queryResult, Hit hit); +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java index 726e33eb72..468e53c709 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java @@ -1113,6 +1113,23 @@ class ResourceLinks { } } + public SearchLinks search() { + return new SearchLinks(scmPathInfoStore.get()); + } + + public static class SearchLinks { + + private final LinkBuilder searchLinkBuilder; + + SearchLinks(ScmPathInfo pathInfo) { + this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class); + } + + public String search() { + return searchLinkBuilder.method("search").parameters().href(); + } + } + public InitialAdminAccountLinks initialAdminAccount() { return new InitialAdminAccountLinks(new LinkBuilder(scmPathInfoStore.get(), InitializationResource.class, AdminAccountStartupResource.class)); } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java new file mode 100644 index 0000000000..4728a2c09d --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import lombok.Getter; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.Size; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.UriInfo; + +@Getter +public class SearchParameters { + + @Context + private UriInfo uriInfo; + + @Size(min = 2) + @QueryParam("q") + private String query; + + @Min(0) + @QueryParam("page") + @DefaultValue("0") + private int page = 0; + + @Min(1) + @Max(100) + @QueryParam("pageSize") + @DefaultValue("10") + private int pageSize = 10; + + String getSelfLink() { + return uriInfo.getAbsolutePath().toASCIIString(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java new file mode 100644 index 0000000000..2defad8aaa --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java @@ -0,0 +1,111 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import sonia.scm.repository.Repository; +import sonia.scm.search.IndexNames; +import sonia.scm.search.QueryResult; +import sonia.scm.search.SearchEngine; +import sonia.scm.web.VndMediaType; + +import javax.inject.Inject; +import javax.validation.Valid; +import javax.ws.rs.BeanParam; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; + +@Path(SearchResource.PATH) +@OpenAPIDefinition(tags = { + @Tag(name = "Search", description = "Search related endpoints") +}) +public class SearchResource { + + static final String PATH = "v2/search"; + + private final SearchEngine engine; + private final QueryResultMapper mapper; + + @Inject + public SearchResource(SearchEngine engine, QueryResultMapper mapper) { + this.engine = engine; + this.mapper = mapper; + } + + @GET + @Path("") + @Produces(VndMediaType.QUERY_RESULT) + @Operation( + summary = "Query result", + description = "Returns a collection of matched hits.", + tags = "Search", + operationId = "search_query" + ) + @ApiResponse( + responseCode = "200", + description = "success", + content = @Content( + mediaType = VndMediaType.QUERY_RESULT, + schema = @Schema(implementation = QueryResultDto.class) + ) + ) + @ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials") + @ApiResponse( + responseCode = "500", + description = "internal server error", + content = @Content( + mediaType = VndMediaType.ERROR_TYPE, + schema = @Schema(implementation = ErrorDto.class) + )) + @Parameter( + name = "query", + description = "The search expression", + required = true + ) + @Parameter( + name = "page", + description = "The requested page number of the search results (zero based, defaults to 0)" + ) + @Parameter( + name = "pageSize", + description = "The maximum number of results per page (defaults to 10)" + ) + public QueryResultDto search(@Valid @BeanParam SearchParameters params) { + QueryResult result = engine.search(IndexNames.DEFAULT) + .start(params.getPage() * params.getPageSize()) + .limit(params.getPageSize()) + .execute(Repository.class, params.getQuery()); + + return mapper.map(params, result); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java index 98445a62a6..baf3923236 100644 --- a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java +++ b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java @@ -98,6 +98,12 @@ import sonia.scm.repository.xml.XmlRepositoryDAO; import sonia.scm.repository.xml.XmlRepositoryRoleDAO; import sonia.scm.schedule.CronScheduler; import sonia.scm.schedule.Scheduler; +import sonia.scm.search.DefaultIndexLogStore; +import sonia.scm.search.DefaultIndexQueue; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexQueue; +import sonia.scm.search.LuceneSearchEngine; +import sonia.scm.search.SearchEngine; import sonia.scm.security.AccessTokenCookieIssuer; import sonia.scm.security.AuthorizationChangedEventProducer; import sonia.scm.security.ConfigurableLoginAttemptHandler; @@ -279,6 +285,11 @@ class ScmServletModule extends ServletModule { bind(NotificationSender.class).to(DefaultNotificationSender.class); bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class); + + // bind search stuff + bind(IndexQueue.class, DefaultIndexQueue.class); + bind(SearchEngine.class, LuceneSearchEngine.class); + bind(IndexLogStore.class, DefaultIndexLogStore.class); } private void bind(Class clazz, Class defaultImplementation) { diff --git a/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java b/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java new file mode 100644 index 0000000000..7bb8010729 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.repository; + +import com.github.legman.Subscribe; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.HandlerEventType; +import sonia.scm.plugin.Extension; +import sonia.scm.search.Id; +import sonia.scm.search.Index; +import sonia.scm.search.IndexLog; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexNames; +import sonia.scm.search.IndexQueue; +import sonia.scm.web.security.AdministrationContext; +import sonia.scm.web.security.PrivilegedAction; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.util.Optional; + +@Extension +@Singleton +public class IndexUpdateListener implements ServletContextListener { + + private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class); + + @VisibleForTesting + static final int INDEX_VERSION = 1; + + private final AdministrationContext administrationContext; + private final IndexQueue queue; + private final IndexLogStore indexLogStore; + + @Inject + public IndexUpdateListener(AdministrationContext administrationContext, IndexQueue queue, IndexLogStore indexLogStore) { + this.administrationContext = administrationContext; + this.queue = queue; + this.indexLogStore = indexLogStore; + } + + @Subscribe(async = false) + public void handleEvent(RepositoryEvent event) { + HandlerEventType type = event.getEventType(); + if (type.isPost()) { + updateIndex(type, event.getItem()); + } + } + + + private void updateIndex(HandlerEventType type, Repository repository) { + try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) { + if (type == HandlerEventType.DELETE) { + index.deleteByRepository(repository.getId()); + } else { + store(index, repository); + } + } + } + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + Optional indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class); + if (!indexLog.isPresent()) { + LOG.debug("could not find log entry for repository index, start reindexing of all repositories"); + administrationContext.runAsAdmin(ReIndexAll.class); + indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION); + } + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + // we have nothing to destroy + } + + private static void store(Index index, Repository repository) { + index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); + } + + static class ReIndexAll implements PrivilegedAction { + + private final RepositoryManager repositoryManager; + private final IndexQueue queue; + + @Inject + public ReIndexAll(RepositoryManager repositoryManager, IndexQueue queue) { + this.repositoryManager = repositoryManager; + this.queue = queue; + } + + @Override + public void run() { + try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) { + for (Repository repository : repositoryManager.getAll()) { + store(index, repository); + } + } + } + + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java b/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java new file mode 100644 index 0000000000..c42bee93ad --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java @@ -0,0 +1,62 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.de.GermanAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.es.SpanishAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +import javax.annotation.Nonnull; + +public class AnalyzerFactory { + + @Nonnull + public Analyzer create(IndexOptions options) { + if (options.getType() == IndexOptions.Type.NATURAL_LANGUAGE) { + return createNaturalLanguageAnalyzer(options.getLocale().getLanguage()); + } + return createDefaultAnalyzer(); + } + + private Analyzer createDefaultAnalyzer() { + return new StandardAnalyzer(); + } + + private Analyzer createNaturalLanguageAnalyzer(String lang) { + switch (lang) { + case "en": + return new EnglishAnalyzer(); + case "de": + return new GermanAnalyzer(); + case "es": + return new SpanishAnalyzer(); + default: + return createDefaultAnalyzer(); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java new file mode 100644 index 0000000000..7f329d2c41 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java @@ -0,0 +1,59 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import sonia.scm.store.DataStore; +import sonia.scm.store.DataStoreFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class DefaultIndexLogStore implements IndexLogStore { + + private final DataStore dataStore; + + @Inject + public DefaultIndexLogStore(DataStoreFactory dataStoreFactory) { + this.dataStore = dataStoreFactory.withType(IndexLog.class).withName("index-log").build(); + } + + @Override + public void log(String index,Class type, int version) { + String id = id(index, type); + dataStore.put(id, new IndexLog(version)); + } + + private String id(String index, Class type) { + return index + "_" + type.getName(); + } + + @Override + public Optional get(String index, Class type) { + String id = id(index, type); + return dataStore.getOptional(id); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java new file mode 100644 index 0000000000..37e71e7b5e --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java @@ -0,0 +1,77 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.VisibleForTesting; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +@Singleton +public class DefaultIndexQueue implements IndexQueue, Closeable { + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + private final AtomicLong size = new AtomicLong(0); + + private final SearchEngine searchEngine; + + @Inject + public DefaultIndexQueue(SearchEngine searchEngine) { + this.searchEngine = searchEngine; + } + + @Override + public Index getQueuedIndex(String name, IndexOptions indexOptions) { + return new QueuedIndex(this, name, indexOptions); + } + + public SearchEngine getSearchEngine() { + return searchEngine; + } + + void enqueue(IndexQueueTaskWrapper task) { + size.incrementAndGet(); + executor.execute(() -> { + task.run(); + size.decrementAndGet(); + }); + } + + @VisibleForTesting + long getSize() { + return size.get(); + } + + @Override + public void close() throws IOException { + executor.shutdown(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java b/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java new file mode 100644 index 0000000000..095f380c9b --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java @@ -0,0 +1,144 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexableField; + +import javax.inject.Singleton; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static java.util.Collections.emptySet; + +@Singleton +public class DocumentConverter { + + private final Map, TypeConverter> typeConverter = new ConcurrentHashMap<>(); + + Document convert(Object object) { + TypeConverter converter = typeConverter.computeIfAbsent(object.getClass(), this::createTypeConverter); + try { + return converter.convert(object); + } catch (IllegalAccessException | InvocationTargetException ex) { + throw new SearchEngineException("failed to create document", ex); + } + } + + private TypeConverter createTypeConverter(Class type) { + List fieldConverters = new ArrayList<>(); + collectFields(fieldConverters, type); + return new TypeConverter(fieldConverters); + } + + private void collectFields(List fieldConverters, Class type) { + Class parent = type.getSuperclass(); + if (parent != null) { + collectFields(fieldConverters, parent); + } + for (Field field : type.getDeclaredFields()) { + Indexed indexed = field.getAnnotation(Indexed.class); + if (indexed != null) { + IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed); + Method getter = findGetter(type, field); + fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory)); + } + } + } + + private Method findGetter(Class type, Field field) { + String name = createGetterName(field); + try { + return type.getMethod(name); + } catch (NoSuchMethodException ex) { + throw new NonReadableFieldException("could not find getter for field", ex); + } + } + + private String createGetterName(Field field) { + String fieldName = field.getName(); + String prefix = "get"; + if (field.getType() == Boolean.TYPE) { + prefix = "is"; + } + return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1); + } + + private static class TypeConverter { + + private final List fieldConverters; + + private TypeConverter(List fieldConverters) { + this.fieldConverters = fieldConverters; + } + + public Document convert(Object object) throws IllegalAccessException, InvocationTargetException { + Document document = new Document(); + for (FieldConverter fieldConverter : fieldConverters) { + for (IndexableField field : fieldConverter.convert(object)) { + document.add(field); + } + } + return document; + } + } + + private static class FieldConverter { + + private final Method getter; + private final IndexableFieldFactory fieldFactory; + private final String name; + + private FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) { + this.getter = getter; + this.fieldFactory = fieldFactory; + this.name = createName(field, indexed); + } + + private String createName(Field field, Indexed indexed) { + String nameFromAnnotation = indexed.name(); + if (Strings.isNullOrEmpty(nameFromAnnotation)) { + return field.getName(); + } + return nameFromAnnotation; + } + + Iterable convert(Object object) throws IllegalAccessException, InvocationTargetException { + Object value = getter.invoke(object); + if (value != null) { + return fieldFactory.create(name, value); + } + return emptySet(); + } + + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java new file mode 100644 index 0000000000..9c11baa34c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java @@ -0,0 +1,35 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +final class FieldNames { + private FieldNames(){} + + static final String UID = "_uid"; + static final String ID = "_id"; + static final String TYPE = "_type"; + static final String REPOSITORY = "_repository"; + static final String PERMISSION = "_permission"; +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java b/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java new file mode 100644 index 0000000000..b24ac37432 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java @@ -0,0 +1,65 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import sonia.scm.SCMContextProvider; + +import javax.inject.Inject; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class IndexOpener { + + private final Path directory; + private final AnalyzerFactory analyzerFactory; + + @Inject + public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) { + directory = context.resolve(Paths.get("index")); + this.analyzerFactory = analyzerFactory; + } + + public IndexReader openForRead(String name) throws IOException { + return DirectoryReader.open(directory(name)); + } + + public IndexWriter openForWrite(String name, IndexOptions options) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options)); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory(name), config); + } + + private Directory directory(String name) throws IOException { + return FSDirectory.open(directory.resolve(name)); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java new file mode 100644 index 0000000000..2ddb227375 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +@FunctionalInterface +public interface IndexQueueTask { + + void updateIndex(Index index); + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java new file mode 100644 index 0000000000..bd09c03780 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class IndexQueueTaskWrapper implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class); + + private final SearchEngine searchEngine; + private final String indexName; + private final IndexOptions options; + private final Iterable tasks; + + IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable tasks) { + this.searchEngine = searchEngine; + this.indexName = indexName; + this.options = options; + this.tasks = tasks; + } + + @Override + public void run() { + try (Index index = searchEngine.getOrCreate(this.indexName, options)) { + for (IndexQueueTask task : tasks) { + task.updateIndex(index); + } + } catch (Exception e) { + LOG.warn("failure during execution of index task for index {}", indexName, e); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java b/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java new file mode 100644 index 0000000000..253361e981 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.IndexableField; + +@FunctionalInterface +interface IndexableFieldFactory { + Iterable create(String name, Object value); +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java b/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java new file mode 100644 index 0000000000..cd662edb35 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java @@ -0,0 +1,172 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; +import java.text.DecimalFormat; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.singleton; +import static sonia.scm.search.TypeCheck.isBoolean; +import static sonia.scm.search.TypeCheck.isInstant; +import static sonia.scm.search.TypeCheck.isInteger; +import static sonia.scm.search.TypeCheck.isLong; + +class IndexableFields { + private IndexableFields() { + } + + static PointsConfig pointConfig(Field field) { + Class type = field.getType(); + if (isLong(type) || isInstant(type)) { + return new PointsConfig(new DecimalFormat(), Long.class); + } else if (isInteger(type)) { + return new PointsConfig(new DecimalFormat(), Integer.class); + } + return null; + } + + static IndexableFieldFactory create(Field field, Indexed indexed) { + Class fieldType = field.getType(); + Indexed.Type indexType = indexed.type(); + if (fieldType == String.class) { + return new StringFieldFactory(indexType); + } else if (isLong(fieldType)) { + return new LongFieldFactory(indexType); + } else if (isInteger(fieldType)) { + return new IntegerFieldFactory(indexType); + } else if (isBoolean(fieldType)) { + return new BooleanFieldFactory(indexType); + } else if (isInstant(fieldType)) { + return new InstantFieldFactory(indexType); + } else { + throw new UnsupportedTypeOfFieldException(fieldType, field.getName()); + } + } + + private static class StringFieldFactory implements IndexableFieldFactory { + private final Indexed.Type type; + + private StringFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + String stringValue = (String) value; + if (type.isTokenized()) { + return singleton(new TextField(name, stringValue, Store.YES)); + } else if (type.isSearchable()) { + return singleton(new StringField(name, stringValue, Store.YES)); + } else { + return singleton(new StoredField(name, stringValue)); + } + } + } + + private static class LongFieldFactory implements IndexableFieldFactory { + + private final Indexed.Type type; + + private LongFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Long longValue = (Long) value; + List fields = new ArrayList<>(); + if (type.isSearchable()) { + fields.add(new LongPoint(name, longValue)); + } + fields.add(new StoredField(name, longValue)); + return Collections.unmodifiableList(fields); + } + } + + private static class IntegerFieldFactory implements IndexableFieldFactory { + + private final Indexed.Type type; + + private IntegerFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Integer integerValue = (Integer) value; + List fields = new ArrayList<>(); + if (type.isSearchable()) { + fields.add(new IntPoint(name, integerValue)); + } + fields.add(new StoredField(name, integerValue)); + return Collections.unmodifiableList(fields); + } + } + + private static class BooleanFieldFactory implements IndexableFieldFactory { + private final Indexed.Type type; + + private BooleanFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Boolean booleanValue = (Boolean) value; + if (type.isSearchable()) { + return singleton(new StringField(name, booleanValue.toString(), Store.YES)); + } else { + return singleton(new StoredField(name, booleanValue.toString())); + } + } + } + + private static class InstantFieldFactory extends LongFieldFactory { + + private InstantFieldFactory(Indexed.Type type) { + super(type); + } + + @Override + public Iterable create(String name, Object value) { + Instant instant = (Instant) value; + return super.create(name, instant.toEpochMilli()); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java new file mode 100644 index 0000000000..91d10b87e6 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java @@ -0,0 +1,109 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; + +import java.io.IOException; + +import static sonia.scm.search.FieldNames.*; + +public class LuceneIndex implements Index { + + private final DocumentConverter converter; + private final IndexWriter writer; + + LuceneIndex(DocumentConverter converter, IndexWriter writer) { + this.converter = converter; + this.writer = writer; + } + + @Override + public void store(Id id, String permission, Object object) { + String uid = createUid(id, object.getClass()); + Document document = converter.convert(object); + try { + field(document, UID, uid); + field(document, ID, id.getValue()); + id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository)); + field(document, TYPE, object.getClass().getName()); + if (!Strings.isNullOrEmpty(permission)) { + field(document, PERMISSION, permission); + } + writer.updateDocument(new Term(UID, uid), document); + } catch (IOException e) { + throw new SearchEngineException("failed to add document to index", e); + } + } + + private String createUid(Id id, Class type) { + return id.asString() + "/" + type.getName(); + } + + private void field(Document document, String type, String name) { + document.add(new StringField(type, name, Field.Store.YES)); + } + + @Override + public void delete(Id id, Class type) { + try { + writer.deleteDocuments(new Term(UID, createUid(id, type))); + } catch (IOException e) { + throw new SearchEngineException("failed to delete document from index", e); + } + } + + @Override + public void deleteByRepository(String repository) { + try { + writer.deleteDocuments(new Term(REPOSITORY, repository)); + } catch (IOException ex) { + throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex); + } + } + + @Override + public void deleteByType(Class type) { + try { + writer.deleteDocuments(new Term(TYPE, type.getName())); + } catch (IOException ex) { + throw new SearchEngineException("failed to delete documents by repository " + type + " from index", ex); + } + } + + @Override + public void close() { + try { + writer.close(); + } catch (IOException e) { + throw new SearchEngineException("failed to close index writer", e); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java new file mode 100644 index 0000000000..3c12c61363 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java @@ -0,0 +1,149 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class LuceneQueryBuilder extends QueryBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class); + + private static final Map, SearchableType> CACHE = new ConcurrentHashMap<>(); + + private final IndexOpener opener; + private final String indexName; + private final Analyzer analyzer; + + LuceneQueryBuilder(IndexOpener opener, String indexName, Analyzer analyzer) { + this.opener = opener; + this.indexName = indexName; + this.analyzer = analyzer; + } + + @Override + protected QueryResult execute(QueryParams queryParams) { + String queryString = Strings.nullToEmpty(queryParams.getQueryString()); + + SearchableType searchableType = CACHE.computeIfAbsent(queryParams.getType(), SearchableTypes::create); + + Query query = Queries.filter(createQuery(searchableType, queryParams, queryString), queryParams); + if (LOG.isDebugEnabled()) { + LOG.debug("execute lucene query: {}", query); + } + try (IndexReader reader = opener.openForRead(indexName)) { + IndexSearcher searcher = new IndexSearcher(reader); + + TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams); + Collector collector = new PermissionAwareCollector(reader, topScoreCollector); + searcher.search(query, collector); + + QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query); + return resultFactory.create(getTopDocs(queryParams, topScoreCollector)); + } catch (IOException e) { + throw new SearchEngineException("failed to search index", e); + } catch (InvalidTokenOffsetsException e) { + throw new SearchEngineException("failed to highlight results", e); + } + } + + @Nonnull + private TopScoreDocCollector createTopScoreCollector(QueryParams queryParams) { + return TopScoreDocCollector.create(queryParams.getStart() + queryParams.getLimit(), Integer.MAX_VALUE); + } + + private TopDocs getTopDocs(QueryParams queryParams, TopScoreDocCollector topScoreCollector) { + return topScoreCollector.topDocs(queryParams.getStart(), queryParams.getLimit()); + } + + private Query createQuery(SearchableType searchableType, QueryParams queryParams, String queryString) { + try { + if (queryString.contains(":")) { + return createExpertQuery(searchableType, queryParams); + } + return createBestGuessQuery(searchableType, queryParams); + } catch (QueryNodeException ex) { + throw new QueryParseException(queryString, "failed to parse query", ex); + } + } + + private Query createExpertQuery(SearchableType searchableType, QueryParams queryParams) throws QueryNodeException { + StandardQueryParser parser = new StandardQueryParser(analyzer); + + parser.setPointsConfigMap(searchableType.getPointsConfig()); + return parser.parse(queryParams.getQueryString(), ""); + } + + public Query createBestGuessQuery(SearchableType searchableType, QueryBuilder.QueryParams queryParams) { + String[] fieldNames = searchableType.getFieldNames(); + if (fieldNames == null || fieldNames.length == 0) { + throw new NoDefaultQueryFieldsFoundException(searchableType.getType()); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String fieldName : fieldNames) { + Term term = new Term(fieldName, appendWildcardIfNotAlreadyUsed(queryParams)); + WildcardQuery query = new WildcardQuery(term); + + Float boost = searchableType.getBoosts().get(fieldName); + if (boost != null) { + builder.add(new BoostQuery(query, boost), BooleanClause.Occur.SHOULD); + } else { + builder.add(query, BooleanClause.Occur.SHOULD); + } + } + return builder.build(); + } + + @Nonnull + private String appendWildcardIfNotAlreadyUsed(QueryParams queryParams) { + String queryString = queryParams.getQueryString().toLowerCase(Locale.ENGLISH); + if (!queryString.contains("?") && !queryString.contains("*")) { + queryString += "*"; + } + return queryString; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java new file mode 100644 index 0000000000..6b86c5c528 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import javax.inject.Inject; + +public class LuceneQueryBuilderFactory { + + private final IndexOpener indexOpener; + private final AnalyzerFactory analyzerFactory; + + @Inject + public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) { + this.indexOpener = indexOpener; + this.analyzerFactory = analyzerFactory; + } + + public LuceneQueryBuilder create(String name, IndexOptions options) { + return new LuceneQueryBuilder(indexOpener, name, analyzerFactory.create(options)); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java new file mode 100644 index 0000000000..06edf78b5b --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import javax.inject.Inject; +import java.io.IOException; + +public class LuceneSearchEngine implements SearchEngine { + + private final IndexOpener indexOpener; + private final DocumentConverter converter; + private final LuceneQueryBuilderFactory queryBuilderFactory; + + @Inject + public LuceneSearchEngine(IndexOpener indexOpener, DocumentConverter converter, LuceneQueryBuilderFactory queryBuilderFactory) { + this.indexOpener = indexOpener; + this.converter = converter; + this.queryBuilderFactory = queryBuilderFactory; + } + + @Override + public Index getOrCreate(String name, IndexOptions options) { + try { + return new LuceneIndex(converter, indexOpener.openForWrite(name, options)); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index", ex); + } + } + + @Override + public QueryBuilder search(String name, IndexOptions options) { + return queryBuilderFactory.create(name, options); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java b/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java new file mode 100644 index 0000000000..c15d8f6cd7 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java @@ -0,0 +1,31 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +public class NonReadableFieldException extends SearchEngineException { + public NonReadableFieldException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java b/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java new file mode 100644 index 0000000000..4fa3ce502f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.shiro.SecurityUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; + +public class PermissionAwareCollector implements Collector { + + private static final String FIELD_PERMISSION = "_permission"; + private static final Set FIELDS = Collections.singleton(FIELD_PERMISSION); + + private final IndexReader reader; + private final Collector delegate; + + public PermissionAwareCollector(IndexReader reader, Collector delegate) { + this.reader = reader; + this.delegate = delegate; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + return new PermissionAwareLeafCollector(delegate.getLeafCollector(context), context.docBase); + } + + @Override + public ScoreMode scoreMode() { + return delegate.scoreMode(); + } + + private class PermissionAwareLeafCollector implements LeafCollector { + + private final LeafCollector delegate; + private final int docBase; + + private PermissionAwareLeafCollector(LeafCollector delegate, int docBase) { + this.delegate = delegate; + this.docBase = docBase; + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.delegate.setScorer(scorer); + } + + @Override + public void collect(int doc) throws IOException { + Document document = reader.document(docBase + doc, FIELDS); + String permission = document.get(FIELD_PERMISSION); + if (Strings.isNullOrEmpty(permission) || SecurityUtils.getSubject().isPermitted(permission)) { + this.delegate.collect(doc); + } + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/Queries.java b/scm-webapp/src/main/java/sonia/scm/search/Queries.java new file mode 100644 index 0000000000..0b69d20194 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/Queries.java @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + +import static org.apache.lucene.search.BooleanClause.Occur.MUST; + +final class Queries { + + private Queries() { + } + + private static Query typeQuery(Class type) { + return new TermQuery(new Term(FieldNames.TYPE, type.getName())); + } + + private static Query repositoryQuery(String repositoryId) { + return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId)); + } + + static Query filter(Query query, QueryBuilder.QueryParams params) { + BooleanQuery.Builder builder = new BooleanQuery.Builder() + .add(query, MUST) + .add(typeQuery(params.getType()), MUST); + params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST)); + return builder.build(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java b/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java new file mode 100644 index 0000000000..7ddfbc3b71 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java @@ -0,0 +1,103 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.highlight.Highlighter; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.apache.lucene.search.highlight.QueryScorer; +import org.apache.lucene.search.highlight.SimpleHTMLFormatter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Optional.empty; +import static java.util.Optional.of; + +public class QueryResultFactory { + + private final Analyzer analyzer; + private final Highlighter highlighter; + private final IndexSearcher searcher; + private final SearchableType searchableType; + + public QueryResultFactory(Analyzer analyzer, IndexSearcher searcher, SearchableType searchableType, Query query) { + this.analyzer = analyzer; + this.searcher = searcher; + this.searchableType = searchableType; + this.highlighter = createHighlighter(query); + } + + private Highlighter createHighlighter(Query query) { + return new Highlighter( + new SimpleHTMLFormatter("**", "**"), + new QueryScorer(query) + ); + } + + public QueryResult create(TopDocs topDocs) throws IOException, InvalidTokenOffsetsException { + List hits = new ArrayList<>(); + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + hits.add(createHit(scoreDoc)); + } + return new QueryResult(topDocs.totalHits.value, searchableType.getType(), hits); + } + + private Hit createHit(ScoreDoc scoreDoc) throws IOException, InvalidTokenOffsetsException { + Document document = searcher.doc(scoreDoc.doc); + Map fields = new HashMap<>(); + for (SearchableField field : searchableType.getFields()) { + field(document, field).ifPresent(f -> fields.put(field.getName(), f)); + } + return new Hit(document.get(FieldNames.ID), scoreDoc.score, fields); + } + private Optional field(Document document, SearchableField field) throws IOException, InvalidTokenOffsetsException { + Object value = field.value(document); + if (value != null) { + if (field.isHighlighted()) { + String[] fragments = createFragments(field, value.toString()); + if (fragments.length > 0) { + return of(new Hit.HighlightedField(fragments)); + } + } + return of(new Hit.ValueField(value)); + } + return empty(); + } + + private String[] createFragments(SearchableField field, String value) throws InvalidTokenOffsetsException, IOException { + return highlighter.getBestFragments(analyzer, field.getName(), value, 5); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java new file mode 100644 index 0000000000..2f0ce2239f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java @@ -0,0 +1,71 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import java.util.ArrayList; +import java.util.List; + +public class QueuedIndex implements Index { + + private final DefaultIndexQueue queue; + private final String indexName; + private final IndexOptions indexOptions; + + private final List tasks = new ArrayList<>(); + + QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) { + this.queue = queue; + this.indexName = indexName; + this.indexOptions = indexOptions; + } + + @Override + public void store(Id id, String permission, Object object) { + tasks.add(index -> index.store(id, permission, object)); + } + + @Override + public void delete(Id id, Class type) { + tasks.add(index -> index.delete(id, type)); + } + + @Override + public void deleteByRepository(String repository) { + tasks.add(index -> index.deleteByRepository(repository)); + } + + @Override + public void deleteByType(Class type) { + tasks.add(index -> index.deleteByType(type)); + } + + @Override + public void close() { + IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper( + queue.getSearchEngine(), indexName, indexOptions, tasks + ); + queue.enqueue(wrappedTask); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java new file mode 100644 index 0000000000..f96cf38972 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import lombok.Getter; +import org.apache.lucene.document.Document; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; + +@Getter +class SearchableField { + + private final String name; + private final Class type; + private final ValueExtractor valueExtractor; + private final float boost; + private final boolean defaultQuery; + private final boolean highlighted; + private final PointsConfig pointsConfig; + + SearchableField(Field field, Indexed indexed) { + this.name = name(field, indexed); + this.type = field.getType(); + this.valueExtractor = ValueExtractors.create(name, type); + this.boost = indexed.boost(); + this.defaultQuery = indexed.defaultQuery(); + this.highlighted = indexed.highlighted(); + this.pointsConfig = IndexableFields.pointConfig(field); + } + + Object value(Document document) { + return valueExtractor.extract(document); + } + + private String name(Field field, Indexed indexed) { + String nameFromAnnotation = indexed.name(); + if (!Strings.isNullOrEmpty(nameFromAnnotation)) { + return nameFromAnnotation; + } + return field.getName(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java new file mode 100644 index 0000000000..14bdc6dce2 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Value; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@Value +public class SearchableType { + + Class type; + String[] fieldNames; + Map boosts; + Map pointsConfig; + List fields; + + SearchableType(Class type, String[] fieldNames, Map boosts, Map pointsConfig, List fields) { + this.type = type; + this.fieldNames = fieldNames; + this.boosts = Collections.unmodifiableMap(boosts); + this.pointsConfig = Collections.unmodifiableMap(pointsConfig); + this.fields = Collections.unmodifiableList(fields); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java new file mode 100644 index 0000000000..dc7bbac77f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java @@ -0,0 +1,82 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +final class SearchableTypes { + + private static final float DEFAULT_BOOST = 1f; + + private SearchableTypes() { + } + + static SearchableType create(Class type) { + List fields = new ArrayList<>(); + collectFields(type, fields); + return createSearchableType(type, fields); + } + + private static SearchableType createSearchableType(Class type, List fields) { + String[] fieldsNames = fields.stream() + .filter(SearchableField::isDefaultQuery) + .map(SearchableField::getName) + .toArray(String[]::new); + + Map boosts = new HashMap<>(); + Map pointsConfig = new HashMap<>(); + for (SearchableField field : fields) { + if (field.isDefaultQuery() && field.getBoost() != DEFAULT_BOOST) { + boosts.put(field.getName(), field.getBoost()); + } + PointsConfig config = field.getPointsConfig(); + if (config != null) { + pointsConfig.put(field.getName(), config); + } + } + + return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields); + } + + private static void collectFields(Class type, List fields) { + Class parent = type.getSuperclass(); + if (parent != null) { + collectFields(parent, fields); + } + for (Field field : type.getDeclaredFields()) { + Indexed indexed = field.getAnnotation(Indexed.class); + if (indexed != null) { + fields.add(new SearchableField(field, indexed)); + } + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java b/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java new file mode 100644 index 0000000000..79bd68bb68 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java @@ -0,0 +1,53 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import java.time.Instant; + +final class TypeCheck { + + private TypeCheck() { + } + + public static boolean isLong(Class type) { + return type == Long.TYPE || type == Long.class; + } + + public static boolean isInteger(Class type) { + return type == Integer.TYPE || type == Integer.class; + } + + public static boolean isBoolean(Class type) { + return type == Boolean.TYPE || type == Boolean.class; + } + + public static boolean isInstant(Class type) { + return type == Instant.class; + } + + public static boolean isString(Class type) { + return type == String.class; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java b/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java new file mode 100644 index 0000000000..bf7acfdbca --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java @@ -0,0 +1,31 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +public class UnsupportedTypeOfFieldException extends SearchEngineException { + public UnsupportedTypeOfFieldException(Class type, String field) { + super("type " + type + " of " + field + " is unsupported."); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java new file mode 100644 index 0000000000..b317c080df --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Document; + +@FunctionalInterface +public interface ValueExtractor { + Object extract(Document document); +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java new file mode 100644 index 0000000000..d002aac246 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java @@ -0,0 +1,95 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.IndexableField; + +import javax.annotation.Nonnull; +import java.time.Instant; + +final class ValueExtractors { + + private ValueExtractors() { + } + + static ValueExtractor create(String name, Class type) { + if (TypeCheck.isString(type)) { + return stringExtractor(name); + } else if (TypeCheck.isLong(type)) { + return longExtractor(name); + } else if (TypeCheck.isInteger(type)) { + return integerExtractor(name); + } else if (TypeCheck.isBoolean(type)) { + return booleanExtractor(name); + } else if (TypeCheck.isInstant(type)) { + return instantExtractor(name); + } else { + throw new UnsupportedTypeOfFieldException(type, name); + } + } + + @Nonnull + private static ValueExtractor stringExtractor(String name) { + return doc -> doc.get(name); + } + + @Nonnull + private static ValueExtractor instantExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return Instant.ofEpochMilli(field.numericValue().longValue()); + } + return null; + }; + } + + @Nonnull + private static ValueExtractor booleanExtractor(String name) { + return doc -> Boolean.parseBoolean(doc.get(name)); + } + + @Nonnull + private static ValueExtractor integerExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return field.numericValue().intValue(); + } + return null; + }; + } + + @Nonnull + private static ValueExtractor longExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return field.numericValue().longValue(); + } + return null; + }; + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java index c9fff0666e..ffa4ac8418 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java @@ -195,5 +195,6 @@ class IndexDtoGeneratorTest { when(resourceLinks.namespaceCollection()).thenReturn(new ResourceLinks.NamespaceCollectionLinks(scmPathInfo)); when(resourceLinks.me()).thenReturn(new ResourceLinks.MeLinks(scmPathInfo, new ResourceLinks.UserLinks(scmPathInfo))); when(resourceLinks.repository()).thenReturn(new ResourceLinks.RepositoryLinks(scmPathInfo)); + when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(scmPathInfo)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java index 2bec6f3166..83c12ce222 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java @@ -82,6 +82,7 @@ public class ResourceLinksMock { lenient().when(resourceLinks.adminInfo()).thenReturn(new ResourceLinks.AdminInfoLinks(pathInfo)); lenient().when(resourceLinks.apiKeyCollection()).thenReturn(new ResourceLinks.ApiKeyCollectionLinks(pathInfo)); lenient().when(resourceLinks.apiKey()).thenReturn(new ResourceLinks.ApiKeyLinks(pathInfo)); + lenient().when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(pathInfo)); return resourceLinks; } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java new file mode 100644 index 0000000000..ac0139bb1f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java @@ -0,0 +1,266 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import com.fasterxml.jackson.databind.JsonNode; +import de.otto.edison.hal.HalRepresentation; +import lombok.Getter; +import lombok.Setter; +import org.jboss.resteasy.mock.MockHttpRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mapstruct.factory.Mappers; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.search.Hit; +import sonia.scm.search.IndexNames; +import sonia.scm.search.QueryResult; +import sonia.scm.search.SearchEngine; +import sonia.scm.web.JsonMockHttpResponse; +import sonia.scm.web.RestDispatcher; +import sonia.scm.web.VndMediaType; + +import javax.annotation.Nonnull; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SearchResourceTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; + + private RestDispatcher dispatcher; + + @Mock + private HalEnricherRegistry enricherRegistry; + + @BeforeEach + void setUpDispatcher() { + QueryResultMapper mapper = Mappers.getMapper(QueryResultMapper.class); + mapper.setRegistry(enricherRegistry); + SearchResource resource = new SearchResource( + searchEngine, mapper + ); + dispatcher = new RestDispatcher(); + dispatcher.addSingletonResource(resource); + } + + @Test + void shouldEnrichQueryResult() throws IOException, URISyntaxException { + when(enricherRegistry.allByType(QueryResult.class)) + .thenReturn(Collections.singleton(new SampleEnricher())); + + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + + JsonNode sample = response.getContentAsJson().get("_embedded").get("sample"); + assertThat(sample.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Test + void shouldEnrichHitResult() throws IOException, URISyntaxException { + when(enricherRegistry.allByType(QueryResult.class)) + .thenReturn(Collections.emptySet()); + when(enricherRegistry.allByType(Hit.class)) + .thenReturn(Collections.singleton(new SampleEnricher())); + + mockQueryResult("Hello", result(1L, "Hello")); + JsonMockHttpResponse response = search("Hello"); + + JsonNode sample = response.getContentAsJson() + .get("_embedded").get("hits").get(0) + .get("_embedded").get("sample"); + assertThat(sample.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Nested + class WithoutEnricher { + + @BeforeEach + void setUpEnricherRegistry() { + when(enricherRegistry.allByType(QueryResult.class)).thenReturn(Collections.emptySet()); + lenient().when(enricherRegistry.allByType(Hit.class)).thenReturn(Collections.emptySet()); + } + + @Test + void shouldReturnVndContentType() throws UnsupportedEncodingException, URISyntaxException { + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + assertThat(response.getStatus()).isEqualTo(HttpServletResponse.SC_OK); + assertHeader(response, "Content-Type", VndMediaType.QUERY_RESULT); + } + + @Test + void shouldReturnPagingLinks() throws IOException, URISyntaxException { + mockQueryResult(20, 20, "paging", result(100)); + JsonMockHttpResponse response = search("paging", 1, 20); + + JsonNode links = response.getContentAsJson().get("_links"); + assertLink(links, "self", "/v2/search?q=paging&page=1&pageSize=20"); + assertLink(links, "first", "/v2/search?q=paging&page=0&pageSize=20"); + assertLink(links, "prev", "/v2/search?q=paging&page=0&pageSize=20"); + assertLink(links, "next", "/v2/search?q=paging&page=2&pageSize=20"); + assertLink(links, "last", "/v2/search?q=paging&page=4&pageSize=20"); + } + + @Test + void shouldPagingFields() throws IOException, URISyntaxException { + mockQueryResult(20, 20, "pagingFields", result(100)); + JsonMockHttpResponse response = search("pagingFields", 1, 20); + + JsonNode root = response.getContentAsJson(); + assertThat(root.get("page").asInt()).isOne(); + assertThat(root.get("pageTotal").asInt()).isEqualTo(5); + } + + @Test + void shouldReturnType() throws IOException, URISyntaxException { + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + + JsonNode root = response.getContentAsJson(); + assertThat(root.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Test + void shouldReturnHitsAsEmbedded() throws IOException, URISyntaxException { + mockQueryResult("Hello", result(2L, "Hello", "Hello Again")); + JsonMockHttpResponse response = search("Hello"); + + JsonNode hits = response.getContentAsJson().get("_embedded").get("hits"); + assertThat(hits.size()).isEqualTo(2); + + JsonNode first = hits.get(0); + assertThat(first.get("score").asDouble()).isEqualTo(2d); + + JsonNode fields = first.get("fields"); + + JsonNode valueField = fields.get("value"); + assertThat(valueField.get("highlighted").asBoolean()).isFalse(); + assertThat(valueField.get("value").asText()).isEqualTo("Hello"); + + JsonNode highlightedField = fields.get("highlighted"); + assertThat(highlightedField.get("highlighted").asBoolean()).isTrue(); + assertThat(highlightedField.get("fragments").get(0).asText()).isEqualTo("Hello"); + } + + } + + private void assertLink(JsonNode links, String self, String s) { + assertThat(links.get(self).get("href").asText()).isEqualTo(s); + } + + private QueryResult result(long totalHits, Object... values) { + List hits = new ArrayList<>(); + for (int i = 0; i < values.length; i++) { + hits.add(hit(i, values)); + } + return new QueryResult(totalHits, String.class, hits); + } + + @Nonnull + private Hit hit(int i, Object[] values) { + Map fields = fields(values[i]); + return new Hit("" + i, values.length - i, fields); + } + + @Nonnull + private Map fields(Object value) { + Map fields = new HashMap<>(); + fields.put("value", new Hit.ValueField(value)); + fields.put("highlighted", new Hit.HighlightedField(new String[]{value.toString()})); + return fields; + } + + private void mockQueryResult(String query, QueryResult result) { + mockQueryResult(0, 10, query, result); + } + + private void mockQueryResult(int start, int limit, String query, QueryResult result) { + when( + searchEngine.search(IndexNames.DEFAULT) + .start(start) + .limit(limit) + .execute(Repository.class, query) + ).thenReturn(result); + } + + private void assertHeader(JsonMockHttpResponse response, String header, String expectedValue) { + assertThat(response.getOutputHeaders().getFirst(header)).hasToString(expectedValue); + } + + private JsonMockHttpResponse search(String query) throws URISyntaxException, UnsupportedEncodingException { + return search(query, null, null); + } + + private JsonMockHttpResponse search(String query, Integer page, Integer pageSize) throws URISyntaxException, UnsupportedEncodingException { + String uri = "/v2/search?q=" + URLEncoder.encode(query, "UTF-8"); + if (page != null) { + uri += "&page=" + page; + } + if (pageSize != null) { + uri += "&pageSize=" + pageSize; + } + MockHttpRequest request = MockHttpRequest.get(uri); + JsonMockHttpResponse response = new JsonMockHttpResponse(); + dispatcher.invoke(request, response); + return response; + } + + @Getter + @Setter + public static class SampleEmbedded extends HalRepresentation { + private Class type; + } + + private static class SampleEnricher implements HalEnricher { + @Override + public void enrich(HalEnricherContext context, HalAppender appender) { + QueryResult result = context.oneRequireByType(QueryResult.class); + + SampleEmbedded embedded = new SampleEmbedded(); + embedded.setType(result.getType()); + + appender.appendEmbedded("sample", embedded); + } + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java b/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java new file mode 100644 index 0000000000..13a2bb02b7 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java @@ -0,0 +1,147 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.repository; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.HandlerEventType; +import sonia.scm.search.Id; +import sonia.scm.search.Index; +import sonia.scm.search.IndexLog; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexNames; +import sonia.scm.search.IndexQueue; +import sonia.scm.web.security.AdministrationContext; + +import java.util.List; +import java.util.Optional; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexUpdateListenerTest { + + @Mock + private RepositoryManager repositoryManager; + + @Mock + private AdministrationContext administrationContext; + + @Mock + private IndexQueue indexQueue; + + @Mock + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @InjectMocks + private IndexUpdateListener updateListener; + + @Test + @SuppressWarnings("java:S6068") + void shouldIndexAllRepositories() { + when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.empty()); + doAnswer(ic -> { + IndexUpdateListener.ReIndexAll reIndexAll = new IndexUpdateListener.ReIndexAll(repositoryManager, indexQueue); + reIndexAll.run(); + return null; + }) + .when(administrationContext) + .runAsAdmin(IndexUpdateListener.ReIndexAll.class); + + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle42 = RepositoryTestData.create42Puzzle(); + List repositories = ImmutableList.of(heartOfGold, puzzle42); + + when(repositoryManager.getAll()).thenReturn(repositories); + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + + updateListener.contextInitialized(null); + + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + verify(index).store(Id.of(puzzle42), RepositoryPermissions.read(puzzle42).asShiroString(), puzzle42); + verify(index).close(); + + verify(indexLogStore).log(IndexNames.DEFAULT, Repository.class, IndexUpdateListener.INDEX_VERSION); + } + + @Test + void shouldSkipReIndex() { + IndexLog log = new IndexLog(1); + when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.of(log)); + + updateListener.contextInitialized(null); + + verifyNoInteractions(indexQueue); + } + + @ParameterizedTest + @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.MATCH_ANY, names = "BEFORE_.*") + void shouldIgnoreBeforeEvents(HandlerEventType type) { + RepositoryEvent event = new RepositoryEvent(type, RepositoryTestData.create42Puzzle()); + + updateListener.handleEvent(event); + + verifyNoInteractions(indexQueue); + } + + @ParameterizedTest + @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.INCLUDE, names = {"CREATE", "MODIFY"}) + void shouldStore(HandlerEventType type) { + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + + Repository puzzle = RepositoryTestData.create42Puzzle(); + RepositoryEvent event = new RepositoryEvent(type, puzzle); + + updateListener.handleEvent(event); + + verify(index).store(Id.of(puzzle), RepositoryPermissions.read(puzzle).asShiroString(), puzzle); + verify(index).close(); + } + + @Test + void shouldDelete() { + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + Repository puzzle = RepositoryTestData.create42Puzzle(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, puzzle); + + updateListener.handleEvent(event); + + verify(index).deleteByRepository(puzzle.getId()); + verify(index).close(); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java b/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java new file mode 100644 index 0000000000..be83b7450a --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.de.GermanAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.es.SpanishAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +class AnalyzerFactoryTest { + + private final AnalyzerFactory analyzerFactory = new AnalyzerFactory(); + + @Test + void shouldReturnStandardAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults()); + assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); + } + + @Test + void shouldReturnStandardAnalyzerForUnknownLocale() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE)); + assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); + } + + @Test + void shouldReturnEnglishAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH)); + assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class); + } + + @Test + void shouldReturnGermanAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN)); + assertThat(analyzer).isInstanceOf(GermanAnalyzer.class); + } + + @Test + void shouldReturnGermanAnalyzerForLocaleGermany() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY)); + assertThat(analyzer).isInstanceOf(GermanAnalyzer.class); + } + + @Test + void shouldReturnSpanishAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES"))); + assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java new file mode 100644 index 0000000000..b6876e19ca --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import sonia.scm.store.InMemoryByteDataStoreFactory; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class DefaultIndexLogStoreTest { + + private IndexLogStore indexLogStore; + + @BeforeEach + void setUpIndexLogStore() { + InMemoryByteDataStoreFactory dataStoreFactory = new InMemoryByteDataStoreFactory(); + indexLogStore = new DefaultIndexLogStore(dataStoreFactory); + } + + @Test + void shouldReturnEmptyOptional() { + Optional indexLog = indexLogStore.get("index", String.class); + assertThat(indexLog).isEmpty(); + } + + @Test + void shouldStoreLog() { + indexLogStore.log("index", String.class, 42); + Optional index = indexLogStore.get("index", String.class); + assertThat(index).hasValueSatisfying(log -> { + assertThat(log.getVersion()).isEqualTo(42); + assertThat(log.getDate()).isNotNull(); + }); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java new file mode 100644 index 0000000000..69683ab4b4 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java @@ -0,0 +1,145 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Value; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DefaultIndexQueueTest { + + private Directory directory; + + private DefaultIndexQueue queue; + + @Mock + private LuceneQueryBuilderFactory queryBuilderFactory; + + @BeforeEach + void createQueue() throws IOException { + directory = new ByteBuffersDirectory(); + IndexOpener factory = mock(IndexOpener.class); + when(factory.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory, config); + }); + SearchEngine engine = new LuceneSearchEngine(factory, new DocumentConverter(), queryBuilderFactory); + queue = new DefaultIndexQueue(engine); + } + + @AfterEach + void closeQueue() throws IOException { + queue.close(); + directory.close(); + } + + @Test + void shouldWriteToIndex() throws Exception { + try (Index index = queue.getQueuedIndex("default")) { + index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan")); + index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent")); + } + assertDocCount(2); + } + + @Test + void shouldWriteMultiThreaded() throws Exception { + ExecutorService executorService = Executors.newFixedThreadPool(4); + for (int i = 0; i < 20; i++) { + executorService.execute(new IndexNumberTask(i)); + } + executorService.execute(() -> { + try (Index index = queue.getQueuedIndex("default")) { + index.delete(Id.of(String.valueOf(12)), IndexedNumber.class); + } + }); + executorService.shutdown(); + + assertDocCount(19); + } + + private void assertDocCount(int expectedCount) throws IOException { + // wait until all tasks are finished + await().until(() -> queue.getSize() == 0); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertThat(reader.numDocs()).isEqualTo(expectedCount); + } + } + + @Value + public static class Account { + @Indexed + String username; + @Indexed + String firstName; + @Indexed + String lastName; + } + + @Value + public static class IndexedNumber { + @Indexed + int value; + } + + public class IndexNumberTask implements Runnable { + + private final int number; + + public IndexNumberTask(int number) { + this.number = number; + } + + @Override + public void run() { + try (Index index = queue.getQueuedIndex("default")) { + index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number)); + } + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java b/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java new file mode 100644 index 0000000000..8980639faf --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java @@ -0,0 +1,288 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class DocumentConverterTest { + + private DocumentConverter documentConverter; + + @BeforeEach + void prepare() { + documentConverter = new DocumentConverter(); + } + + @Test + void shouldConvertPersonToDocument() { + Person person = new Person("Arthur", "Dent"); + + Document document = documentConverter.convert(person); + + assertThat(document.getField("firstName").stringValue()).isEqualTo("Arthur"); + assertThat(document.getField("lastName").stringValue()).isEqualTo("Dent"); + } + + @Test + void shouldUseNameFromAnnotation() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("username").stringValue()).isEqualTo("dent"); + } + + @Test + void shouldBeIndexedAsTextFieldByDefault() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("username")).isInstanceOf(TextField.class); + } + + @Test + void shouldBeIndexedAsStringField() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("searchable")).isInstanceOf(StringField.class); + } + + @Test + void shouldBeIndexedAsStoredField() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("storedOnly")).isInstanceOf(StoredField.class); + } + + @Test + void shouldIgnoreNonIndexedFields() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("notIndexed")).isNull(); + } + + @Test + void shouldSupportInheritance() { + Account account = new Account("Arthur", "Dent", "arthur@hitchhiker.com"); + + Document document = documentConverter.convert(account); + + assertThat(document.getField("firstName")).isNotNull(); + assertThat(document.getField("lastName")).isNotNull(); + assertThat(document.getField("mail")).isNotNull(); + } + + @Test + void shouldFailWithoutGetter() { + WithoutGetter withoutGetter = new WithoutGetter(); + assertThrows(NonReadableFieldException.class, () -> documentConverter.convert(withoutGetter)); + } + + @Test + void shouldFailOnUnsupportedFieldType() { + UnsupportedFieldType unsupportedFieldType = new UnsupportedFieldType(); + assertThrows(UnsupportedTypeOfFieldException.class, () -> documentConverter.convert(unsupportedFieldType)); + } + + @Test + void shouldStoreLongFieldsAsPointAndStoredByDefault() { + Document document = documentConverter.convert(new SupportedTypes()); + + assertPointField(document, "longType", + field -> assertThat(field.numericValue().longValue()).isEqualTo(42L) + ); + } + + @Test + void shouldStoreLongFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyLongType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().longValue()).isEqualTo(42L); + } + + @Test + void shouldStoreIntegerFieldsAsPointAndStoredByDefault() { + Document document = documentConverter.convert(new SupportedTypes()); + + assertPointField(document, "intType", + field -> assertThat(field.numericValue().intValue()).isEqualTo(42) + ); + } + + @Test + void shouldStoreIntegerFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyIntegerType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().intValue()).isEqualTo(42); + } + + @Test + void shouldStoreBooleanFieldsAsStringField() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("boolType"); + assertThat(field).isInstanceOf(StringField.class); + assertThat(field.stringValue()).isEqualTo("true"); + assertThat(field.fieldType().stored()).isTrue(); + } + + @Test + void shouldStoreBooleanFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyBoolType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.stringValue()).isEqualTo("true"); + } + + @Test + void shouldStoreInstantFieldsAsPointAndStoredByDefault() { + Instant now = Instant.now(); + Document document = documentConverter.convert(new DateTypes(now)); + + assertPointField(document, "instant", + field -> assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli()) + ); + } + + @Test + void shouldStoreInstantFieldAsStored() { + Instant now = Instant.now(); + Document document = documentConverter.convert(new DateTypes(now)); + + IndexableField field = document.getField("storedOnlyInstant"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli()); + } + + @Test + void shouldCreateNoFieldForNullValues() { + Document document = documentConverter.convert(new Person("Trillian", null)); + + assertThat(document.getField("firstName")).isNotNull(); + assertThat(document.getField("lastName")).isNull(); + } + + private void assertPointField(Document document, String name, Consumer consumer) { + IndexableField[] fields = document.getFields(name); + assertThat(fields) + .allSatisfy(consumer) + .anySatisfy(field -> assertThat(field.fieldType().stored()).isFalse()) + .anySatisfy(field -> assertThat(field.fieldType().stored()).isTrue()); + } + + @Getter + @AllArgsConstructor + public static class Person { + @Indexed + private String firstName; + @Indexed + private String lastName; + } + + @Getter + public static class Account extends Person { + @Indexed + private String mail; + + public Account(String firstName, String lastName, String mail) { + super(firstName, lastName); + this.mail = mail; + } + } + + @Getter + public static class ParamSample { + @Indexed(name = "username") + private final String name = "dent"; + + @Indexed(type = Indexed.Type.SEARCHABLE) + private final String searchable = "--"; + + @Indexed(type = Indexed.Type.STORED_ONLY) + private final String storedOnly = "--"; + + private final String notIndexed = "--"; + } + + public static class WithoutGetter { + @Indexed + private final String value = "one"; + } + + @Getter + public static class UnsupportedFieldType { + @Indexed + private final Object value = "one"; + } + + @Getter + public static class SupportedTypes { + @Indexed + private final Long longType = 42L; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final long storedOnlyLongType = 42L; + + @Indexed + private final int intType = 42; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final Integer storedOnlyIntegerType = 42; + + @Indexed + private final boolean boolType = true; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final boolean storedOnlyBoolType = true; + } + + @Getter + private static class DateTypes { + @Indexed + private final Instant instant; + + @Indexed(type = Indexed.Type.STORED_ONLY) + private final Instant storedOnlyInstant; + + private DateTypes(Instant instant) { + this.instant = instant; + this.storedOnlyInstant = instant; + } + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java new file mode 100644 index 0000000000..5f70b6bfab --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java @@ -0,0 +1,99 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.core.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexOpenerTest { + + private Path directory; + + @Mock + private AnalyzerFactory analyzerFactory; + + private IndexOpener indexOpener; + + @BeforeEach + void createIndexWriterFactory(@TempDir Path tempDirectory) { + this.directory = tempDirectory; + SCMContextProvider context = mock(SCMContextProvider.class); + when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory); + when(analyzerFactory.create(any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); + indexOpener = new IndexOpener(context, analyzerFactory); + } + + @Test + void shouldCreateNewIndex() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("new-index")).exists(); + } + + @Test + void shouldOpenExistingIndex() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { + addDoc(writer, "Dent"); + } + try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { + assertThat(writer.getFieldNames()).contains("hitchhiker"); + } + } + + @Test + void shouldUseAnalyzerFromFactory() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { + assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); + } + } + + private void addDoc(IndexWriter writer, String name) throws IOException { + Document doc = new Document(); + doc.add(new TextField("hitchhiker", name, Field.Store.YES)); + writer.addDocument(doc); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java new file mode 100644 index 0000000000..58c79c35b5 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java @@ -0,0 +1,238 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import lombok.Value; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static sonia.scm.search.FieldNames.*; + +class LuceneIndexTest { + + private static final Id ONE = Id.of("one"); + + private Directory directory; + + @BeforeEach + void createDirectory() { + directory = new ByteBuffersDirectory(); + } + + @Test + void shouldStoreObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldUpdateObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + index.store(ONE, null, new Storable("Awesome content")); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldStoreUidOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + } + + assertHits(UID, "one/" + Storable.class.getName(), 1); + } + + @Test + void shouldStoreIdOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some text")); + } + + assertHits(ID, "one", 1); + } + + @Test + void shouldStoreRepositoryOfId() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), null, new Storable("Some text")); + } + + assertHits(REPOSITORY, "4211", 1); + } + + @Test + void shouldStoreTypeOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + } + + assertHits(TYPE, Storable.class.getName(), 1); + } + + @Test + void shouldDeleteById() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(ONE, Storable.class); + } + + assertHits(ID, "one", 0); + } + + @Test + void shouldDeleteAllByType() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("content")); + index.store(Id.of("two"), null, new Storable("content")); + index.store(Id.of("three"), null, new OtherStorable("content")); + } + + try (LuceneIndex index = createIndex()) { + index.deleteByType(Storable.class); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldDeleteByIdAnyType() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some text")); + index.store(ONE, null, new OtherStorable("Some other text")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(ONE, Storable.class); + } + + assertHits(ID, "one", 1); + ScoreDoc[] docs = assertHits(ID, "one", 1); + Document doc = doc(docs[0].doc); + assertThat(doc.get("value")).isEqualTo("Some other text"); + } + + @Test + void shouldDeleteByIdAndRepository() throws IOException { + Id withRepository = ONE.withRepository("4211"); + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + index.store(withRepository, null, new Storable("New stuff")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(withRepository, Storable.class); + } + + ScoreDoc[] docs = assertHits(ID, "one", 1); + Document doc = doc(docs[0].doc); + assertThat(doc.get("value")).isEqualTo("Some other text"); + } + + @Test + void shouldDeleteByRepository() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), null, new Storable("Some other text")); + index.store(ONE.withRepository("4212"), null, new Storable("New stuff")); + } + + try (LuceneIndex index = createIndex()) { + index.deleteByRepository("4212"); + } + + assertHits(ID, "one", 1); + } + + @Test + void shouldStorePermission() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text")); + } + + assertHits(PERMISSION, "repo:4211:read", 1); + } + + private Document doc(int doc) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + return reader.document(doc); + } + } + + @CanIgnoreReturnValue + private ScoreDoc[] assertHits(String field, String value, int expectedHits) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs docs = searcher.search(new TermQuery(new Term(field, value)), 10); + assertThat(docs.totalHits.value).isEqualTo(expectedHits); + return docs.scoreDocs; + } + } + + private LuceneIndex createIndex() throws IOException { + return new LuceneIndex(new DocumentConverter(), createWriter()); + } + + private IndexWriter createWriter() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory, config); + } + + @Value + private static class Storable { + @Indexed + String value; + } + + @Value + private static class OtherStorable { + @Indexed + String value; + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java new file mode 100644 index 0000000000..fff26de53c --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java @@ -0,0 +1,592 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@SubjectAware(value = "trillian", permissions = "abc") +@ExtendWith({MockitoExtension.class, ShiroExtension.class}) +class LuceneQueryBuilderTest { + + private Directory directory; + + @Mock + private IndexOpener opener; + + @BeforeEach + void setUpDirectory() { + directory = new ByteBuffersDirectory(); + } + + @Test + void shouldReturnHitsForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldMatchPartial() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(personDoc("Trillian")); + } + + QueryResult result = query(Person.class, "Trill"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfStarIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "Tr*ll"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfQuestionMarkIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "Tr?ll"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfExpertQueryIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "lastName:Trill"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + void shouldSupportFieldsFromParentClass() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Dent"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldIgnoreHitsOfOtherType() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + writer.addDocument(personDoc("Dent")); + } + + QueryResult result = query(InetOrgPerson.class, "Dent"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldThrowQueryParseExceptionOnInvalidQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(personDoc("Dent")); + } + assertThrows(QueryParseException.class, () -> query(String.class, ":~:~")); + } + + @Test + void shouldIgnoreNonDefaultFieldsForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "car")); + } + + QueryResult result = query(InetOrgPerson.class, "car"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + void shouldUseBoostFromAnnotationForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arti", "car")); + writer.addDocument(inetOrgPersonDoc("Fake", "Dent", "Arthur, Arthur, Arthur", "mycar")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + assertThat(result.getTotalHits()).isEqualTo(2); + + List hits = result.getHits(); + Hit arthur = hits.get(0); + assertValueField(arthur, "firstName", "Arthur"); + + Hit fake = hits.get(1); + assertValueField(fake, "firstName", "Fake"); + + assertThat(arthur.getScore()).isGreaterThan(fake.getScore()); + } + + private void assertValueField(Hit hit, String fieldName, Object value) { + assertThat(hit.getFields().get(fieldName)) + .isInstanceOfSatisfying(Hit.ValueField.class, (field) -> { + assertThat(field.isHighlighted()).isFalse(); + assertThat(field.getValue()).isEqualTo(value); + }); + } + + @Test + void shouldReturnHitsForExpertQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Awesome content one")); + writer.addDocument(simpleDoc("Awesome content two")); + writer.addDocument(simpleDoc("Awesome content three")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isEqualTo(3L); + assertThat(result.getHits()).hasSize(3); + } + + @Test + void shouldReturnOnlyHitsOfTypeForExpertQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Ford", "Prefect", "Ford Prefect", "4211")); + writer.addDocument(personDoc("Prefect")); + } + + QueryResult result = query(InetOrgPerson.class, "lastName:prefect"); + assertThat(result.getTotalHits()).isEqualTo(1L); + } + + @Test + void shouldReturnOnlyPermittedHits() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(permissionDoc("Awesome content one", "abc")); + writer.addDocument(permissionDoc("Awesome content two", "cde")); + writer.addDocument(permissionDoc("Awesome content three", "fgh")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isOne(); + + List hits = result.getHits(); + assertThat(hits).hasSize(1).allSatisfy(hit -> { + assertValueField(hit, "content", "Awesome content one"); + assertThat(hit.getScore()).isGreaterThan(0f); + }); + } + + @Test + void shouldFilterByRepository() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(repositoryDoc("Awesome content one", "abc")); + writer.addDocument(repositoryDoc("Awesome content two", "cde")); + writer.addDocument(repositoryDoc("Awesome content three", "fgh")); + } + + QueryResult result; + try (DirectoryReader reader = DirectoryReader.open(directory)) { + when(opener.openForRead("default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder( + opener, "default", new StandardAnalyzer() + ); + result = builder.repository("cde").execute(Simple.class, "content:awesome"); + } + + assertThat(result.getTotalHits()).isOne(); + + List hits = result.getHits(); + assertThat(hits).hasSize(1).allSatisfy(hit -> { + assertValueField(hit, "content", "Awesome content two"); + assertThat(hit.getScore()).isGreaterThan(0f); + }); + } + + @Test + void shouldReturnStringFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Awesome")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "content", "Awesome") + ); + } + + @Test + void shouldReturnIdOfHit() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Slarti", "Bartfass", "Slartibartfass", "-")); + } + + QueryResult result = query(InetOrgPerson.class, "lastName:Bartfass"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertThat(hit.getId()).isEqualTo("Bartfass") + ); + } + + @Test + void shouldReturnTypeOfHits() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("We need the type")); + } + + QueryResult result = query(Simple.class, "content:type"); + assertThat(result.getType()).isEqualTo(Simple.class); + } + + @Test + void shouldSupportIntRangeQueries() throws IOException { + Instant now = Instant.now(); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + + QueryResult result = query(Types.class, "intValue:[0 TO 100]"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "intValue", 42) + ); + } + + @Test + void shouldSupportLongRangeQueries() throws IOException { + Instant now = Instant.now(); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + + QueryResult result = query(Types.class, "longValue:[0 TO 100]"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "longValue", 21L) + ); + } + + @Test + void shouldSupportInstantRangeQueries() throws IOException { + Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli()); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + long before = now.minus(1, ChronoUnit.MINUTES).toEpochMilli(); + long after = now.plus(1, ChronoUnit.MINUTES).toEpochMilli(); + + String queryString = String.format("instantValue:[%d TO %d]", before, after); + + QueryResult result = query(Types.class, queryString); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "instantValue", now) + ); + } + + @Test + void shouldSupportQueryForBooleanFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(21, 42L, true, Instant.now())); + } + + QueryResult result = query(Types.class, "boolValue:true"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "boolValue", Boolean.TRUE) + ); + } + + @Test + void shouldReturnValueFieldForHighlightedFieldWithoutFragment() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Marvin", "HoG", "Paranoid Android", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Marvin"); + Hit hit = result.getHits().get(0); + assertValueField(hit, "displayName", "Paranoid Android"); + } + + @Test + void shouldFailBestGuessQueryWithoutDefaultQueryFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(1, 2L, false, Instant.now())); + } + assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something")); + } + + @Test + void shouldLimitHitsByDefaultSize() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + i)); + } + + QueryResult result = query(Simple.class, "content:counter"); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(10); + } + + @Test + void shouldLimitHitsByConfiguredSize() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + (i + 1))); + } + + QueryResult result = query(Simple.class, "content:counter", null, 2); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(2); + + assertContainsValues( + result, "content", "counter 1", "counter 2" + ); + } + + @Test + void shouldRespectStartValue() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + (i + 1))); + } + + QueryResult result = query(Simple.class, "content:counter", 10, 3); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(3); + + assertContainsValues( + result, "content", "counter 11", "counter 12", "counter 13" + ); + } + + private void assertContainsValues(QueryResult result, String fieldName, Object... expectedValues) { + List values = result.getHits().stream().map(hit -> { + Hit.ValueField content = (Hit.ValueField) hit.getFields().get(fieldName); + return content.getValue(); + }).collect(Collectors.toList()); + assertThat(values).containsExactly(expectedValues); + } + + @Test + void shouldBeAbleToMarshalQueryResultToJson() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + ObjectMapper mapper = new ObjectMapper(); + + JsonNode root = mapper.valueToTree(result); + assertThat(root.get("totalHits").asInt()).isOne(); + + JsonNode hit = root.get("hits").get(0); + assertThat(hit.get("score").asDouble()).isGreaterThan(0d); + + JsonNode fields = hit.get("fields"); + JsonNode firstName = fields.get("firstName"); + assertThat(firstName.get("highlighted").asBoolean()).isFalse(); + assertThat(firstName.get("value").asText()).isEqualTo("Arthur"); + + JsonNode displayName = fields.get("displayName"); + assertThat(displayName.get("highlighted").asBoolean()).isTrue(); + assertThat(displayName.get("fragments").get(0).asText()).contains("**Arthur**"); + } + + @Test + void shouldBeAbleToMarshalDifferentTypesOfQueryResultToJson() throws IOException { + Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli()); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(21, 42L, true, now)); + } + + QueryResult result = query(Types.class, "intValue:21"); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + mapper.registerModule(new JavaTimeModule()); + + JsonNode root = mapper.valueToTree(result); + JsonNode fields = root.get("hits").get(0).get("fields"); + assertThat(fields.get("intValue").get("value").asInt()).isEqualTo(21); + assertThat(fields.get("longValue").get("value").asLong()).isEqualTo(42L); + assertThat(fields.get("boolValue").get("value").asBoolean()).isTrue(); + assertThat(fields.get("instantValue").get("value").asText()).isEqualTo(now.toString()); + } + + private QueryResult query(Class type, String queryString) throws IOException { + return query(type, queryString, null, null); + } + + private QueryResult query(Class type, String queryString, Integer start, Integer limit) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + lenient().when(opener.openForRead("default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder( + opener, "default", new StandardAnalyzer() + ); + if (start != null) { + builder.start(start); + } + if (limit != null) { + builder.limit(limit); + } + return builder.execute(type, queryString); + } + } + + private IndexWriter writer() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); + return new IndexWriter(directory, config); + } + + private Document simpleDoc(String content) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + return document; + } + + private Document permissionDoc(String content, String permission) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES)); + return document; + } + + private Document repositoryDoc(String content, String repository) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES)); + return document; + } + + private Document inetOrgPersonDoc(String firstName, String lastName, String displayName, String carLicense) { + Document document = new Document(); + document.add(new TextField("firstName", firstName, Field.Store.YES)); + document.add(new TextField("lastName", lastName, Field.Store.YES)); + document.add(new TextField("displayName", displayName, Field.Store.YES)); + document.add(new TextField("carLicense", carLicense, Field.Store.YES)); + document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, InetOrgPerson.class.getName(), Field.Store.YES)); + return document; + } + + private Document personDoc(String lastName) { + Document document = new Document(); + document.add(new TextField("lastName", lastName, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Person.class.getName(), Field.Store.YES)); + return document; + } + + private Document typesDoc(int intValue, long longValue, boolean boolValue, Instant instantValue) { + Document document = new Document(); + document.add(new IntPoint("intValue", intValue)); + document.add(new StoredField("intValue", intValue)); + document.add(new LongPoint("longValue", longValue)); + document.add(new StoredField("longValue", longValue)); + document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES)); + document.add(new LongPoint("instantValue", instantValue.toEpochMilli())); + document.add(new StoredField("instantValue", instantValue.toEpochMilli())); + document.add(new StringField(FieldNames.TYPE, Types.class.getName(), Field.Store.YES)); + return document; + } + + static class Types { + + @Indexed + private Integer intValue; + @Indexed + private long longValue; + @Indexed + private boolean boolValue; + @Indexed + private Instant instantValue; + + } + + static class Person { + + @Indexed(defaultQuery = true) + private String lastName; + } + + static class InetOrgPerson extends Person { + + @Indexed(defaultQuery = true, boost = 2f) + private String firstName; + + @Indexed(defaultQuery = true, highlighted = true) + private String displayName; + + @Indexed + private String carLicense; + } + + static class Simple { + @Indexed(defaultQuery = true) + private String content; + } + +}