From e321133ff72a6767cb92e31150eec879e48062bf Mon Sep 17 00:00:00 2001 From: Sebastian Sdorra Date: Wed, 14 Jul 2021 11:49:38 +0200 Subject: [PATCH] Add search engine and quick search for repositories (#1727) Add a powerful search engine based on lucene to the scm-manager api. The api can be used to index objects, simply by annotating them and add them to an index. The first indexed object is the repository which could queried by quick search in the header. --- gradle/changelog/search.yaml | 4 + gradle/dependencies.gradle | 7 + .../java/sonia/scm/repository/Repository.java | 27 +- .../src/main/java/sonia/scm/search/Hit.java | 95 +++ .../src/main/java/sonia/scm/search/Id.java | 158 +++++ .../src/main/java/sonia/scm/search/Index.java | 75 +++ .../main/java/sonia/scm/search/IndexLog.java | 57 ++ .../java/sonia/scm/search/IndexLogStore.java | 61 ++ .../java/sonia/scm/search/IndexNames.java | 43 ++ .../java/sonia/scm/search/IndexOptions.java | 101 +++ .../java/sonia/scm/search/IndexQueue.java | 60 ++ .../main/java/sonia/scm/search/Indexed.java | 144 +++++ .../NoDefaultQueryFieldsFoundException.java | 41 ++ .../java/sonia/scm/search/QueryBuilder.java | 113 ++++ .../sonia/scm/search/QueryParseException.java | 51 ++ .../java/sonia/scm/search/QueryResult.java | 57 ++ .../java/sonia/scm/search/SearchEngine.java | 83 +++ .../scm/search/SearchEngineException.java | 44 ++ .../main/java/sonia/scm/web/VndMediaType.java | 2 + .../test/java/sonia/scm/search/IdTest.java | 128 ++++ scm-ui/ui-api/src/index.ts | 1 + scm-ui/ui-api/src/search.ts | 56 ++ .../ui-components/src/forms/FilterInput.tsx | 2 +- scm-ui/ui-types/src/Search.ts | 50 ++ scm-ui/ui-types/src/index.ts | 1 + .../ui-webapp/public/locales/de/commons.json | 7 + .../ui-webapp/public/locales/de/groups.json | 2 +- scm-ui/ui-webapp/public/locales/de/repos.json | 2 +- scm-ui/ui-webapp/public/locales/de/users.json | 2 +- .../ui-webapp/public/locales/en/commons.json | 7 + .../ui-webapp/public/locales/en/groups.json | 2 +- scm-ui/ui-webapp/public/locales/en/repos.json | 2 +- scm-ui/ui-webapp/public/locales/en/users.json | 2 +- .../src/containers/NavigationBar.tsx | 3 + .../src/containers/Notifications.tsx | 10 +- .../ui-webapp/src/containers/OmniSearch.tsx | 254 ++++++++ .../src/groups/containers/Groups.tsx | 2 +- .../src/repos/containers/Overview.tsx | 2 +- .../ui-webapp/src/users/containers/Users.tsx | 2 +- scm-webapp/build.gradle | 5 + .../scm/api/v2/resources/CollectionDto.java | 3 +- .../sonia/scm/api/v2/resources/HitDto.java | 49 ++ .../api/v2/resources/IndexDtoGenerator.java | 2 + .../scm/api/v2/resources/MapperModule.java | 2 + .../scm/api/v2/resources/QueryResultDto.java | 42 ++ .../api/v2/resources/QueryResultMapper.java | 120 ++++ .../scm/api/v2/resources/ResourceLinks.java | 17 + .../api/v2/resources/SearchParameters.java | 61 ++ .../scm/api/v2/resources/SearchResource.java | 111 ++++ .../lifecycle/modules/ScmServletModule.java | 11 + .../scm/repository/IndexUpdateListener.java | 128 ++++ .../sonia/scm/search/AnalyzerFactory.java | 62 ++ .../scm/search/DefaultIndexLogStore.java | 59 ++ .../sonia/scm/search/DefaultIndexQueue.java | 77 +++ .../sonia/scm/search/DocumentConverter.java | 144 +++++ .../java/sonia/scm/search/FieldNames.java | 35 ++ .../java/sonia/scm/search/IndexOpener.java | 65 ++ .../java/sonia/scm/search/IndexQueueTask.java | 32 + .../scm/search/IndexQueueTaskWrapper.java | 56 ++ .../scm/search/IndexableFieldFactory.java | 32 + .../sonia/scm/search/IndexableFields.java | 172 +++++ .../java/sonia/scm/search/LuceneIndex.java | 109 ++++ .../sonia/scm/search/LuceneQueryBuilder.java | 149 +++++ .../scm/search/LuceneQueryBuilderFactory.java | 44 ++ .../sonia/scm/search/LuceneSearchEngine.java | 57 ++ .../scm/search/NonReadableFieldException.java | 31 + .../scm/search/PermissionAwareCollector.java | 88 +++ .../main/java/sonia/scm/search/Queries.java | 54 ++ .../sonia/scm/search/QueryResultFactory.java | 103 +++ .../java/sonia/scm/search/QueuedIndex.java | 71 +++ .../sonia/scm/search/SearchableField.java | 66 ++ .../java/sonia/scm/search/SearchableType.java | 51 ++ .../sonia/scm/search/SearchableTypes.java | 82 +++ .../main/java/sonia/scm/search/TypeCheck.java | 53 ++ .../UnsupportedTypeOfFieldException.java | 31 + .../java/sonia/scm/search/ValueExtractor.java | 32 + .../sonia/scm/search/ValueExtractors.java | 95 +++ .../v2/resources/IndexDtoGeneratorTest.java | 1 + .../api/v2/resources/ResourceLinksMock.java | 1 + .../api/v2/resources/SearchResourceTest.java | 266 ++++++++ .../repository/IndexUpdateListenerTest.java | 147 +++++ .../sonia/scm/search/AnalyzerFactoryTest.java | 78 +++ .../scm/search/DefaultIndexLogStoreTest.java | 61 ++ .../scm/search/DefaultIndexQueueTest.java | 145 +++++ .../scm/search/DocumentConverterTest.java | 288 +++++++++ .../sonia/scm/search/IndexOpenerTest.java | 99 +++ .../sonia/scm/search/LuceneIndexTest.java | 238 +++++++ .../scm/search/LuceneQueryBuilderTest.java | 592 ++++++++++++++++++ 88 files changed, 6052 insertions(+), 25 deletions(-) create mode 100644 gradle/changelog/search.yaml create mode 100644 scm-core/src/main/java/sonia/scm/search/Hit.java create mode 100644 scm-core/src/main/java/sonia/scm/search/Id.java create mode 100644 scm-core/src/main/java/sonia/scm/search/Index.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexLog.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexLogStore.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexNames.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexOptions.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexQueue.java create mode 100644 scm-core/src/main/java/sonia/scm/search/Indexed.java create mode 100644 scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java create mode 100644 scm-core/src/main/java/sonia/scm/search/QueryBuilder.java create mode 100644 scm-core/src/main/java/sonia/scm/search/QueryParseException.java create mode 100644 scm-core/src/main/java/sonia/scm/search/QueryResult.java create mode 100644 scm-core/src/main/java/sonia/scm/search/SearchEngine.java create mode 100644 scm-core/src/main/java/sonia/scm/search/SearchEngineException.java create mode 100644 scm-core/src/test/java/sonia/scm/search/IdTest.java create mode 100644 scm-ui/ui-api/src/search.ts create mode 100644 scm-ui/ui-types/src/Search.ts create mode 100644 scm-ui/ui-webapp/src/containers/OmniSearch.tsx create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java create mode 100644 scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java create mode 100644 scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/FieldNames.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/Queries.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/SearchableField.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/SearchableType.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java create mode 100644 scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java diff --git a/gradle/changelog/search.yaml b/gradle/changelog/search.yaml new file mode 100644 index 0000000000..20b7b8d661 --- /dev/null +++ b/gradle/changelog/search.yaml @@ -0,0 +1,4 @@ +- type: Added + description: API to index and query objects ([#1727](https://github.com/scm-manager/scm-manager/pull/1727)) +- type: Added + description: Quick search for repositories ([#1727](https://github.com/scm-manager/scm-manager/pull/1727)) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 9a61eb6229..9c96c249cb 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -15,6 +15,7 @@ ext { jjwtVersion = '0.11.2' bouncycastleVersion = '1.67' jettyVersion = '9.4.35.v20201120' + luceneVersion = '8.9.0' junitJupiterVersion = '5.7.0' hamcrestVersion = '2.1' @@ -148,6 +149,12 @@ ext { jettyJmx: "org.eclipse.jetty:jetty-jmx:${jettyVersion}", jettyClient: "org.eclipse.jetty:jetty-client:${jettyVersion}", + // search + luceneCore: "org.apache.lucene:lucene-core:${luceneVersion}", + luceneQueryParser: "org.apache.lucene:lucene-queryparser:${luceneVersion}", + luceneHighlighter: "org.apache.lucene:lucene-highlighter:${luceneVersion}", + luceneAnalyzersCommon: "org.apache.lucene:lucene-analyzers-common:${luceneVersion}", + // tests junitJupiterApi: "org.junit.jupiter:junit-jupiter-api:${junitJupiterVersion}", junitJupiterParams: "org.junit.jupiter:junit-jupiter-params:${junitJupiterVersion}", diff --git a/scm-core/src/main/java/sonia/scm/repository/Repository.java b/scm-core/src/main/java/sonia/scm/repository/Repository.java index 59cabbbe38..70fb56909c 100644 --- a/scm-core/src/main/java/sonia/scm/repository/Repository.java +++ b/scm-core/src/main/java/sonia/scm/repository/Repository.java @@ -31,6 +31,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; import sonia.scm.BasicPropertiesAware; import sonia.scm.ModelObject; +import sonia.scm.search.Indexed; import sonia.scm.util.Util; import sonia.scm.util.ValidationUtil; @@ -65,21 +66,28 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per private static final long serialVersionUID = 3486560714961909711L; - private String contact; - private Long creationDate; + private String id; + + @Indexed(defaultQuery = true, boost = 1.25f) + private String namespace; + + @Indexed(defaultQuery = true, boost = 1.5f) + private String name; + @Indexed(type = Indexed.Type.SEARCHABLE) + private String type; + @Indexed(defaultQuery = true, highlighted = true) private String description; + private String contact; + @Indexed + private Long creationDate; + @Indexed + private Long lastModified; @XmlTransient private List healthCheckFailures; - private String id; - private Long lastModified; - private String namespace; - private String name; @XmlElement(name = "permission") private Set permissions = new HashSet<>(); - private String type; private boolean archived; - /** * Constructs a new {@link Repository}. * This constructor is used by JAXB. @@ -162,10 +170,9 @@ public class Repository extends BasicPropertiesAware implements ModelObject, Per * @return {@link List} of {@link HealthCheckFailure}s * @since 1.36 */ - @SuppressWarnings("unchecked") public List getHealthCheckFailures() { if (healthCheckFailures == null) { - healthCheckFailures = Collections.EMPTY_LIST; + healthCheckFailures = Collections.emptyList(); } return healthCheckFailures; diff --git a/scm-core/src/main/java/sonia/scm/search/Hit.java b/scm-core/src/main/java/sonia/scm/search/Hit.java new file mode 100644 index 0000000000..b19a69951c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Hit.java @@ -0,0 +1,95 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Value; + +import java.util.Map; + +/** + * Represents an object which matched the search query. + * + * @since 2.21.0 + */ +@Beta +@Value +public class Hit { + + /** + * Id of the matched object. + */ + String id; + + /** + * The score describes how good the match was. + */ + float score; + + /** + * Fields of the matched object. + * Key of the map is the name of the field and the value is either a {@link ValueField} or a {@link HighlightedField}. + */ + Map fields; + + /** + * Base class of hit field types. + */ + @Getter + @AllArgsConstructor(access = AccessLevel.PRIVATE) + public abstract static class Field { + boolean highlighted; + } + + /** + * A field holding a complete value. + */ + @Getter + public static class ValueField extends Field { + Object value; + + public ValueField(Object value) { + super(false); + this.value = value; + } + } + + /** + * A field which consists of fragments that contain a match of the search query. + */ + @Getter + public static class HighlightedField extends Field { + String[] fragments; + + public HighlightedField(String[] fragments) { + super(true); + this.fragments = fragments; + } + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Id.java b/scm-core/src/main/java/sonia/scm/search/Id.java new file mode 100644 index 0000000000..5d47c19252 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Id.java @@ -0,0 +1,158 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import sonia.scm.ModelObject; +import sonia.scm.repository.Repository; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Optional; + +/** + * Describes the id of an indexed object. + * + * @since 2.21.0 + */ +@Beta +@ToString +@EqualsAndHashCode +public final class Id { + + private final String value; + private final String repository; + + private Id(@Nonnull String value, @Nullable String repository) { + this.value = value; + this.repository = repository; + } + + /** + * Returns the string representation of the id without the repository part. + * + * @return string representation without repository. + */ + public String getValue() { + return value; + } + + /** + * Returns the repository id part of the id or an empty optional if the id does not belong to a repository. + * @return repository id or empty + */ + public Optional getRepository() { + return Optional.ofNullable(repository); + } + + /** + * Creates the id with the id of the given repository. + * @param repository repository + * @return id with repository id + */ + public Id withRepository(@Nonnull Repository repository) { + checkRepository(repository); + return withRepository(repository.getId()); + } + + /** + * Creates the id with the given repository id. + * @param repository repository id + * @return id with repository id + */ + public Id withRepository(@Nonnull String repository) { + checkRepository(repository); + return new Id(value, repository); + } + + /** + * Returns the string representation of the id including the repository. + * @return string representation + */ + public String asString() { + if (repository != null) { + return value + "/" + repository; + } + return value; + } + + /** + * Creates a new id. + * + * @param value primary value of the id + * @param others additional values which should be part of the id + * + * @return new id + */ + public static Id of(@Nonnull String value, String... others) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(value), "primary value is required"); + String idValue = value; + if (others.length > 0) { + idValue += ":" + Joiner.on(':').join(others); + } + return new Id(idValue, null); + } + + /** + * Creates a new id for the given repository. + * + * @param repository repository + * @return id of repository + */ + public static Id of(@Nonnull Repository repository) { + checkRepository(repository); + String id = repository.getId(); + checkRepository(id); + return new Id(id, id); + } + + /** + * Creates a new id for the given model object. + * @param object model object + * @param others additional values which should be part of the id + * @return new id from model object + */ + public static Id of(@Nonnull ModelObject object, String... others) { + checkObject(object); + return of(object.getId(), others); + } + + private static void checkRepository(Repository repository) { + Preconditions.checkArgument(repository != null, "repository is required"); + } + + private static void checkRepository(String repository) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(repository), "repository id is required"); + } + + private static void checkObject(@Nonnull ModelObject object) { + Preconditions.checkArgument(object != null, "object is required"); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/Index.java b/scm-core/src/main/java/sonia/scm/search/Index.java new file mode 100644 index 0000000000..bebdfed559 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Index.java @@ -0,0 +1,75 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Can be used to index objects for full text searches. + * + * @since 2.21.0 + */ +@Beta +public interface Index extends AutoCloseable { + + /** + * Store the given object in the index. + * All fields of the object annotated with {@link Indexed} will be stored in the index. + * + * @param id identifier of the object in the index + * @param permission Shiro permission string representing the required permission to see this object as a result + * @param object object to index + * + * @see Indexed + */ + void store(Id id, String permission, Object object); + + /** + * Delete the object with the given id and type from index. + * + * @param id id of object + * @param type type of object + */ + void delete(Id id, Class type); + + /** + * Delete all objects which are related the given repository from index. + * + * @param repositoryId id of repository + */ + void deleteByRepository(String repositoryId); + + /** + * Delete all objects with the given type from index. + * @param type type of objects + */ + void deleteByType(Class type); + + /** + * Close index and commit changes. + */ + @Override + void close(); +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexLog.java b/scm-core/src/main/java/sonia/scm/search/IndexLog.java new file mode 100644 index 0000000000..76ea32d357 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexLog.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Data; +import sonia.scm.xml.XmlInstantAdapter; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.time.Instant; + +/** + * A marker keeping track of when and with which model version an object type was last indexed. + * @since 2.21 + */ +@Beta +@Data +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class IndexLog { + + private int version = 1; + @XmlJavaTypeAdapter(XmlInstantAdapter.class) + private Instant date = Instant.now(); + + public IndexLog() { + } + + public IndexLog(int version) { + this.version = version; + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java b/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java new file mode 100644 index 0000000000..9b3d7e66ef --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexLogStore.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.util.Optional; + +/** + * Can be used to mark when a type of object was last indexed and with which version. + * This is useful to detect and mark if a bootstrap index was created for the kind of object + * or if the way how an object is indexed has changed. + * + * @since 2.21.0 + */ +@Beta +public interface IndexLogStore { + + /** + * Log index and version of a type which is now indexed. + * + * @param index name of index + * @param type type which was indexed + * @param version model version + */ + void log(String index, Class type, int version); + + /** + * Returns version and date of the indexed type or an empty object, + * if the object was not indexed at all. + * + * @param index name if index + * @param type type of object + * + * @return log entry or empty + */ + Optional get(String index, Class type); + +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexNames.java b/scm-core/src/main/java/sonia/scm/search/IndexNames.java new file mode 100644 index 0000000000..e264be3e07 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexNames.java @@ -0,0 +1,43 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Names of predefined indexes. + * @since 2.21.0 + */ +@Beta +public final class IndexNames { + + /** + * The default index. + */ + public static final String DEFAULT = "_default"; + + private IndexNames() { + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java new file mode 100644 index 0000000000..a528ee5628 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java @@ -0,0 +1,101 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.util.Locale; + +/** + * Options to configure how things are indexed and searched. + * + * @since 2.21.0 + */ +@Beta +public class IndexOptions { + + private final Type type; + private final Locale locale; + + private IndexOptions(Type type, Locale locale) { + this.type = type; + this.locale = locale; + } + + /** + * Returns the type of the index. + * @return type of index + */ + public Type getType() { + return type; + } + + /** + * Returns the locale of the index content. + * + * @return locale of index content + */ + public Locale getLocale() { + return locale; + } + + /** + * Returns the default index options which should match most of the use cases. + * + * @return default index options + */ + public static IndexOptions defaults() { + return new IndexOptions(Type.GENERIC, Locale.ENGLISH); + } + + /** + * Returns index options for a specific language. + * This options should be used if the content is written in a specific natural language. + * + * @param locale natural language of content + * + * @return options for content in natural language + */ + public static IndexOptions naturalLanguage(Locale locale) { + return new IndexOptions(Type.NATURAL_LANGUAGE, locale); + } + + /** + * Type of indexing. + */ + public enum Type { + + /** + * Not specified content. + */ + GENERIC, + + /** + * Content in natural language. + */ + NATURAL_LANGUAGE; + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexQueue.java b/scm-core/src/main/java/sonia/scm/search/IndexQueue.java new file mode 100644 index 0000000000..2a49c51e6c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexQueue.java @@ -0,0 +1,60 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Queue the work of indexing. + * An index can't be opened in parallel, so the queue coordinates the work of indexing in an asynchronous manner. + * {@link IndexQueue} should be used most of the time to index content. + * + * @since 2.21.0 + */ +@Beta +public interface IndexQueue { + + /** + * Returns an index which queues every change to the content. + * + * @param name name of index + * @param indexOptions options for the index + * + * @return index which queues changes + */ + Index getQueuedIndex(String name, IndexOptions indexOptions); + + /** + * Returns an index which with default options which queues every change to the content. + * @param name name of index + * + * @return index with default options which queues changes + * @see IndexOptions#defaults() + */ + default Index getQueuedIndex(String name) { + return getQueuedIndex(name, IndexOptions.defaults()); + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Indexed.java b/scm-core/src/main/java/sonia/scm/search/Indexed.java new file mode 100644 index 0000000000..f54e7c879a --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/Indexed.java @@ -0,0 +1,144 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Mark a field which should be indexed. + * @since 2.21.0 + */ +@Beta +@Documented +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Indexed { + + /** + * Name of the field. + * If not set the name of the annotated field is used. + * + * @return name of field + */ + String name() default ""; + + /** + * Describes how the field is indexed. + * + * @return type of indexing + */ + Type type() default Type.TOKENIZED; + + /** + * {@code true} if this field should be part of default query for this type of object. + * + * @return {@code true} if field is part of default query + */ + boolean defaultQuery() default false; + + /** + * Boost the object if the searched query matches this field. + * Greater than one brings the object further up in the search results. + * Smaller than one devalues the object in the search results. + * + * @return boost score + */ + float boost() default 1f; + + /** + * {@code true} to search the field value for matches and returns fragments with those matches instead of the whole value. + * + * @return {@code true} to return matched fragments + */ + boolean highlighted() default false; + + /** + * Describes how the field is indexed. + */ + enum Type { + /** + * The value of the field is analyzed and split into tokens, which allows searches for parts of the value. + * Tokenization only works for string values. If a field with another type is marked as tokenized, + * the field is indexed as if it was marked as {@link #SEARCHABLE}. + */ + TOKENIZED(true, true, true), + + /** + * The value can only be searched as a whole. + * Numeric fields can also be search as part of a range, + * but strings are only found if the query contains the whole field value. + */ + SEARCHABLE(false, true, true), + + /** + * Value of the field cannot be searched for, but is returned in the result. + */ + STORED_ONLY(false, false, true); + + private final boolean tokenized; + private final boolean searchable; + private final boolean stored; + + Type(boolean tokenized, boolean searchable, boolean stored) { + this.tokenized = tokenized; + this.searchable = searchable; + this.stored = stored; + } + + /** + * Returns {@code true} if the field is tokenized. + * + * @return {@code true} if tokenized + * @see #TOKENIZED + */ + public boolean isTokenized() { + return tokenized; + } + + /** + * Returns {@code true} if the field is searchable. + * + * @return {@code true} if searchable + * @see #SEARCHABLE + */ + public boolean isSearchable() { + return searchable; + } + + /** + * Returns {@code true} if the field is stored. + * @return {@code true} if stored + */ + public boolean isStored() { + return stored; + } + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java b/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java new file mode 100644 index 0000000000..20c11cc5ab --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/NoDefaultQueryFieldsFoundException.java @@ -0,0 +1,41 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Exception is thrown if a best guess query is executed + * and the search type does not contain a field which is marked for default query. + * + * @see Indexed#defaultQuery() + * @since 2.21.0 + */ +@Beta +public class NoDefaultQueryFieldsFoundException extends SearchEngineException { + public NoDefaultQueryFieldsFoundException(Class type) { + super("no default query fields defined for " + type); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java b/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java new file mode 100644 index 0000000000..7c1e81a2a0 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryBuilder.java @@ -0,0 +1,113 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Value; +import sonia.scm.repository.Repository; + +import java.util.Optional; + +/** + * Build and execute queries against an index. + * + * @since 2.21.0 + */ +@Beta +public abstract class QueryBuilder { + + private String repositoryId; + private int start = 0; + private int limit = 10; + + /** + * Return only results which are related to the given repository. + * @param repository repository + * @return {@code this} + */ + public QueryBuilder repository(Repository repository) { + return repository(repository.getId()); + } + + /** + * Return only results which are related to the repository with the given id. + * @param repositoryId id of the repository + * @return {@code this} + */ + public QueryBuilder repository(String repositoryId) { + this.repositoryId = repositoryId; + return this; + } + + /** + * The result should start at the given number. + * All matching objects before the given start are skipped. + * @param start start of result + * @return {@code this} + */ + public QueryBuilder start(int start) { + this.start = start; + return this; + } + + /** + * Defines how many hits are returned. + * @param limit limit of hits + * @return {@code this} + */ + public QueryBuilder limit(int limit) { + this.limit = limit; + return this; + } + + /** + * Executes the query and returns the matches. + * + * @param type type of objects which are searched + * @param queryString searched query + * @return result of query + */ + public QueryResult execute(Class type, String queryString){ + return execute(new QueryParams(type, repositoryId, queryString, start, limit)); + } + + protected abstract QueryResult execute(QueryParams queryParams); + + /** + * The searched query and all parameters, which belong to the query. + */ + @Value + static class QueryParams { + Class type; + String repositoryId; + String queryString; + int start; + int limit; + + public Optional getRepositoryId() { + return Optional.ofNullable(repositoryId); + } + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryParseException.java b/scm-core/src/main/java/sonia/scm/search/QueryParseException.java new file mode 100644 index 0000000000..11d943c898 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryParseException.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import sonia.scm.BadRequestException; + +import static sonia.scm.ContextEntry.ContextBuilder.only; + +/** + * Is thrown if a query with invalid syntax was executed. + * + * @since 2.21.0 + */ +@Beta +@SuppressWarnings("java:S110") // large inheritance is ok for exceptions +public class QueryParseException extends BadRequestException { + + private static final String CODE = "5VScek8Xp1"; + + public QueryParseException(String query, String message, Exception cause) { + super(only("query", query), message, cause); + } + + @Override + public String getCode() { + return CODE; + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/QueryResult.java b/scm-core/src/main/java/sonia/scm/search/QueryResult.java new file mode 100644 index 0000000000..28409ab72b --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/QueryResult.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; +import lombok.Value; + +import java.util.List; + +/** + * Result of a query. + * @since 2.21.0 + */ +@Beta +@Value +public class QueryResult { + + /** + * Total count of hits, which are matched by the query. + */ + long totalHits; + + /** + * Searched type of object. + */ + Class type; + + /** + * List of hits found by the query. + * The list contains only those hits which are starting at start and they are limit by the given amount. + * @see QueryBuilder + */ + List hits; + +} diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java new file mode 100644 index 0000000000..245db00c05 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java @@ -0,0 +1,83 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * The {@link SearchEngine} is the main entry point for indexing and searching. + * Note that this is kind of a low level api for indexing. + * For non expert indexing the {@link IndexQueue} should be used. + * + * @see IndexQueue + * @since 2.21.0 + */ +@Beta +public interface SearchEngine { + + /** + * Returns the index with the given name and the given options. + * The index is created if it does not exist. + * Warning: Be careful, because an index can't be opened multiple times in parallel. + * If you are not sure how you should index your objects, use the {@link IndexQueue}. + * + * @param name name of the index + * @param options index options + * @return existing index or a new one if none exists + */ + Index getOrCreate(String name, IndexOptions options); + + /** + * Same as {@link #getOrCreate(String, IndexOptions)} with default options. + * + * @param name name of the index + * @return existing index or a new one if none exists + * @see IndexOptions#defaults() + */ + default Index getOrCreate(String name) { + return getOrCreate(name, IndexOptions.defaults()); + } + + /** + * Search the index. + * Returns a {@link QueryBuilder} which allows to query the index. + * + * @param name name of the index + * @param options options for searching the index + * @return query builder + */ + QueryBuilder search(String name, IndexOptions options); + + /** + * Same as {@link #search(String, IndexOptions)} with default options. + * + * @param name name of the index + * @return query builder + * @see IndexOptions#defaults() + */ + default QueryBuilder search(String name) { + return search(name, IndexOptions.defaults()); + } +} diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java b/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java new file mode 100644 index 0000000000..ccc0afed5f --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngineException.java @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Generic exception which could by thrown by any part of the search engine. + * + * @since 2.21.0 + */ +@Beta +public class SearchEngineException extends RuntimeException { + + public SearchEngineException(String message) { + super(message); + } + + public SearchEngineException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-core/src/main/java/sonia/scm/web/VndMediaType.java b/scm-core/src/main/java/sonia/scm/web/VndMediaType.java index 0fe8252b21..dbb310ad7d 100644 --- a/scm-core/src/main/java/sonia/scm/web/VndMediaType.java +++ b/scm-core/src/main/java/sonia/scm/web/VndMediaType.java @@ -95,6 +95,8 @@ public class VndMediaType { public static final String NOTIFICATION_COLLECTION = PREFIX + "notificationCollection" + SUFFIX; + public static final String QUERY_RESULT = PREFIX + "queryResult" + SUFFIX; + private VndMediaType() { } diff --git a/scm-core/src/test/java/sonia/scm/search/IdTest.java b/scm-core/src/test/java/sonia/scm/search/IdTest.java new file mode 100644 index 0000000000..ae38ec2682 --- /dev/null +++ b/scm-core/src/test/java/sonia/scm/search/IdTest.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.Test; +import sonia.scm.ModelObject; +import sonia.scm.repository.Repository; +import sonia.scm.user.User; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class IdTest { + + @Test + void shouldCreateIdFromPrimary() { + Id id = Id.of("one"); + assertThat(id.getValue()).isEqualTo("one"); + } + + @Test + void shouldCreateIdWithoutRepository() { + Id id = Id.of("one"); + assertThat(id.getRepository()).isEmpty(); + } + + @Test + void shouldFailWithoutPrimaryValue() { + assertThrows(IllegalArgumentException.class, () -> Id.of((String) null)); + } + + @Test + void shouldFailWithEmptyPrimaryValue() { + assertThrows(IllegalArgumentException.class, () -> Id.of("")); + } + + @Test + void shouldCreateCombinedValue() { + Id id = Id.of("one", "two", "three"); + assertThat(id.getValue()).isEqualTo("one:two:three"); + } + + @Test + void shouldAddRepositoryId() { + Id id = Id.of("one").withRepository("4211"); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldAddRepository() { + Repository repository = new Repository(); + repository.setId("4211"); + Id id = Id.of("one").withRepository(repository); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldCreateIdFromRepository() { + Repository repository = new Repository(); + repository.setId("4211"); + Id id = Id.of(repository); + assertThat(id.getRepository()).contains("4211"); + } + + @Test + void shouldFailWithoutRepository() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((Repository) null)); + } + + @Test + void shouldFailWithoutRepositoryId() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null)); + } + + @Test + void shouldFailWithEmptyRepositoryId() { + Id id = Id.of("one"); + assertThrows(IllegalArgumentException.class, () -> id.withRepository((String) null)); + } + + @Test + void shouldCreateIdFromModelObject() { + Id id = Id.of(new User("trillian")); + assertThat(id.getValue()).isEqualTo("trillian"); + } + + @Test + void shouldFailWithoutModelObject() { + assertThrows(IllegalArgumentException.class, () -> Id.of((ModelObject) null)); + } + + @Test + void shouldReturnSimpleIdAsString() { + Id id = Id.of("one", "two"); + assertThat(id.asString()).isEqualTo("one:two"); + } + + @Test + void shouldReturnIdWithRepositoryAsString() { + Id id = Id.of("one", "two").withRepository("4211"); + assertThat(id.asString()).isEqualTo("one:two/4211"); + } + +} diff --git a/scm-ui/ui-api/src/index.ts b/scm-ui/ui-api/src/index.ts index 3a007af22c..5cb88941a6 100644 --- a/scm-ui/ui-api/src/index.ts +++ b/scm-ui/ui-api/src/index.ts @@ -56,6 +56,7 @@ export * from "./fileContent"; export * from "./history"; export * from "./contentType"; export * from "./annotations"; +export * from "./search"; export { default as ApiProvider } from "./ApiProvider"; export * from "./ApiProvider"; diff --git a/scm-ui/ui-api/src/search.ts b/scm-ui/ui-api/src/search.ts new file mode 100644 index 0000000000..1773fa348b --- /dev/null +++ b/scm-ui/ui-api/src/search.ts @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { ApiResult, useRequiredIndexLink } from "./base"; +import { QueryResult } from "@scm-manager/ui-types"; +import { apiClient } from "./apiclient"; +import { createQueryString } from "./utils"; +import { useQuery } from "react-query"; + +export type SearchOptions = { + page?: number; + pageSize?: number; +}; + +const defaultSearchOptions: SearchOptions = {}; + +export const useSearch = (query: string, options = defaultSearchOptions): ApiResult => { + const link = useRequiredIndexLink("search"); + + const queryParams: Record = {}; + queryParams.q = query; + if (options.page) { + queryParams.page = options.page.toString(); + } + if (options.pageSize) { + queryParams.pageSize = options.pageSize.toString(); + } + return useQuery( + ["search", query], + () => apiClient.get(`${link}?${createQueryString(queryParams)}`).then((response) => response.json()), + { + enabled: query.length > 1, + } + ); +}; diff --git a/scm-ui/ui-components/src/forms/FilterInput.tsx b/scm-ui/ui-components/src/forms/FilterInput.tsx index 66d4b27fac..6d9d4f4475 100644 --- a/scm-ui/ui-components/src/forms/FilterInput.tsx +++ b/scm-ui/ui-components/src/forms/FilterInput.tsx @@ -81,7 +81,7 @@ const FilterInput: FC = ({ filter, value, testId, placeholder, autoFocus, autoFocus={autoFocus || false} /> - + diff --git a/scm-ui/ui-types/src/Search.ts b/scm-ui/ui-types/src/Search.ts new file mode 100644 index 0000000000..da9bdced1b --- /dev/null +++ b/scm-ui/ui-types/src/Search.ts @@ -0,0 +1,50 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { HalRepresentation, PagedCollection } from "./hal"; + +export type ValueField = { + highlighted: false; + value: unknown; +}; + +export type HighligthedField = { + highlighted: true; + fragments: string[]; +}; + +export type Field = ValueField | HighligthedField; + +export type Hit = HalRepresentation & { + score: number; + fields: { [name: string]: Field }; +}; + +export type HitEmbedded = { + hits: Hit[]; +}; + +export type QueryResult = PagedCollection & { + type: string; +}; diff --git a/scm-ui/ui-types/src/index.ts b/scm-ui/ui-types/src/index.ts index 84a7e872b2..8d6d1e13bf 100644 --- a/scm-ui/ui-types/src/index.ts +++ b/scm-ui/ui-types/src/index.ts @@ -70,3 +70,4 @@ export * from "./Notifications"; export * from "./ApiKeys"; export * from "./PublicKeys"; export * from "./GlobalPermissions"; +export * from "./Search"; diff --git a/scm-ui/ui-webapp/public/locales/de/commons.json b/scm-ui/ui-webapp/public/locales/de/commons.json index 58eed0b2dc..71c8f5ed1d 100644 --- a/scm-ui/ui-webapp/public/locales/de/commons.json +++ b/scm-ui/ui-webapp/public/locales/de/commons.json @@ -150,5 +150,12 @@ "d_plural": "{{count}} Tage", "w": "{{count}} Woche", "w_plural": "{{count}} Wochen" + }, + "search": { + "quickSearch": { + "resultHeading": "Top-Ergebnisse Repositories", + "parseError": "Der Suchstring is ungültig", + "noResults": "Es konnten keine Repositories gefunden werden" + } } } diff --git a/scm-ui/ui-webapp/public/locales/de/groups.json b/scm-ui/ui-webapp/public/locales/de/groups.json index a351a3eda1..0e32d34da5 100644 --- a/scm-ui/ui-webapp/public/locales/de/groups.json +++ b/scm-ui/ui-webapp/public/locales/de/groups.json @@ -26,7 +26,7 @@ } }, "overview": { - "searchGroup": "Gruppe suchen" + "filterGroup": "Gruppen filtern" }, "add-group": { "title": "Gruppe erstellen", diff --git a/scm-ui/ui-webapp/public/locales/de/repos.json b/scm-ui/ui-webapp/public/locales/de/repos.json index ef26670e0b..f2d3641264 100644 --- a/scm-ui/ui-webapp/public/locales/de/repos.json +++ b/scm-ui/ui-webapp/public/locales/de/repos.json @@ -59,7 +59,7 @@ "noRepositories": "Keine Repositories gefunden.", "invalidNamespace": "Keine Repositories gefunden. Möglicherweise existiert der ausgewählte Namespace nicht.", "createButton": "Repository hinzufügen", - "searchRepository": "Repository suchen", + "filterRepositories": "Repositories filtern", "allNamespaces": "Alle Namespaces" }, "create": { diff --git a/scm-ui/ui-webapp/public/locales/de/users.json b/scm-ui/ui-webapp/public/locales/de/users.json index 3936edb43b..aa7185f2c2 100644 --- a/scm-ui/ui-webapp/public/locales/de/users.json +++ b/scm-ui/ui-webapp/public/locales/de/users.json @@ -31,7 +31,7 @@ "createButton": "Benutzer erstellen" }, "overview": { - "searchUser": "Benutzer suchen" + "filterUser": "Benutzer filtern" }, "singleUser": { "errorTitle": "Fehler", diff --git a/scm-ui/ui-webapp/public/locales/en/commons.json b/scm-ui/ui-webapp/public/locales/en/commons.json index 47fbba6a71..3af4588929 100644 --- a/scm-ui/ui-webapp/public/locales/en/commons.json +++ b/scm-ui/ui-webapp/public/locales/en/commons.json @@ -151,5 +151,12 @@ "d_plural": "{{count}} days", "w": "{{count}} week", "w_plural": "{{count}} weeks" + }, + "search": { + "quickSearch": { + "resultHeading": "Top repository results", + "parseError": "Failed to parse query", + "noResults": "Could not find matching repository" + } } } diff --git a/scm-ui/ui-webapp/public/locales/en/groups.json b/scm-ui/ui-webapp/public/locales/en/groups.json index 51b421deb4..f190558845 100644 --- a/scm-ui/ui-webapp/public/locales/en/groups.json +++ b/scm-ui/ui-webapp/public/locales/en/groups.json @@ -26,7 +26,7 @@ } }, "overview": { - "searchGroup": "Search group" + "filterGroup": "Filter groups" }, "add-group": { "title": "Create Group", diff --git a/scm-ui/ui-webapp/public/locales/en/repos.json b/scm-ui/ui-webapp/public/locales/en/repos.json index dd4e3ce9f7..ffa98e167b 100644 --- a/scm-ui/ui-webapp/public/locales/en/repos.json +++ b/scm-ui/ui-webapp/public/locales/en/repos.json @@ -59,7 +59,7 @@ "noRepositories": "No repositories found.", "invalidNamespace": "No repositories found. It's likely that the selected namespace does not exist.", "createButton": "Add Repository", - "searchRepository": "Search repository", + "filterRepositories": "Filter repositories", "allNamespaces": "All namespaces" }, "create": { diff --git a/scm-ui/ui-webapp/public/locales/en/users.json b/scm-ui/ui-webapp/public/locales/en/users.json index ca13a6a1c9..6d2cd20f54 100644 --- a/scm-ui/ui-webapp/public/locales/en/users.json +++ b/scm-ui/ui-webapp/public/locales/en/users.json @@ -45,7 +45,7 @@ } }, "overview": { - "searchUser": "Search user" + "filterUser": "Filter users" }, "createUser": { "title": "Create User", diff --git a/scm-ui/ui-webapp/src/containers/NavigationBar.tsx b/scm-ui/ui-webapp/src/containers/NavigationBar.tsx index e5ff8cabae..11c2a96f28 100644 --- a/scm-ui/ui-webapp/src/containers/NavigationBar.tsx +++ b/scm-ui/ui-webapp/src/containers/NavigationBar.tsx @@ -27,6 +27,7 @@ import classNames from "classnames"; import styled from "styled-components"; import { devices, Logo, PrimaryNavigation } from "@scm-manager/ui-components"; import Notifications from "./Notifications"; +import OmniSearch from "./OmniSearch"; import LogoutButton from "./LogoutButton"; import LoginButton from "./LoginButton"; @@ -44,6 +45,7 @@ const StyledNavBar = styled.nav` position: absolute; top: 0; left: 52px; + flex-direction: row-reverse; } } @@ -142,6 +144,7 @@ const NavigationBar: FC = ({ links }) => {
+
diff --git a/scm-ui/ui-webapp/src/containers/Notifications.tsx b/scm-ui/ui-webapp/src/containers/Notifications.tsx index 4b6e28ed00..b4d239cb3f 100644 --- a/scm-ui/ui-webapp/src/containers/Notifications.tsx +++ b/scm-ui/ui-webapp/src/containers/Notifications.tsx @@ -60,13 +60,18 @@ const DropDownMenu = styled.div` min-width: 35rem; @media screen and (max-width: ${devices.mobile.width}px) { - min-width: 25rem; + min-width: 20rem; } @media screen and (max-width: ${devices.desktop.width}px) { margin-right: 1rem; } + @media screen and (min-width: ${devices.desktop.width}px) { + right: 0; + left: auto; + } + &:before { position: absolute; content: ""; @@ -338,15 +343,12 @@ const Notifications: FC = ({ className }) => { return () => window.removeEventListener("click", close); }, []); - const isMobileView = window.matchMedia(`(max-width: ${devices.desktop.width - 1}px)`).matches; - return ( <>
{ + const namespace = (hit.fields["namespace"] as ValueField).value as string; + const name = (hit.fields["name"] as ValueField).value as string; + return `${namespace}/${name}`; +}; + +type HitsProps = { + hits: Hit[]; + index: number; + clear: () => void; +}; + +const QuickSearchNotification: FC = ({ children }) =>
{children}
; + +const EmptyHits = () => { + const [t] = useTranslation("commons"); + return ( + + {t("search.quickSearch.noResults")} + + ); +}; + +type ErrorProps = { + error: Error; +}; + +const ParseErrorNotification: FC = () => { + const [t] = useTranslation("commons"); + // TODO add link to query syntax page/modal + return ( + + {t("search.quickSearch.parseError")} + + ); +}; + +const isBackendError = (error: Error | BackendError): error is BackendError => { + return (error as BackendError).errorCode !== undefined; +}; + +const SearchErrorNotification: FC = ({ error }) => { + // 5VScek8Xp1 is the id of sonia.scm.search.QueryParseException + if (isBackendError(error) && error.errorCode === "5VScek8Xp1") { + return ; + } + return ( + + + + ); +}; + +const ResultHeading = styled.div` + border-bottom: 1px solid lightgray; + margin: 0 0.5rem; + padding: 0.375rem 0.5rem; +`; + +const Hits: FC = ({ hits, index, clear }) => { + const id = useCallback(namespaceAndName, [hits]); + const [t] = useTranslation("commons"); + + if (hits.length === 0) { + return ; + } + + return ( +
+ {t("search.quickSearch.resultHeading")} + {hits.map((hit, idx) => ( +
e.preventDefault()} onClick={clear}> + + {id(hit)} + +
+ ))} +
+ ); +}; + +const useKeyBoardNavigation = (clear: () => void, hits?: Array) => { + const [index, setIndex] = useState(-1); + const history = useHistory(); + useEffect(() => { + setIndex(-1); + }, [hits]); + + const onKeyDown = (e: KeyboardEvent) => { + // We use e.which, because ie 11 does not support e.code + // https://caniuse.com/keyboardevent-code + switch (e.which) { + case 40: // e.code: ArrowDown + if (hits) { + setIndex((idx) => { + if (idx + 1 < hits.length) { + return idx + 1; + } + return idx; + }); + } + break; + case 38: // e.code: ArrowUp + if (hits) { + setIndex((idx) => { + if (idx > 0) { + return idx - 1; + } + return idx; + }); + } + break; + case 13: // e.code: Enter + if (hits && index >= 0) { + const hit = hits[index]; + history.push(`/repo/${namespaceAndName(hit)}`); + clear(); + } + break; + case 27: // e.code: Escape + clear(); + break; + } + }; + + return { + onKeyDown, + index, + }; +}; + +const useDebounce = (value: string, delay: number) => { + const [debouncedValue, setDebouncedValue] = useState(value); + useEffect(() => { + const handler = setTimeout(() => { + setDebouncedValue(value); + }, delay); + return () => { + clearTimeout(handler); + }; + }, [value, delay]); + return debouncedValue; +}; + +const useShowResultsOnFocus = () => { + const [showResults, setShowResults] = useState(false); + return { + showResults, + onClick: (e: MouseEvent) => e.stopPropagation(), + onFocus: () => setShowResults(true), + onBlur: () => setShowResults(false) + }; +}; + +const OmniSearch: FC = () => { + const [query, setQuery] = useState(""); + const debouncedQuery = useDebounce(query, 250); + const { data, isLoading, error } = useSearch(debouncedQuery, { pageSize: 5 }); + const { showResults, ...handlers } = useShowResultsOnFocus(); + + const clearQuery = () => { + setQuery(""); + }; + const { onKeyDown, index } = useKeyBoardNavigation(clearQuery, data?._embedded.hits); + + return ( + +
+
+
+ setQuery(e.target.value)} + onKeyDown={onKeyDown} + value={query} + {...handlers} + /> + {isLoading ? null : ( + + + + )} +
+
+ {error ? : null} + {!error && data ? : null} +
+
+
+ + ); +}; + +const OmniSearchGuard: FC = ({ links }) => { + if (!links.search) { + return null; + } + return ; +}; + +export default OmniSearchGuard; diff --git a/scm-ui/ui-webapp/src/groups/containers/Groups.tsx b/scm-ui/ui-webapp/src/groups/containers/Groups.tsx index 160f1279e3..e42f0e028d 100644 --- a/scm-ui/ui-webapp/src/groups/containers/Groups.tsx +++ b/scm-ui/ui-webapp/src/groups/containers/Groups.tsx @@ -72,7 +72,7 @@ const Groups: FC = () => { showCreateButton={canCreateGroups} link="groups" label={t("create-group-button.label")} - searchPlaceholder={t("overview.searchGroup")} + searchPlaceholder={t("overview.filterGroup")} /> diff --git a/scm-ui/ui-webapp/src/repos/containers/Overview.tsx b/scm-ui/ui-webapp/src/repos/containers/Overview.tsx index bd6bfdf5d0..9fff58af5b 100644 --- a/scm-ui/ui-webapp/src/repos/containers/Overview.tsx +++ b/scm-ui/ui-webapp/src/repos/containers/Overview.tsx @@ -153,7 +153,7 @@ const Overview: FC = () => { createLink="/repos/create/" label={t("overview.createButton")} testId="repository-overview" - searchPlaceholder={t("overview.searchRepository")} + searchPlaceholder={t("overview.filterRepositories")} /> ) : null} diff --git a/scm-ui/ui-webapp/src/users/containers/Users.tsx b/scm-ui/ui-webapp/src/users/containers/Users.tsx index bf6edbbf77..b15a174945 100644 --- a/scm-ui/ui-webapp/src/users/containers/Users.tsx +++ b/scm-ui/ui-webapp/src/users/containers/Users.tsx @@ -79,7 +79,7 @@ const Users: FC = () => { showCreateButton={canAddUsers} link="users" label={t("users.createButton")} - searchPlaceholder={t("overview.searchUser")} + searchPlaceholder={t("overview.filterUser")} /> diff --git a/scm-webapp/build.gradle b/scm-webapp/build.gradle index 7ee90c3c31..474cf784e0 100644 --- a/scm-webapp/build.gradle +++ b/scm-webapp/build.gradle @@ -120,6 +120,11 @@ dependencies { // metrics implementation libraries.micrometerExtra + implementation libraries.luceneCore + implementation libraries.luceneQueryParser + implementation libraries.luceneHighlighter + implementation libraries.luceneAnalyzersCommon + // lombok compileOnly libraries.lombok testCompileOnly libraries.lombok diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java index a58430efa8..bc18b97df9 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java @@ -21,7 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.api.v2.resources; import de.otto.edison.hal.Embedded; @@ -31,6 +31,7 @@ import lombok.Getter; import lombok.Setter; @Getter @Setter +@SuppressWarnings("squid:S2160") // we do not need equals for dto class CollectionDto extends HalRepresentation { private int page; diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java new file mode 100644 index 0000000000..186967388a --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.HalRepresentation; +import de.otto.edison.hal.Links; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import sonia.scm.search.Hit; + +import java.util.Map; + +@Getter +@Setter +@NoArgsConstructor +@SuppressWarnings("java:S2160") // we do not need this for dto +public class HitDto extends HalRepresentation { + + private float score; + private Map fields; + + public HitDto(Links links, Embedded embedded) { + super(links, embedded); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java index 826b7558a9..a01cce0902 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java @@ -131,6 +131,8 @@ public class IndexDtoGenerator extends HalAppenderMapper { builder.single(link("namespaceStrategies", resourceLinks.namespaceStrategies().self())); builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self())); builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}"))); + + builder.single(link("search", resourceLinks.search().search())); } else { builder.single(link("login", resourceLinks.authentication().jsonLogin())); } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java index 06dbcff502..7d973d0d1a 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java @@ -91,5 +91,7 @@ public class MapperModule extends AbstractModule { bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class)); bind(RepositoryImportDtoToRepositoryImportParametersMapper.class).to(Mappers.getMapperClass(RepositoryImportDtoToRepositoryImportParametersMapper.class)); + + bind(QueryResultMapper.class).to(Mappers.getMapperClass(QueryResultMapper.class)); } } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java new file mode 100644 index 0000000000..f99865d9b5 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java @@ -0,0 +1,42 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.Links; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@SuppressWarnings("squid:S2160") // we do not need equals for dto +public class QueryResultDto extends CollectionDto { + + private Class type; + + QueryResultDto(Links links, Embedded embedded) { + super(links, embedded); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java new file mode 100644 index 0000000000..fa3a1e088c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java @@ -0,0 +1,120 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import com.damnhandy.uri.template.UriTemplate; +import de.otto.edison.hal.Embedded; +import de.otto.edison.hal.Links; +import de.otto.edison.hal.paging.NumberedPaging; +import de.otto.edison.hal.paging.PagingRel; +import org.mapstruct.AfterMapping; +import org.mapstruct.Context; +import org.mapstruct.Mapper; +import org.mapstruct.MappingTarget; +import org.mapstruct.ObjectFactory; +import sonia.scm.search.Hit; +import sonia.scm.search.QueryResult; +import sonia.scm.web.EdisonHalAppender; + +import javax.annotation.Nonnull; +import java.util.EnumSet; +import java.util.List; +import java.util.stream.Collectors; + +import static com.damnhandy.uri.template.UriTemplate.fromTemplate; +import static de.otto.edison.hal.Links.linkingTo; +import static de.otto.edison.hal.paging.NumberedPaging.zeroBasedNumberedPaging; + +@Mapper +public abstract class QueryResultMapper extends HalAppenderMapper { + + public abstract QueryResultDto map(@Context SearchParameters params, QueryResult result); + + @AfterMapping + void setPageValues(@MappingTarget QueryResultDto dto, QueryResult result, @Context SearchParameters params) { + int totalHits = (int) result.getTotalHits(); + dto.setPageTotal(computePageTotal(totalHits, params.getPageSize())); + dto.setPage(params.getPage()); + } + + @Nonnull + @ObjectFactory + QueryResultDto createDto(@Context SearchParameters params, QueryResult result) { + int totalHits = (int) result.getTotalHits(); + Links.Builder links = links(params, totalHits); + Embedded.Builder embedded = hits(result); + applyEnrichers(new EdisonHalAppender(links, embedded), result); + return new QueryResultDto(links.build(), embedded.build()); + } + + @Nonnull + private QueryResultDto createDto(SearchParameters params, QueryResult result, int totalHits) { + Links.Builder links = links(params, totalHits); + Embedded.Builder embedded = hits(result); + applyEnrichers(new EdisonHalAppender(links, embedded), result); + return new QueryResultDto(links.build(), embedded.build()); + } + + private Links.Builder links(SearchParameters params, int totalHits) { + NumberedPaging paging = zeroBasedNumberedPaging(params.getPage(), params.getPageSize(), totalHits); + + UriTemplate uriTemplate = fromTemplate(params.getSelfLink() + "{?q,page,pageSize}"); + uriTemplate.set("q", params.getQuery()); + + return linkingTo() + .with(paging.links( + uriTemplate, + EnumSet.allOf(PagingRel.class)) + ); + } + + @Nonnull + private Embedded.Builder hits(QueryResult result) { + List hits = result.getHits() + .stream() + .map(hit -> map(result, hit)) + .collect(Collectors.toList()); + return Embedded.embeddedBuilder().with("hits", hits); + } + + @ObjectFactory + protected HitDto createHitDto(@Context QueryResult queryResult, Hit hit) { + Links.Builder links = linkingTo(); + Embedded.Builder embedded = Embedded.embeddedBuilder(); + + applyEnrichers(new EdisonHalAppender(links, embedded), hit, queryResult); + return new HitDto(links.build(), embedded.build()); + } + + private int computePageTotal(int totalHits, int pageSize) { + if (totalHits % pageSize > 0) { + return totalHits / pageSize + 1; + } else { + return totalHits / pageSize; + } + } + + protected abstract HitDto map(@Context QueryResult queryResult, Hit hit); +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java index 726e33eb72..468e53c709 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java @@ -1113,6 +1113,23 @@ class ResourceLinks { } } + public SearchLinks search() { + return new SearchLinks(scmPathInfoStore.get()); + } + + public static class SearchLinks { + + private final LinkBuilder searchLinkBuilder; + + SearchLinks(ScmPathInfo pathInfo) { + this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class); + } + + public String search() { + return searchLinkBuilder.method("search").parameters().href(); + } + } + public InitialAdminAccountLinks initialAdminAccount() { return new InitialAdminAccountLinks(new LinkBuilder(scmPathInfoStore.get(), InitializationResource.class, AdminAccountStartupResource.class)); } diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java new file mode 100644 index 0000000000..4728a2c09d --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import lombok.Getter; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.Size; +import javax.ws.rs.DefaultValue; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.UriInfo; + +@Getter +public class SearchParameters { + + @Context + private UriInfo uriInfo; + + @Size(min = 2) + @QueryParam("q") + private String query; + + @Min(0) + @QueryParam("page") + @DefaultValue("0") + private int page = 0; + + @Min(1) + @Max(100) + @QueryParam("pageSize") + @DefaultValue("10") + private int pageSize = 10; + + String getSelfLink() { + return uriInfo.getAbsolutePath().toASCIIString(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java new file mode 100644 index 0000000000..2defad8aaa --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java @@ -0,0 +1,111 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import io.swagger.v3.oas.annotations.OpenAPIDefinition; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.tags.Tag; +import sonia.scm.repository.Repository; +import sonia.scm.search.IndexNames; +import sonia.scm.search.QueryResult; +import sonia.scm.search.SearchEngine; +import sonia.scm.web.VndMediaType; + +import javax.inject.Inject; +import javax.validation.Valid; +import javax.ws.rs.BeanParam; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; + +@Path(SearchResource.PATH) +@OpenAPIDefinition(tags = { + @Tag(name = "Search", description = "Search related endpoints") +}) +public class SearchResource { + + static final String PATH = "v2/search"; + + private final SearchEngine engine; + private final QueryResultMapper mapper; + + @Inject + public SearchResource(SearchEngine engine, QueryResultMapper mapper) { + this.engine = engine; + this.mapper = mapper; + } + + @GET + @Path("") + @Produces(VndMediaType.QUERY_RESULT) + @Operation( + summary = "Query result", + description = "Returns a collection of matched hits.", + tags = "Search", + operationId = "search_query" + ) + @ApiResponse( + responseCode = "200", + description = "success", + content = @Content( + mediaType = VndMediaType.QUERY_RESULT, + schema = @Schema(implementation = QueryResultDto.class) + ) + ) + @ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials") + @ApiResponse( + responseCode = "500", + description = "internal server error", + content = @Content( + mediaType = VndMediaType.ERROR_TYPE, + schema = @Schema(implementation = ErrorDto.class) + )) + @Parameter( + name = "query", + description = "The search expression", + required = true + ) + @Parameter( + name = "page", + description = "The requested page number of the search results (zero based, defaults to 0)" + ) + @Parameter( + name = "pageSize", + description = "The maximum number of results per page (defaults to 10)" + ) + public QueryResultDto search(@Valid @BeanParam SearchParameters params) { + QueryResult result = engine.search(IndexNames.DEFAULT) + .start(params.getPage() * params.getPageSize()) + .limit(params.getPageSize()) + .execute(Repository.class, params.getQuery()); + + return mapper.map(params, result); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java index 98445a62a6..baf3923236 100644 --- a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java +++ b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java @@ -98,6 +98,12 @@ import sonia.scm.repository.xml.XmlRepositoryDAO; import sonia.scm.repository.xml.XmlRepositoryRoleDAO; import sonia.scm.schedule.CronScheduler; import sonia.scm.schedule.Scheduler; +import sonia.scm.search.DefaultIndexLogStore; +import sonia.scm.search.DefaultIndexQueue; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexQueue; +import sonia.scm.search.LuceneSearchEngine; +import sonia.scm.search.SearchEngine; import sonia.scm.security.AccessTokenCookieIssuer; import sonia.scm.security.AuthorizationChangedEventProducer; import sonia.scm.security.ConfigurableLoginAttemptHandler; @@ -279,6 +285,11 @@ class ScmServletModule extends ServletModule { bind(NotificationSender.class).to(DefaultNotificationSender.class); bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class); + + // bind search stuff + bind(IndexQueue.class, DefaultIndexQueue.class); + bind(SearchEngine.class, LuceneSearchEngine.class); + bind(IndexLogStore.class, DefaultIndexLogStore.class); } private void bind(Class clazz, Class defaultImplementation) { diff --git a/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java b/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java new file mode 100644 index 0000000000..7bb8010729 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.repository; + +import com.github.legman.Subscribe; +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.HandlerEventType; +import sonia.scm.plugin.Extension; +import sonia.scm.search.Id; +import sonia.scm.search.Index; +import sonia.scm.search.IndexLog; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexNames; +import sonia.scm.search.IndexQueue; +import sonia.scm.web.security.AdministrationContext; +import sonia.scm.web.security.PrivilegedAction; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import java.util.Optional; + +@Extension +@Singleton +public class IndexUpdateListener implements ServletContextListener { + + private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class); + + @VisibleForTesting + static final int INDEX_VERSION = 1; + + private final AdministrationContext administrationContext; + private final IndexQueue queue; + private final IndexLogStore indexLogStore; + + @Inject + public IndexUpdateListener(AdministrationContext administrationContext, IndexQueue queue, IndexLogStore indexLogStore) { + this.administrationContext = administrationContext; + this.queue = queue; + this.indexLogStore = indexLogStore; + } + + @Subscribe(async = false) + public void handleEvent(RepositoryEvent event) { + HandlerEventType type = event.getEventType(); + if (type.isPost()) { + updateIndex(type, event.getItem()); + } + } + + + private void updateIndex(HandlerEventType type, Repository repository) { + try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) { + if (type == HandlerEventType.DELETE) { + index.deleteByRepository(repository.getId()); + } else { + store(index, repository); + } + } + } + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + Optional indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class); + if (!indexLog.isPresent()) { + LOG.debug("could not find log entry for repository index, start reindexing of all repositories"); + administrationContext.runAsAdmin(ReIndexAll.class); + indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION); + } + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + // we have nothing to destroy + } + + private static void store(Index index, Repository repository) { + index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); + } + + static class ReIndexAll implements PrivilegedAction { + + private final RepositoryManager repositoryManager; + private final IndexQueue queue; + + @Inject + public ReIndexAll(RepositoryManager repositoryManager, IndexQueue queue) { + this.repositoryManager = repositoryManager; + this.queue = queue; + } + + @Override + public void run() { + try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) { + for (Repository repository : repositoryManager.getAll()) { + store(index, repository); + } + } + } + + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java b/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java new file mode 100644 index 0000000000..c42bee93ad --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java @@ -0,0 +1,62 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.de.GermanAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.es.SpanishAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +import javax.annotation.Nonnull; + +public class AnalyzerFactory { + + @Nonnull + public Analyzer create(IndexOptions options) { + if (options.getType() == IndexOptions.Type.NATURAL_LANGUAGE) { + return createNaturalLanguageAnalyzer(options.getLocale().getLanguage()); + } + return createDefaultAnalyzer(); + } + + private Analyzer createDefaultAnalyzer() { + return new StandardAnalyzer(); + } + + private Analyzer createNaturalLanguageAnalyzer(String lang) { + switch (lang) { + case "en": + return new EnglishAnalyzer(); + case "de": + return new GermanAnalyzer(); + case "es": + return new SpanishAnalyzer(); + default: + return createDefaultAnalyzer(); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java new file mode 100644 index 0000000000..7f329d2c41 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java @@ -0,0 +1,59 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import sonia.scm.store.DataStore; +import sonia.scm.store.DataStoreFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class DefaultIndexLogStore implements IndexLogStore { + + private final DataStore dataStore; + + @Inject + public DefaultIndexLogStore(DataStoreFactory dataStoreFactory) { + this.dataStore = dataStoreFactory.withType(IndexLog.class).withName("index-log").build(); + } + + @Override + public void log(String index,Class type, int version) { + String id = id(index, type); + dataStore.put(id, new IndexLog(version)); + } + + private String id(String index, Class type) { + return index + "_" + type.getName(); + } + + @Override + public Optional get(String index, Class type) { + String id = id(index, type); + return dataStore.getOptional(id); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java new file mode 100644 index 0000000000..37e71e7b5e --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java @@ -0,0 +1,77 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.VisibleForTesting; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +@Singleton +public class DefaultIndexQueue implements IndexQueue, Closeable { + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + private final AtomicLong size = new AtomicLong(0); + + private final SearchEngine searchEngine; + + @Inject + public DefaultIndexQueue(SearchEngine searchEngine) { + this.searchEngine = searchEngine; + } + + @Override + public Index getQueuedIndex(String name, IndexOptions indexOptions) { + return new QueuedIndex(this, name, indexOptions); + } + + public SearchEngine getSearchEngine() { + return searchEngine; + } + + void enqueue(IndexQueueTaskWrapper task) { + size.incrementAndGet(); + executor.execute(() -> { + task.run(); + size.decrementAndGet(); + }); + } + + @VisibleForTesting + long getSize() { + return size.get(); + } + + @Override + public void close() throws IOException { + executor.shutdown(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java b/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java new file mode 100644 index 0000000000..095f380c9b --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java @@ -0,0 +1,144 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexableField; + +import javax.inject.Singleton; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static java.util.Collections.emptySet; + +@Singleton +public class DocumentConverter { + + private final Map, TypeConverter> typeConverter = new ConcurrentHashMap<>(); + + Document convert(Object object) { + TypeConverter converter = typeConverter.computeIfAbsent(object.getClass(), this::createTypeConverter); + try { + return converter.convert(object); + } catch (IllegalAccessException | InvocationTargetException ex) { + throw new SearchEngineException("failed to create document", ex); + } + } + + private TypeConverter createTypeConverter(Class type) { + List fieldConverters = new ArrayList<>(); + collectFields(fieldConverters, type); + return new TypeConverter(fieldConverters); + } + + private void collectFields(List fieldConverters, Class type) { + Class parent = type.getSuperclass(); + if (parent != null) { + collectFields(fieldConverters, parent); + } + for (Field field : type.getDeclaredFields()) { + Indexed indexed = field.getAnnotation(Indexed.class); + if (indexed != null) { + IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed); + Method getter = findGetter(type, field); + fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory)); + } + } + } + + private Method findGetter(Class type, Field field) { + String name = createGetterName(field); + try { + return type.getMethod(name); + } catch (NoSuchMethodException ex) { + throw new NonReadableFieldException("could not find getter for field", ex); + } + } + + private String createGetterName(Field field) { + String fieldName = field.getName(); + String prefix = "get"; + if (field.getType() == Boolean.TYPE) { + prefix = "is"; + } + return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1); + } + + private static class TypeConverter { + + private final List fieldConverters; + + private TypeConverter(List fieldConverters) { + this.fieldConverters = fieldConverters; + } + + public Document convert(Object object) throws IllegalAccessException, InvocationTargetException { + Document document = new Document(); + for (FieldConverter fieldConverter : fieldConverters) { + for (IndexableField field : fieldConverter.convert(object)) { + document.add(field); + } + } + return document; + } + } + + private static class FieldConverter { + + private final Method getter; + private final IndexableFieldFactory fieldFactory; + private final String name; + + private FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) { + this.getter = getter; + this.fieldFactory = fieldFactory; + this.name = createName(field, indexed); + } + + private String createName(Field field, Indexed indexed) { + String nameFromAnnotation = indexed.name(); + if (Strings.isNullOrEmpty(nameFromAnnotation)) { + return field.getName(); + } + return nameFromAnnotation; + } + + Iterable convert(Object object) throws IllegalAccessException, InvocationTargetException { + Object value = getter.invoke(object); + if (value != null) { + return fieldFactory.create(name, value); + } + return emptySet(); + } + + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java new file mode 100644 index 0000000000..9c11baa34c --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java @@ -0,0 +1,35 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +final class FieldNames { + private FieldNames(){} + + static final String UID = "_uid"; + static final String ID = "_id"; + static final String TYPE = "_type"; + static final String REPOSITORY = "_repository"; + static final String PERMISSION = "_permission"; +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java b/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java new file mode 100644 index 0000000000..b24ac37432 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java @@ -0,0 +1,65 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import sonia.scm.SCMContextProvider; + +import javax.inject.Inject; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class IndexOpener { + + private final Path directory; + private final AnalyzerFactory analyzerFactory; + + @Inject + public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) { + directory = context.resolve(Paths.get("index")); + this.analyzerFactory = analyzerFactory; + } + + public IndexReader openForRead(String name) throws IOException { + return DirectoryReader.open(directory(name)); + } + + public IndexWriter openForWrite(String name, IndexOptions options) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options)); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory(name), config); + } + + private Directory directory(String name) throws IOException { + return FSDirectory.open(directory.resolve(name)); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java new file mode 100644 index 0000000000..2ddb227375 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +@FunctionalInterface +public interface IndexQueueTask { + + void updateIndex(Index index); + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java new file mode 100644 index 0000000000..bd09c03780 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class IndexQueueTaskWrapper implements Runnable { + + private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class); + + private final SearchEngine searchEngine; + private final String indexName; + private final IndexOptions options; + private final Iterable tasks; + + IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable tasks) { + this.searchEngine = searchEngine; + this.indexName = indexName; + this.options = options; + this.tasks = tasks; + } + + @Override + public void run() { + try (Index index = searchEngine.getOrCreate(this.indexName, options)) { + for (IndexQueueTask task : tasks) { + task.updateIndex(index); + } + } catch (Exception e) { + LOG.warn("failure during execution of index task for index {}", indexName, e); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java b/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java new file mode 100644 index 0000000000..253361e981 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.IndexableField; + +@FunctionalInterface +interface IndexableFieldFactory { + Iterable create(String name, Object value); +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java b/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java new file mode 100644 index 0000000000..cd662edb35 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java @@ -0,0 +1,172 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; +import java.text.DecimalFormat; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.util.Collections.singleton; +import static sonia.scm.search.TypeCheck.isBoolean; +import static sonia.scm.search.TypeCheck.isInstant; +import static sonia.scm.search.TypeCheck.isInteger; +import static sonia.scm.search.TypeCheck.isLong; + +class IndexableFields { + private IndexableFields() { + } + + static PointsConfig pointConfig(Field field) { + Class type = field.getType(); + if (isLong(type) || isInstant(type)) { + return new PointsConfig(new DecimalFormat(), Long.class); + } else if (isInteger(type)) { + return new PointsConfig(new DecimalFormat(), Integer.class); + } + return null; + } + + static IndexableFieldFactory create(Field field, Indexed indexed) { + Class fieldType = field.getType(); + Indexed.Type indexType = indexed.type(); + if (fieldType == String.class) { + return new StringFieldFactory(indexType); + } else if (isLong(fieldType)) { + return new LongFieldFactory(indexType); + } else if (isInteger(fieldType)) { + return new IntegerFieldFactory(indexType); + } else if (isBoolean(fieldType)) { + return new BooleanFieldFactory(indexType); + } else if (isInstant(fieldType)) { + return new InstantFieldFactory(indexType); + } else { + throw new UnsupportedTypeOfFieldException(fieldType, field.getName()); + } + } + + private static class StringFieldFactory implements IndexableFieldFactory { + private final Indexed.Type type; + + private StringFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + String stringValue = (String) value; + if (type.isTokenized()) { + return singleton(new TextField(name, stringValue, Store.YES)); + } else if (type.isSearchable()) { + return singleton(new StringField(name, stringValue, Store.YES)); + } else { + return singleton(new StoredField(name, stringValue)); + } + } + } + + private static class LongFieldFactory implements IndexableFieldFactory { + + private final Indexed.Type type; + + private LongFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Long longValue = (Long) value; + List fields = new ArrayList<>(); + if (type.isSearchable()) { + fields.add(new LongPoint(name, longValue)); + } + fields.add(new StoredField(name, longValue)); + return Collections.unmodifiableList(fields); + } + } + + private static class IntegerFieldFactory implements IndexableFieldFactory { + + private final Indexed.Type type; + + private IntegerFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Integer integerValue = (Integer) value; + List fields = new ArrayList<>(); + if (type.isSearchable()) { + fields.add(new IntPoint(name, integerValue)); + } + fields.add(new StoredField(name, integerValue)); + return Collections.unmodifiableList(fields); + } + } + + private static class BooleanFieldFactory implements IndexableFieldFactory { + private final Indexed.Type type; + + private BooleanFieldFactory(Indexed.Type type) { + this.type = type; + } + + @Override + public Iterable create(String name, Object value) { + Boolean booleanValue = (Boolean) value; + if (type.isSearchable()) { + return singleton(new StringField(name, booleanValue.toString(), Store.YES)); + } else { + return singleton(new StoredField(name, booleanValue.toString())); + } + } + } + + private static class InstantFieldFactory extends LongFieldFactory { + + private InstantFieldFactory(Indexed.Type type) { + super(type); + } + + @Override + public Iterable create(String name, Object value) { + Instant instant = (Instant) value; + return super.create(name, instant.toEpochMilli()); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java new file mode 100644 index 0000000000..91d10b87e6 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java @@ -0,0 +1,109 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; + +import java.io.IOException; + +import static sonia.scm.search.FieldNames.*; + +public class LuceneIndex implements Index { + + private final DocumentConverter converter; + private final IndexWriter writer; + + LuceneIndex(DocumentConverter converter, IndexWriter writer) { + this.converter = converter; + this.writer = writer; + } + + @Override + public void store(Id id, String permission, Object object) { + String uid = createUid(id, object.getClass()); + Document document = converter.convert(object); + try { + field(document, UID, uid); + field(document, ID, id.getValue()); + id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository)); + field(document, TYPE, object.getClass().getName()); + if (!Strings.isNullOrEmpty(permission)) { + field(document, PERMISSION, permission); + } + writer.updateDocument(new Term(UID, uid), document); + } catch (IOException e) { + throw new SearchEngineException("failed to add document to index", e); + } + } + + private String createUid(Id id, Class type) { + return id.asString() + "/" + type.getName(); + } + + private void field(Document document, String type, String name) { + document.add(new StringField(type, name, Field.Store.YES)); + } + + @Override + public void delete(Id id, Class type) { + try { + writer.deleteDocuments(new Term(UID, createUid(id, type))); + } catch (IOException e) { + throw new SearchEngineException("failed to delete document from index", e); + } + } + + @Override + public void deleteByRepository(String repository) { + try { + writer.deleteDocuments(new Term(REPOSITORY, repository)); + } catch (IOException ex) { + throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex); + } + } + + @Override + public void deleteByType(Class type) { + try { + writer.deleteDocuments(new Term(TYPE, type.getName())); + } catch (IOException ex) { + throw new SearchEngineException("failed to delete documents by repository " + type + " from index", ex); + } + } + + @Override + public void close() { + try { + writer.close(); + } catch (IOException e) { + throw new SearchEngineException("failed to close index writer", e); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java new file mode 100644 index 0000000000..3c12c61363 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java @@ -0,0 +1,149 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopScoreDocCollector; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class LuceneQueryBuilder extends QueryBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class); + + private static final Map, SearchableType> CACHE = new ConcurrentHashMap<>(); + + private final IndexOpener opener; + private final String indexName; + private final Analyzer analyzer; + + LuceneQueryBuilder(IndexOpener opener, String indexName, Analyzer analyzer) { + this.opener = opener; + this.indexName = indexName; + this.analyzer = analyzer; + } + + @Override + protected QueryResult execute(QueryParams queryParams) { + String queryString = Strings.nullToEmpty(queryParams.getQueryString()); + + SearchableType searchableType = CACHE.computeIfAbsent(queryParams.getType(), SearchableTypes::create); + + Query query = Queries.filter(createQuery(searchableType, queryParams, queryString), queryParams); + if (LOG.isDebugEnabled()) { + LOG.debug("execute lucene query: {}", query); + } + try (IndexReader reader = opener.openForRead(indexName)) { + IndexSearcher searcher = new IndexSearcher(reader); + + TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams); + Collector collector = new PermissionAwareCollector(reader, topScoreCollector); + searcher.search(query, collector); + + QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query); + return resultFactory.create(getTopDocs(queryParams, topScoreCollector)); + } catch (IOException e) { + throw new SearchEngineException("failed to search index", e); + } catch (InvalidTokenOffsetsException e) { + throw new SearchEngineException("failed to highlight results", e); + } + } + + @Nonnull + private TopScoreDocCollector createTopScoreCollector(QueryParams queryParams) { + return TopScoreDocCollector.create(queryParams.getStart() + queryParams.getLimit(), Integer.MAX_VALUE); + } + + private TopDocs getTopDocs(QueryParams queryParams, TopScoreDocCollector topScoreCollector) { + return topScoreCollector.topDocs(queryParams.getStart(), queryParams.getLimit()); + } + + private Query createQuery(SearchableType searchableType, QueryParams queryParams, String queryString) { + try { + if (queryString.contains(":")) { + return createExpertQuery(searchableType, queryParams); + } + return createBestGuessQuery(searchableType, queryParams); + } catch (QueryNodeException ex) { + throw new QueryParseException(queryString, "failed to parse query", ex); + } + } + + private Query createExpertQuery(SearchableType searchableType, QueryParams queryParams) throws QueryNodeException { + StandardQueryParser parser = new StandardQueryParser(analyzer); + + parser.setPointsConfigMap(searchableType.getPointsConfig()); + return parser.parse(queryParams.getQueryString(), ""); + } + + public Query createBestGuessQuery(SearchableType searchableType, QueryBuilder.QueryParams queryParams) { + String[] fieldNames = searchableType.getFieldNames(); + if (fieldNames == null || fieldNames.length == 0) { + throw new NoDefaultQueryFieldsFoundException(searchableType.getType()); + } + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (String fieldName : fieldNames) { + Term term = new Term(fieldName, appendWildcardIfNotAlreadyUsed(queryParams)); + WildcardQuery query = new WildcardQuery(term); + + Float boost = searchableType.getBoosts().get(fieldName); + if (boost != null) { + builder.add(new BoostQuery(query, boost), BooleanClause.Occur.SHOULD); + } else { + builder.add(query, BooleanClause.Occur.SHOULD); + } + } + return builder.build(); + } + + @Nonnull + private String appendWildcardIfNotAlreadyUsed(QueryParams queryParams) { + String queryString = queryParams.getQueryString().toLowerCase(Locale.ENGLISH); + if (!queryString.contains("?") && !queryString.contains("*")) { + queryString += "*"; + } + return queryString; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java new file mode 100644 index 0000000000..6b86c5c528 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import javax.inject.Inject; + +public class LuceneQueryBuilderFactory { + + private final IndexOpener indexOpener; + private final AnalyzerFactory analyzerFactory; + + @Inject + public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) { + this.indexOpener = indexOpener; + this.analyzerFactory = analyzerFactory; + } + + public LuceneQueryBuilder create(String name, IndexOptions options) { + return new LuceneQueryBuilder(indexOpener, name, analyzerFactory.create(options)); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java new file mode 100644 index 0000000000..06edf78b5b --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java @@ -0,0 +1,57 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import javax.inject.Inject; +import java.io.IOException; + +public class LuceneSearchEngine implements SearchEngine { + + private final IndexOpener indexOpener; + private final DocumentConverter converter; + private final LuceneQueryBuilderFactory queryBuilderFactory; + + @Inject + public LuceneSearchEngine(IndexOpener indexOpener, DocumentConverter converter, LuceneQueryBuilderFactory queryBuilderFactory) { + this.indexOpener = indexOpener; + this.converter = converter; + this.queryBuilderFactory = queryBuilderFactory; + } + + @Override + public Index getOrCreate(String name, IndexOptions options) { + try { + return new LuceneIndex(converter, indexOpener.openForWrite(name, options)); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index", ex); + } + } + + @Override + public QueryBuilder search(String name, IndexOptions options) { + return queryBuilderFactory.create(name, options); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java b/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java new file mode 100644 index 0000000000..c15d8f6cd7 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java @@ -0,0 +1,31 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +public class NonReadableFieldException extends SearchEngineException { + public NonReadableFieldException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java b/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java new file mode 100644 index 0000000000..4fa3ce502f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.shiro.SecurityUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; + +public class PermissionAwareCollector implements Collector { + + private static final String FIELD_PERMISSION = "_permission"; + private static final Set FIELDS = Collections.singleton(FIELD_PERMISSION); + + private final IndexReader reader; + private final Collector delegate; + + public PermissionAwareCollector(IndexReader reader, Collector delegate) { + this.reader = reader; + this.delegate = delegate; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + return new PermissionAwareLeafCollector(delegate.getLeafCollector(context), context.docBase); + } + + @Override + public ScoreMode scoreMode() { + return delegate.scoreMode(); + } + + private class PermissionAwareLeafCollector implements LeafCollector { + + private final LeafCollector delegate; + private final int docBase; + + private PermissionAwareLeafCollector(LeafCollector delegate, int docBase) { + this.delegate = delegate; + this.docBase = docBase; + } + + @Override + public void setScorer(Scorable scorer) throws IOException { + this.delegate.setScorer(scorer); + } + + @Override + public void collect(int doc) throws IOException { + Document document = reader.document(docBase + doc, FIELDS); + String permission = document.get(FIELD_PERMISSION); + if (Strings.isNullOrEmpty(permission) || SecurityUtils.getSubject().isPermitted(permission)) { + this.delegate.collect(doc); + } + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/Queries.java b/scm-webapp/src/main/java/sonia/scm/search/Queries.java new file mode 100644 index 0000000000..0b69d20194 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/Queries.java @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + +import static org.apache.lucene.search.BooleanClause.Occur.MUST; + +final class Queries { + + private Queries() { + } + + private static Query typeQuery(Class type) { + return new TermQuery(new Term(FieldNames.TYPE, type.getName())); + } + + private static Query repositoryQuery(String repositoryId) { + return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId)); + } + + static Query filter(Query query, QueryBuilder.QueryParams params) { + BooleanQuery.Builder builder = new BooleanQuery.Builder() + .add(query, MUST) + .add(typeQuery(params.getType()), MUST); + params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST)); + return builder.build(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java b/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java new file mode 100644 index 0000000000..7ddfbc3b71 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java @@ -0,0 +1,103 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.highlight.Highlighter; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.apache.lucene.search.highlight.QueryScorer; +import org.apache.lucene.search.highlight.SimpleHTMLFormatter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.util.Optional.empty; +import static java.util.Optional.of; + +public class QueryResultFactory { + + private final Analyzer analyzer; + private final Highlighter highlighter; + private final IndexSearcher searcher; + private final SearchableType searchableType; + + public QueryResultFactory(Analyzer analyzer, IndexSearcher searcher, SearchableType searchableType, Query query) { + this.analyzer = analyzer; + this.searcher = searcher; + this.searchableType = searchableType; + this.highlighter = createHighlighter(query); + } + + private Highlighter createHighlighter(Query query) { + return new Highlighter( + new SimpleHTMLFormatter("**", "**"), + new QueryScorer(query) + ); + } + + public QueryResult create(TopDocs topDocs) throws IOException, InvalidTokenOffsetsException { + List hits = new ArrayList<>(); + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + hits.add(createHit(scoreDoc)); + } + return new QueryResult(topDocs.totalHits.value, searchableType.getType(), hits); + } + + private Hit createHit(ScoreDoc scoreDoc) throws IOException, InvalidTokenOffsetsException { + Document document = searcher.doc(scoreDoc.doc); + Map fields = new HashMap<>(); + for (SearchableField field : searchableType.getFields()) { + field(document, field).ifPresent(f -> fields.put(field.getName(), f)); + } + return new Hit(document.get(FieldNames.ID), scoreDoc.score, fields); + } + private Optional field(Document document, SearchableField field) throws IOException, InvalidTokenOffsetsException { + Object value = field.value(document); + if (value != null) { + if (field.isHighlighted()) { + String[] fragments = createFragments(field, value.toString()); + if (fragments.length > 0) { + return of(new Hit.HighlightedField(fragments)); + } + } + return of(new Hit.ValueField(value)); + } + return empty(); + } + + private String[] createFragments(SearchableField field, String value) throws InvalidTokenOffsetsException, IOException { + return highlighter.getBestFragments(analyzer, field.getName(), value, 5); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java new file mode 100644 index 0000000000..2f0ce2239f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java @@ -0,0 +1,71 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import java.util.ArrayList; +import java.util.List; + +public class QueuedIndex implements Index { + + private final DefaultIndexQueue queue; + private final String indexName; + private final IndexOptions indexOptions; + + private final List tasks = new ArrayList<>(); + + QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) { + this.queue = queue; + this.indexName = indexName; + this.indexOptions = indexOptions; + } + + @Override + public void store(Id id, String permission, Object object) { + tasks.add(index -> index.store(id, permission, object)); + } + + @Override + public void delete(Id id, Class type) { + tasks.add(index -> index.delete(id, type)); + } + + @Override + public void deleteByRepository(String repository) { + tasks.add(index -> index.deleteByRepository(repository)); + } + + @Override + public void deleteByType(Class type) { + tasks.add(index -> index.deleteByType(type)); + } + + @Override + public void close() { + IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper( + queue.getSearchEngine(), indexName, indexOptions, tasks + ); + queue.enqueue(wrappedTask); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java new file mode 100644 index 0000000000..f96cf38972 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableField.java @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.base.Strings; +import lombok.Getter; +import org.apache.lucene.document.Document; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; + +@Getter +class SearchableField { + + private final String name; + private final Class type; + private final ValueExtractor valueExtractor; + private final float boost; + private final boolean defaultQuery; + private final boolean highlighted; + private final PointsConfig pointsConfig; + + SearchableField(Field field, Indexed indexed) { + this.name = name(field, indexed); + this.type = field.getType(); + this.valueExtractor = ValueExtractors.create(name, type); + this.boost = indexed.boost(); + this.defaultQuery = indexed.defaultQuery(); + this.highlighted = indexed.highlighted(); + this.pointsConfig = IndexableFields.pointConfig(field); + } + + Object value(Document document) { + return valueExtractor.extract(document); + } + + private String name(Field field, Indexed indexed) { + String nameFromAnnotation = indexed.name(); + if (!Strings.isNullOrEmpty(nameFromAnnotation)) { + return nameFromAnnotation; + } + return field.getName(); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java new file mode 100644 index 0000000000..14bdc6dce2 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableType.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Value; +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +@Value +public class SearchableType { + + Class type; + String[] fieldNames; + Map boosts; + Map pointsConfig; + List fields; + + SearchableType(Class type, String[] fieldNames, Map boosts, Map pointsConfig, List fields) { + this.type = type; + this.fieldNames = fieldNames; + this.boosts = Collections.unmodifiableMap(boosts); + this.pointsConfig = Collections.unmodifiableMap(pointsConfig); + this.fields = Collections.unmodifiableList(fields); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java b/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java new file mode 100644 index 0000000000..dc7bbac77f --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java @@ -0,0 +1,82 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +final class SearchableTypes { + + private static final float DEFAULT_BOOST = 1f; + + private SearchableTypes() { + } + + static SearchableType create(Class type) { + List fields = new ArrayList<>(); + collectFields(type, fields); + return createSearchableType(type, fields); + } + + private static SearchableType createSearchableType(Class type, List fields) { + String[] fieldsNames = fields.stream() + .filter(SearchableField::isDefaultQuery) + .map(SearchableField::getName) + .toArray(String[]::new); + + Map boosts = new HashMap<>(); + Map pointsConfig = new HashMap<>(); + for (SearchableField field : fields) { + if (field.isDefaultQuery() && field.getBoost() != DEFAULT_BOOST) { + boosts.put(field.getName(), field.getBoost()); + } + PointsConfig config = field.getPointsConfig(); + if (config != null) { + pointsConfig.put(field.getName(), config); + } + } + + return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields); + } + + private static void collectFields(Class type, List fields) { + Class parent = type.getSuperclass(); + if (parent != null) { + collectFields(parent, fields); + } + for (Field field : type.getDeclaredFields()) { + Indexed indexed = field.getAnnotation(Indexed.class); + if (indexed != null) { + fields.add(new SearchableField(field, indexed)); + } + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java b/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java new file mode 100644 index 0000000000..79bd68bb68 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java @@ -0,0 +1,53 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import java.time.Instant; + +final class TypeCheck { + + private TypeCheck() { + } + + public static boolean isLong(Class type) { + return type == Long.TYPE || type == Long.class; + } + + public static boolean isInteger(Class type) { + return type == Integer.TYPE || type == Integer.class; + } + + public static boolean isBoolean(Class type) { + return type == Boolean.TYPE || type == Boolean.class; + } + + public static boolean isInstant(Class type) { + return type == Instant.class; + } + + public static boolean isString(Class type) { + return type == String.class; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java b/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java new file mode 100644 index 0000000000..bf7acfdbca --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java @@ -0,0 +1,31 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +public class UnsupportedTypeOfFieldException extends SearchEngineException { + public UnsupportedTypeOfFieldException(Class type, String field) { + super("type " + type + " of " + field + " is unsupported."); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java new file mode 100644 index 0000000000..b317c080df --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Document; + +@FunctionalInterface +public interface ValueExtractor { + Object extract(Document document); +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java new file mode 100644 index 0000000000..d002aac246 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java @@ -0,0 +1,95 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.index.IndexableField; + +import javax.annotation.Nonnull; +import java.time.Instant; + +final class ValueExtractors { + + private ValueExtractors() { + } + + static ValueExtractor create(String name, Class type) { + if (TypeCheck.isString(type)) { + return stringExtractor(name); + } else if (TypeCheck.isLong(type)) { + return longExtractor(name); + } else if (TypeCheck.isInteger(type)) { + return integerExtractor(name); + } else if (TypeCheck.isBoolean(type)) { + return booleanExtractor(name); + } else if (TypeCheck.isInstant(type)) { + return instantExtractor(name); + } else { + throw new UnsupportedTypeOfFieldException(type, name); + } + } + + @Nonnull + private static ValueExtractor stringExtractor(String name) { + return doc -> doc.get(name); + } + + @Nonnull + private static ValueExtractor instantExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return Instant.ofEpochMilli(field.numericValue().longValue()); + } + return null; + }; + } + + @Nonnull + private static ValueExtractor booleanExtractor(String name) { + return doc -> Boolean.parseBoolean(doc.get(name)); + } + + @Nonnull + private static ValueExtractor integerExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return field.numericValue().intValue(); + } + return null; + }; + } + + @Nonnull + private static ValueExtractor longExtractor(String name) { + return doc -> { + IndexableField field = doc.getField(name); + if (field != null) { + return field.numericValue().longValue(); + } + return null; + }; + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java index c9fff0666e..ffa4ac8418 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/IndexDtoGeneratorTest.java @@ -195,5 +195,6 @@ class IndexDtoGeneratorTest { when(resourceLinks.namespaceCollection()).thenReturn(new ResourceLinks.NamespaceCollectionLinks(scmPathInfo)); when(resourceLinks.me()).thenReturn(new ResourceLinks.MeLinks(scmPathInfo, new ResourceLinks.UserLinks(scmPathInfo))); when(resourceLinks.repository()).thenReturn(new ResourceLinks.RepositoryLinks(scmPathInfo)); + when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(scmPathInfo)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java index 2bec6f3166..83c12ce222 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/ResourceLinksMock.java @@ -82,6 +82,7 @@ public class ResourceLinksMock { lenient().when(resourceLinks.adminInfo()).thenReturn(new ResourceLinks.AdminInfoLinks(pathInfo)); lenient().when(resourceLinks.apiKeyCollection()).thenReturn(new ResourceLinks.ApiKeyCollectionLinks(pathInfo)); lenient().when(resourceLinks.apiKey()).thenReturn(new ResourceLinks.ApiKeyLinks(pathInfo)); + lenient().when(resourceLinks.search()).thenReturn(new ResourceLinks.SearchLinks(pathInfo)); return resourceLinks; } diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java new file mode 100644 index 0000000000..ac0139bb1f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/SearchResourceTest.java @@ -0,0 +1,266 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.api.v2.resources; + +import com.fasterxml.jackson.databind.JsonNode; +import de.otto.edison.hal.HalRepresentation; +import lombok.Getter; +import lombok.Setter; +import org.jboss.resteasy.mock.MockHttpRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mapstruct.factory.Mappers; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.search.Hit; +import sonia.scm.search.IndexNames; +import sonia.scm.search.QueryResult; +import sonia.scm.search.SearchEngine; +import sonia.scm.web.JsonMockHttpResponse; +import sonia.scm.web.RestDispatcher; +import sonia.scm.web.VndMediaType; + +import javax.annotation.Nonnull; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SearchResourceTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; + + private RestDispatcher dispatcher; + + @Mock + private HalEnricherRegistry enricherRegistry; + + @BeforeEach + void setUpDispatcher() { + QueryResultMapper mapper = Mappers.getMapper(QueryResultMapper.class); + mapper.setRegistry(enricherRegistry); + SearchResource resource = new SearchResource( + searchEngine, mapper + ); + dispatcher = new RestDispatcher(); + dispatcher.addSingletonResource(resource); + } + + @Test + void shouldEnrichQueryResult() throws IOException, URISyntaxException { + when(enricherRegistry.allByType(QueryResult.class)) + .thenReturn(Collections.singleton(new SampleEnricher())); + + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + + JsonNode sample = response.getContentAsJson().get("_embedded").get("sample"); + assertThat(sample.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Test + void shouldEnrichHitResult() throws IOException, URISyntaxException { + when(enricherRegistry.allByType(QueryResult.class)) + .thenReturn(Collections.emptySet()); + when(enricherRegistry.allByType(Hit.class)) + .thenReturn(Collections.singleton(new SampleEnricher())); + + mockQueryResult("Hello", result(1L, "Hello")); + JsonMockHttpResponse response = search("Hello"); + + JsonNode sample = response.getContentAsJson() + .get("_embedded").get("hits").get(0) + .get("_embedded").get("sample"); + assertThat(sample.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Nested + class WithoutEnricher { + + @BeforeEach + void setUpEnricherRegistry() { + when(enricherRegistry.allByType(QueryResult.class)).thenReturn(Collections.emptySet()); + lenient().when(enricherRegistry.allByType(Hit.class)).thenReturn(Collections.emptySet()); + } + + @Test + void shouldReturnVndContentType() throws UnsupportedEncodingException, URISyntaxException { + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + assertThat(response.getStatus()).isEqualTo(HttpServletResponse.SC_OK); + assertHeader(response, "Content-Type", VndMediaType.QUERY_RESULT); + } + + @Test + void shouldReturnPagingLinks() throws IOException, URISyntaxException { + mockQueryResult(20, 20, "paging", result(100)); + JsonMockHttpResponse response = search("paging", 1, 20); + + JsonNode links = response.getContentAsJson().get("_links"); + assertLink(links, "self", "/v2/search?q=paging&page=1&pageSize=20"); + assertLink(links, "first", "/v2/search?q=paging&page=0&pageSize=20"); + assertLink(links, "prev", "/v2/search?q=paging&page=0&pageSize=20"); + assertLink(links, "next", "/v2/search?q=paging&page=2&pageSize=20"); + assertLink(links, "last", "/v2/search?q=paging&page=4&pageSize=20"); + } + + @Test + void shouldPagingFields() throws IOException, URISyntaxException { + mockQueryResult(20, 20, "pagingFields", result(100)); + JsonMockHttpResponse response = search("pagingFields", 1, 20); + + JsonNode root = response.getContentAsJson(); + assertThat(root.get("page").asInt()).isOne(); + assertThat(root.get("pageTotal").asInt()).isEqualTo(5); + } + + @Test + void shouldReturnType() throws IOException, URISyntaxException { + mockQueryResult("Hello", result(0L)); + JsonMockHttpResponse response = search("Hello"); + + JsonNode root = response.getContentAsJson(); + assertThat(root.get("type").asText()).isEqualTo("java.lang.String"); + } + + @Test + void shouldReturnHitsAsEmbedded() throws IOException, URISyntaxException { + mockQueryResult("Hello", result(2L, "Hello", "Hello Again")); + JsonMockHttpResponse response = search("Hello"); + + JsonNode hits = response.getContentAsJson().get("_embedded").get("hits"); + assertThat(hits.size()).isEqualTo(2); + + JsonNode first = hits.get(0); + assertThat(first.get("score").asDouble()).isEqualTo(2d); + + JsonNode fields = first.get("fields"); + + JsonNode valueField = fields.get("value"); + assertThat(valueField.get("highlighted").asBoolean()).isFalse(); + assertThat(valueField.get("value").asText()).isEqualTo("Hello"); + + JsonNode highlightedField = fields.get("highlighted"); + assertThat(highlightedField.get("highlighted").asBoolean()).isTrue(); + assertThat(highlightedField.get("fragments").get(0).asText()).isEqualTo("Hello"); + } + + } + + private void assertLink(JsonNode links, String self, String s) { + assertThat(links.get(self).get("href").asText()).isEqualTo(s); + } + + private QueryResult result(long totalHits, Object... values) { + List hits = new ArrayList<>(); + for (int i = 0; i < values.length; i++) { + hits.add(hit(i, values)); + } + return new QueryResult(totalHits, String.class, hits); + } + + @Nonnull + private Hit hit(int i, Object[] values) { + Map fields = fields(values[i]); + return new Hit("" + i, values.length - i, fields); + } + + @Nonnull + private Map fields(Object value) { + Map fields = new HashMap<>(); + fields.put("value", new Hit.ValueField(value)); + fields.put("highlighted", new Hit.HighlightedField(new String[]{value.toString()})); + return fields; + } + + private void mockQueryResult(String query, QueryResult result) { + mockQueryResult(0, 10, query, result); + } + + private void mockQueryResult(int start, int limit, String query, QueryResult result) { + when( + searchEngine.search(IndexNames.DEFAULT) + .start(start) + .limit(limit) + .execute(Repository.class, query) + ).thenReturn(result); + } + + private void assertHeader(JsonMockHttpResponse response, String header, String expectedValue) { + assertThat(response.getOutputHeaders().getFirst(header)).hasToString(expectedValue); + } + + private JsonMockHttpResponse search(String query) throws URISyntaxException, UnsupportedEncodingException { + return search(query, null, null); + } + + private JsonMockHttpResponse search(String query, Integer page, Integer pageSize) throws URISyntaxException, UnsupportedEncodingException { + String uri = "/v2/search?q=" + URLEncoder.encode(query, "UTF-8"); + if (page != null) { + uri += "&page=" + page; + } + if (pageSize != null) { + uri += "&pageSize=" + pageSize; + } + MockHttpRequest request = MockHttpRequest.get(uri); + JsonMockHttpResponse response = new JsonMockHttpResponse(); + dispatcher.invoke(request, response); + return response; + } + + @Getter + @Setter + public static class SampleEmbedded extends HalRepresentation { + private Class type; + } + + private static class SampleEnricher implements HalEnricher { + @Override + public void enrich(HalEnricherContext context, HalAppender appender) { + QueryResult result = context.oneRequireByType(QueryResult.class); + + SampleEmbedded embedded = new SampleEmbedded(); + embedded.setType(result.getType()); + + appender.appendEmbedded("sample", embedded); + } + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java b/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java new file mode 100644 index 0000000000..13a2bb02b7 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/repository/IndexUpdateListenerTest.java @@ -0,0 +1,147 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.repository; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.HandlerEventType; +import sonia.scm.search.Id; +import sonia.scm.search.Index; +import sonia.scm.search.IndexLog; +import sonia.scm.search.IndexLogStore; +import sonia.scm.search.IndexNames; +import sonia.scm.search.IndexQueue; +import sonia.scm.web.security.AdministrationContext; + +import java.util.List; +import java.util.Optional; + +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexUpdateListenerTest { + + @Mock + private RepositoryManager repositoryManager; + + @Mock + private AdministrationContext administrationContext; + + @Mock + private IndexQueue indexQueue; + + @Mock + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @InjectMocks + private IndexUpdateListener updateListener; + + @Test + @SuppressWarnings("java:S6068") + void shouldIndexAllRepositories() { + when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.empty()); + doAnswer(ic -> { + IndexUpdateListener.ReIndexAll reIndexAll = new IndexUpdateListener.ReIndexAll(repositoryManager, indexQueue); + reIndexAll.run(); + return null; + }) + .when(administrationContext) + .runAsAdmin(IndexUpdateListener.ReIndexAll.class); + + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle42 = RepositoryTestData.create42Puzzle(); + List repositories = ImmutableList.of(heartOfGold, puzzle42); + + when(repositoryManager.getAll()).thenReturn(repositories); + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + + updateListener.contextInitialized(null); + + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + verify(index).store(Id.of(puzzle42), RepositoryPermissions.read(puzzle42).asShiroString(), puzzle42); + verify(index).close(); + + verify(indexLogStore).log(IndexNames.DEFAULT, Repository.class, IndexUpdateListener.INDEX_VERSION); + } + + @Test + void shouldSkipReIndex() { + IndexLog log = new IndexLog(1); + when(indexLogStore.get(IndexNames.DEFAULT, Repository.class)).thenReturn(Optional.of(log)); + + updateListener.contextInitialized(null); + + verifyNoInteractions(indexQueue); + } + + @ParameterizedTest + @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.MATCH_ANY, names = "BEFORE_.*") + void shouldIgnoreBeforeEvents(HandlerEventType type) { + RepositoryEvent event = new RepositoryEvent(type, RepositoryTestData.create42Puzzle()); + + updateListener.handleEvent(event); + + verifyNoInteractions(indexQueue); + } + + @ParameterizedTest + @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.INCLUDE, names = {"CREATE", "MODIFY"}) + void shouldStore(HandlerEventType type) { + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + + Repository puzzle = RepositoryTestData.create42Puzzle(); + RepositoryEvent event = new RepositoryEvent(type, puzzle); + + updateListener.handleEvent(event); + + verify(index).store(Id.of(puzzle), RepositoryPermissions.read(puzzle).asShiroString(), puzzle); + verify(index).close(); + } + + @Test + void shouldDelete() { + when(indexQueue.getQueuedIndex(IndexNames.DEFAULT)).thenReturn(index); + Repository puzzle = RepositoryTestData.create42Puzzle(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, puzzle); + + updateListener.handleEvent(event); + + verify(index).deleteByRepository(puzzle.getId()); + verify(index).close(); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java b/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java new file mode 100644 index 0000000000..be83b7450a --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/AnalyzerFactoryTest.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.de.GermanAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.es.SpanishAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.junit.jupiter.api.Test; + +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; + +class AnalyzerFactoryTest { + + private final AnalyzerFactory analyzerFactory = new AnalyzerFactory(); + + @Test + void shouldReturnStandardAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.defaults()); + assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); + } + + @Test + void shouldReturnStandardAnalyzerForUnknownLocale() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.CHINESE)); + assertThat(analyzer).isInstanceOf(StandardAnalyzer.class); + } + + @Test + void shouldReturnEnglishAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.ENGLISH)); + assertThat(analyzer).isInstanceOf(EnglishAnalyzer.class); + } + + @Test + void shouldReturnGermanAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMAN)); + assertThat(analyzer).isInstanceOf(GermanAnalyzer.class); + } + + @Test + void shouldReturnGermanAnalyzerForLocaleGermany() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(Locale.GERMANY)); + assertThat(analyzer).isInstanceOf(GermanAnalyzer.class); + } + + @Test + void shouldReturnSpanishAnalyzer() { + Analyzer analyzer = analyzerFactory.create(IndexOptions.naturalLanguage(new Locale("es", "ES"))); + assertThat(analyzer).isInstanceOf(SpanishAnalyzer.class); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java new file mode 100644 index 0000000000..b6876e19ca --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexLogStoreTest.java @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import sonia.scm.store.InMemoryByteDataStoreFactory; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +class DefaultIndexLogStoreTest { + + private IndexLogStore indexLogStore; + + @BeforeEach + void setUpIndexLogStore() { + InMemoryByteDataStoreFactory dataStoreFactory = new InMemoryByteDataStoreFactory(); + indexLogStore = new DefaultIndexLogStore(dataStoreFactory); + } + + @Test + void shouldReturnEmptyOptional() { + Optional indexLog = indexLogStore.get("index", String.class); + assertThat(indexLog).isEmpty(); + } + + @Test + void shouldStoreLog() { + indexLogStore.log("index", String.class, 42); + Optional index = indexLogStore.get("index", String.class); + assertThat(index).hasValueSatisfying(log -> { + assertThat(log.getVersion()).isEqualTo(42); + assertThat(log.getDate()).isNotNull(); + }); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java new file mode 100644 index 0000000000..69683ab4b4 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DefaultIndexQueueTest.java @@ -0,0 +1,145 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Value; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DefaultIndexQueueTest { + + private Directory directory; + + private DefaultIndexQueue queue; + + @Mock + private LuceneQueryBuilderFactory queryBuilderFactory; + + @BeforeEach + void createQueue() throws IOException { + directory = new ByteBuffersDirectory(); + IndexOpener factory = mock(IndexOpener.class); + when(factory.openForWrite(any(String.class), any(IndexOptions.class))).thenAnswer(ic -> { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory, config); + }); + SearchEngine engine = new LuceneSearchEngine(factory, new DocumentConverter(), queryBuilderFactory); + queue = new DefaultIndexQueue(engine); + } + + @AfterEach + void closeQueue() throws IOException { + queue.close(); + directory.close(); + } + + @Test + void shouldWriteToIndex() throws Exception { + try (Index index = queue.getQueuedIndex("default")) { + index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan")); + index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent")); + } + assertDocCount(2); + } + + @Test + void shouldWriteMultiThreaded() throws Exception { + ExecutorService executorService = Executors.newFixedThreadPool(4); + for (int i = 0; i < 20; i++) { + executorService.execute(new IndexNumberTask(i)); + } + executorService.execute(() -> { + try (Index index = queue.getQueuedIndex("default")) { + index.delete(Id.of(String.valueOf(12)), IndexedNumber.class); + } + }); + executorService.shutdown(); + + assertDocCount(19); + } + + private void assertDocCount(int expectedCount) throws IOException { + // wait until all tasks are finished + await().until(() -> queue.getSize() == 0); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertThat(reader.numDocs()).isEqualTo(expectedCount); + } + } + + @Value + public static class Account { + @Indexed + String username; + @Indexed + String firstName; + @Indexed + String lastName; + } + + @Value + public static class IndexedNumber { + @Indexed + int value; + } + + public class IndexNumberTask implements Runnable { + + private final int number; + + public IndexNumberTask(int number) { + this.number = number; + } + + @Override + public void run() { + try (Index index = queue.getQueuedIndex("default")) { + index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number)); + } + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java b/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java new file mode 100644 index 0000000000..8980639faf --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/DocumentConverterTest.java @@ -0,0 +1,288 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class DocumentConverterTest { + + private DocumentConverter documentConverter; + + @BeforeEach + void prepare() { + documentConverter = new DocumentConverter(); + } + + @Test + void shouldConvertPersonToDocument() { + Person person = new Person("Arthur", "Dent"); + + Document document = documentConverter.convert(person); + + assertThat(document.getField("firstName").stringValue()).isEqualTo("Arthur"); + assertThat(document.getField("lastName").stringValue()).isEqualTo("Dent"); + } + + @Test + void shouldUseNameFromAnnotation() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("username").stringValue()).isEqualTo("dent"); + } + + @Test + void shouldBeIndexedAsTextFieldByDefault() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("username")).isInstanceOf(TextField.class); + } + + @Test + void shouldBeIndexedAsStringField() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("searchable")).isInstanceOf(StringField.class); + } + + @Test + void shouldBeIndexedAsStoredField() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("storedOnly")).isInstanceOf(StoredField.class); + } + + @Test + void shouldIgnoreNonIndexedFields() { + Document document = documentConverter.convert(new ParamSample()); + + assertThat(document.getField("notIndexed")).isNull(); + } + + @Test + void shouldSupportInheritance() { + Account account = new Account("Arthur", "Dent", "arthur@hitchhiker.com"); + + Document document = documentConverter.convert(account); + + assertThat(document.getField("firstName")).isNotNull(); + assertThat(document.getField("lastName")).isNotNull(); + assertThat(document.getField("mail")).isNotNull(); + } + + @Test + void shouldFailWithoutGetter() { + WithoutGetter withoutGetter = new WithoutGetter(); + assertThrows(NonReadableFieldException.class, () -> documentConverter.convert(withoutGetter)); + } + + @Test + void shouldFailOnUnsupportedFieldType() { + UnsupportedFieldType unsupportedFieldType = new UnsupportedFieldType(); + assertThrows(UnsupportedTypeOfFieldException.class, () -> documentConverter.convert(unsupportedFieldType)); + } + + @Test + void shouldStoreLongFieldsAsPointAndStoredByDefault() { + Document document = documentConverter.convert(new SupportedTypes()); + + assertPointField(document, "longType", + field -> assertThat(field.numericValue().longValue()).isEqualTo(42L) + ); + } + + @Test + void shouldStoreLongFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyLongType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().longValue()).isEqualTo(42L); + } + + @Test + void shouldStoreIntegerFieldsAsPointAndStoredByDefault() { + Document document = documentConverter.convert(new SupportedTypes()); + + assertPointField(document, "intType", + field -> assertThat(field.numericValue().intValue()).isEqualTo(42) + ); + } + + @Test + void shouldStoreIntegerFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyIntegerType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().intValue()).isEqualTo(42); + } + + @Test + void shouldStoreBooleanFieldsAsStringField() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("boolType"); + assertThat(field).isInstanceOf(StringField.class); + assertThat(field.stringValue()).isEqualTo("true"); + assertThat(field.fieldType().stored()).isTrue(); + } + + @Test + void shouldStoreBooleanFieldAsStored() { + Document document = documentConverter.convert(new SupportedTypes()); + + IndexableField field = document.getField("storedOnlyBoolType"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.stringValue()).isEqualTo("true"); + } + + @Test + void shouldStoreInstantFieldsAsPointAndStoredByDefault() { + Instant now = Instant.now(); + Document document = documentConverter.convert(new DateTypes(now)); + + assertPointField(document, "instant", + field -> assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli()) + ); + } + + @Test + void shouldStoreInstantFieldAsStored() { + Instant now = Instant.now(); + Document document = documentConverter.convert(new DateTypes(now)); + + IndexableField field = document.getField("storedOnlyInstant"); + assertThat(field).isInstanceOf(StoredField.class); + assertThat(field.numericValue().longValue()).isEqualTo(now.toEpochMilli()); + } + + @Test + void shouldCreateNoFieldForNullValues() { + Document document = documentConverter.convert(new Person("Trillian", null)); + + assertThat(document.getField("firstName")).isNotNull(); + assertThat(document.getField("lastName")).isNull(); + } + + private void assertPointField(Document document, String name, Consumer consumer) { + IndexableField[] fields = document.getFields(name); + assertThat(fields) + .allSatisfy(consumer) + .anySatisfy(field -> assertThat(field.fieldType().stored()).isFalse()) + .anySatisfy(field -> assertThat(field.fieldType().stored()).isTrue()); + } + + @Getter + @AllArgsConstructor + public static class Person { + @Indexed + private String firstName; + @Indexed + private String lastName; + } + + @Getter + public static class Account extends Person { + @Indexed + private String mail; + + public Account(String firstName, String lastName, String mail) { + super(firstName, lastName); + this.mail = mail; + } + } + + @Getter + public static class ParamSample { + @Indexed(name = "username") + private final String name = "dent"; + + @Indexed(type = Indexed.Type.SEARCHABLE) + private final String searchable = "--"; + + @Indexed(type = Indexed.Type.STORED_ONLY) + private final String storedOnly = "--"; + + private final String notIndexed = "--"; + } + + public static class WithoutGetter { + @Indexed + private final String value = "one"; + } + + @Getter + public static class UnsupportedFieldType { + @Indexed + private final Object value = "one"; + } + + @Getter + public static class SupportedTypes { + @Indexed + private final Long longType = 42L; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final long storedOnlyLongType = 42L; + + @Indexed + private final int intType = 42; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final Integer storedOnlyIntegerType = 42; + + @Indexed + private final boolean boolType = true; + @Indexed(type = Indexed.Type.STORED_ONLY) + private final boolean storedOnlyBoolType = true; + } + + @Getter + private static class DateTypes { + @Indexed + private final Instant instant; + + @Indexed(type = Indexed.Type.STORED_ONLY) + private final Instant storedOnlyInstant; + + private DateTypes(Instant instant) { + this.instant = instant; + this.storedOnlyInstant = instant; + } + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java new file mode 100644 index 0000000000..5f70b6bfab --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java @@ -0,0 +1,99 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.core.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexOpenerTest { + + private Path directory; + + @Mock + private AnalyzerFactory analyzerFactory; + + private IndexOpener indexOpener; + + @BeforeEach + void createIndexWriterFactory(@TempDir Path tempDirectory) { + this.directory = tempDirectory; + SCMContextProvider context = mock(SCMContextProvider.class); + when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory); + when(analyzerFactory.create(any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); + indexOpener = new IndexOpener(context, analyzerFactory); + } + + @Test + void shouldCreateNewIndex() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("new-index")).exists(); + } + + @Test + void shouldOpenExistingIndex() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { + addDoc(writer, "Dent"); + } + try (IndexWriter writer = indexOpener.openForWrite("reused", IndexOptions.defaults())) { + assertThat(writer.getFieldNames()).contains("hitchhiker"); + } + } + + @Test + void shouldUseAnalyzerFromFactory() throws IOException { + try (IndexWriter writer = indexOpener.openForWrite("new-index", IndexOptions.defaults())) { + assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); + } + } + + private void addDoc(IndexWriter writer, String name) throws IOException { + Document doc = new Document(); + doc.add(new TextField("hitchhiker", name, Field.Store.YES)); + writer.addDocument(doc); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java new file mode 100644 index 0000000000..58c79c35b5 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java @@ -0,0 +1,238 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import lombok.Value; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static sonia.scm.search.FieldNames.*; + +class LuceneIndexTest { + + private static final Id ONE = Id.of("one"); + + private Directory directory; + + @BeforeEach + void createDirectory() { + directory = new ByteBuffersDirectory(); + } + + @Test + void shouldStoreObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldUpdateObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + index.store(ONE, null, new Storable("Awesome content")); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldStoreUidOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Awesome content which should be indexed")); + } + + assertHits(UID, "one/" + Storable.class.getName(), 1); + } + + @Test + void shouldStoreIdOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some text")); + } + + assertHits(ID, "one", 1); + } + + @Test + void shouldStoreRepositoryOfId() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), null, new Storable("Some text")); + } + + assertHits(REPOSITORY, "4211", 1); + } + + @Test + void shouldStoreTypeOfObject() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + } + + assertHits(TYPE, Storable.class.getName(), 1); + } + + @Test + void shouldDeleteById() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(ONE, Storable.class); + } + + assertHits(ID, "one", 0); + } + + @Test + void shouldDeleteAllByType() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("content")); + index.store(Id.of("two"), null, new Storable("content")); + index.store(Id.of("three"), null, new OtherStorable("content")); + } + + try (LuceneIndex index = createIndex()) { + index.deleteByType(Storable.class); + } + + assertHits("value", "content", 1); + } + + @Test + void shouldDeleteByIdAnyType() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some text")); + index.store(ONE, null, new OtherStorable("Some other text")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(ONE, Storable.class); + } + + assertHits(ID, "one", 1); + ScoreDoc[] docs = assertHits(ID, "one", 1); + Document doc = doc(docs[0].doc); + assertThat(doc.get("value")).isEqualTo("Some other text"); + } + + @Test + void shouldDeleteByIdAndRepository() throws IOException { + Id withRepository = ONE.withRepository("4211"); + try (LuceneIndex index = createIndex()) { + index.store(ONE, null, new Storable("Some other text")); + index.store(withRepository, null, new Storable("New stuff")); + } + + try (LuceneIndex index = createIndex()) { + index.delete(withRepository, Storable.class); + } + + ScoreDoc[] docs = assertHits(ID, "one", 1); + Document doc = doc(docs[0].doc); + assertThat(doc.get("value")).isEqualTo("Some other text"); + } + + @Test + void shouldDeleteByRepository() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), null, new Storable("Some other text")); + index.store(ONE.withRepository("4212"), null, new Storable("New stuff")); + } + + try (LuceneIndex index = createIndex()) { + index.deleteByRepository("4212"); + } + + assertHits(ID, "one", 1); + } + + @Test + void shouldStorePermission() throws IOException { + try (LuceneIndex index = createIndex()) { + index.store(ONE.withRepository("4211"), "repo:4211:read", new Storable("Some other text")); + } + + assertHits(PERMISSION, "repo:4211:read", 1); + } + + private Document doc(int doc) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + return reader.document(doc); + } + } + + @CanIgnoreReturnValue + private ScoreDoc[] assertHits(String field, String value, int expectedHits) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = new IndexSearcher(reader); + TopDocs docs = searcher.search(new TermQuery(new Term(field, value)), 10); + assertThat(docs.totalHits.value).isEqualTo(expectedHits); + return docs.scoreDocs; + } + } + + private LuceneIndex createIndex() throws IOException { + return new LuceneIndex(new DocumentConverter(), createWriter()); + } + + private IndexWriter createWriter() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory, config); + } + + @Value + private static class Storable { + @Indexed + String value; + } + + @Value + private static class OtherStorable { + @Indexed + String value; + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java new file mode 100644 index 0000000000..fff26de53c --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java @@ -0,0 +1,592 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@SubjectAware(value = "trillian", permissions = "abc") +@ExtendWith({MockitoExtension.class, ShiroExtension.class}) +class LuceneQueryBuilderTest { + + private Directory directory; + + @Mock + private IndexOpener opener; + + @BeforeEach + void setUpDirectory() { + directory = new ByteBuffersDirectory(); + } + + @Test + void shouldReturnHitsForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldMatchPartial() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(personDoc("Trillian")); + } + + QueryResult result = query(Person.class, "Trill"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfStarIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "Tr*ll"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfQuestionMarkIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "Tr?ll"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + @SuppressWarnings("java:S5976") + void shouldNotAppendWildcardIfExpertQueryIsUsed() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Trillian")); + } + + QueryResult result = query(Simple.class, "lastName:Trill"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + void shouldSupportFieldsFromParentClass() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Dent"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldIgnoreHitsOfOtherType() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + writer.addDocument(personDoc("Dent")); + } + + QueryResult result = query(InetOrgPerson.class, "Dent"); + assertThat(result.getTotalHits()).isOne(); + } + + @Test + void shouldThrowQueryParseExceptionOnInvalidQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(personDoc("Dent")); + } + assertThrows(QueryParseException.class, () -> query(String.class, ":~:~")); + } + + @Test + void shouldIgnoreNonDefaultFieldsForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "car")); + } + + QueryResult result = query(InetOrgPerson.class, "car"); + assertThat(result.getTotalHits()).isZero(); + } + + @Test + void shouldUseBoostFromAnnotationForBestGuessQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arti", "car")); + writer.addDocument(inetOrgPersonDoc("Fake", "Dent", "Arthur, Arthur, Arthur", "mycar")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + assertThat(result.getTotalHits()).isEqualTo(2); + + List hits = result.getHits(); + Hit arthur = hits.get(0); + assertValueField(arthur, "firstName", "Arthur"); + + Hit fake = hits.get(1); + assertValueField(fake, "firstName", "Fake"); + + assertThat(arthur.getScore()).isGreaterThan(fake.getScore()); + } + + private void assertValueField(Hit hit, String fieldName, Object value) { + assertThat(hit.getFields().get(fieldName)) + .isInstanceOfSatisfying(Hit.ValueField.class, (field) -> { + assertThat(field.isHighlighted()).isFalse(); + assertThat(field.getValue()).isEqualTo(value); + }); + } + + @Test + void shouldReturnHitsForExpertQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Awesome content one")); + writer.addDocument(simpleDoc("Awesome content two")); + writer.addDocument(simpleDoc("Awesome content three")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isEqualTo(3L); + assertThat(result.getHits()).hasSize(3); + } + + @Test + void shouldReturnOnlyHitsOfTypeForExpertQuery() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Ford", "Prefect", "Ford Prefect", "4211")); + writer.addDocument(personDoc("Prefect")); + } + + QueryResult result = query(InetOrgPerson.class, "lastName:prefect"); + assertThat(result.getTotalHits()).isEqualTo(1L); + } + + @Test + void shouldReturnOnlyPermittedHits() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(permissionDoc("Awesome content one", "abc")); + writer.addDocument(permissionDoc("Awesome content two", "cde")); + writer.addDocument(permissionDoc("Awesome content three", "fgh")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isOne(); + + List hits = result.getHits(); + assertThat(hits).hasSize(1).allSatisfy(hit -> { + assertValueField(hit, "content", "Awesome content one"); + assertThat(hit.getScore()).isGreaterThan(0f); + }); + } + + @Test + void shouldFilterByRepository() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(repositoryDoc("Awesome content one", "abc")); + writer.addDocument(repositoryDoc("Awesome content two", "cde")); + writer.addDocument(repositoryDoc("Awesome content three", "fgh")); + } + + QueryResult result; + try (DirectoryReader reader = DirectoryReader.open(directory)) { + when(opener.openForRead("default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder( + opener, "default", new StandardAnalyzer() + ); + result = builder.repository("cde").execute(Simple.class, "content:awesome"); + } + + assertThat(result.getTotalHits()).isOne(); + + List hits = result.getHits(); + assertThat(hits).hasSize(1).allSatisfy(hit -> { + assertValueField(hit, "content", "Awesome content two"); + assertThat(hit.getScore()).isGreaterThan(0f); + }); + } + + @Test + void shouldReturnStringFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("Awesome")); + } + + QueryResult result = query(Simple.class, "content:awesome"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "content", "Awesome") + ); + } + + @Test + void shouldReturnIdOfHit() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Slarti", "Bartfass", "Slartibartfass", "-")); + } + + QueryResult result = query(InetOrgPerson.class, "lastName:Bartfass"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertThat(hit.getId()).isEqualTo("Bartfass") + ); + } + + @Test + void shouldReturnTypeOfHits() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(simpleDoc("We need the type")); + } + + QueryResult result = query(Simple.class, "content:type"); + assertThat(result.getType()).isEqualTo(Simple.class); + } + + @Test + void shouldSupportIntRangeQueries() throws IOException { + Instant now = Instant.now(); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + + QueryResult result = query(Types.class, "intValue:[0 TO 100]"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "intValue", 42) + ); + } + + @Test + void shouldSupportLongRangeQueries() throws IOException { + Instant now = Instant.now(); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + + QueryResult result = query(Types.class, "longValue:[0 TO 100]"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "longValue", 21L) + ); + } + + @Test + void shouldSupportInstantRangeQueries() throws IOException { + Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli()); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(42, 21L, false, now)); + } + long before = now.minus(1, ChronoUnit.MINUTES).toEpochMilli(); + long after = now.plus(1, ChronoUnit.MINUTES).toEpochMilli(); + + String queryString = String.format("instantValue:[%d TO %d]", before, after); + + QueryResult result = query(Types.class, queryString); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "instantValue", now) + ); + } + + @Test + void shouldSupportQueryForBooleanFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(21, 42L, true, Instant.now())); + } + + QueryResult result = query(Types.class, "boolValue:true"); + assertThat(result.getTotalHits()).isOne(); + assertThat(result.getHits()).allSatisfy( + hit -> assertValueField(hit, "boolValue", Boolean.TRUE) + ); + } + + @Test + void shouldReturnValueFieldForHighlightedFieldWithoutFragment() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Marvin", "HoG", "Paranoid Android", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Marvin"); + Hit hit = result.getHits().get(0); + assertValueField(hit, "displayName", "Paranoid Android"); + } + + @Test + void shouldFailBestGuessQueryWithoutDefaultQueryFields() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(1, 2L, false, Instant.now())); + } + assertThrows(NoDefaultQueryFieldsFoundException.class, () -> query(Types.class, "something")); + } + + @Test + void shouldLimitHitsByDefaultSize() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + i)); + } + + QueryResult result = query(Simple.class, "content:counter"); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(10); + } + + @Test + void shouldLimitHitsByConfiguredSize() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + (i + 1))); + } + + QueryResult result = query(Simple.class, "content:counter", null, 2); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(2); + + assertContainsValues( + result, "content", "counter 1", "counter 2" + ); + } + + @Test + void shouldRespectStartValue() throws IOException { + try (IndexWriter writer = writer()) { + for (int i = 0; i < 20; i++) + writer.addDocument(simpleDoc("counter " + (i + 1))); + } + + QueryResult result = query(Simple.class, "content:counter", 10, 3); + assertThat(result.getTotalHits()).isEqualTo(20L); + assertThat(result.getHits()).hasSize(3); + + assertContainsValues( + result, "content", "counter 11", "counter 12", "counter 13" + ); + } + + private void assertContainsValues(QueryResult result, String fieldName, Object... expectedValues) { + List values = result.getHits().stream().map(hit -> { + Hit.ValueField content = (Hit.ValueField) hit.getFields().get(fieldName); + return content.getValue(); + }).collect(Collectors.toList()); + assertThat(values).containsExactly(expectedValues); + } + + @Test + void shouldBeAbleToMarshalQueryResultToJson() throws IOException { + try (IndexWriter writer = writer()) { + writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); + } + + QueryResult result = query(InetOrgPerson.class, "Arthur"); + ObjectMapper mapper = new ObjectMapper(); + + JsonNode root = mapper.valueToTree(result); + assertThat(root.get("totalHits").asInt()).isOne(); + + JsonNode hit = root.get("hits").get(0); + assertThat(hit.get("score").asDouble()).isGreaterThan(0d); + + JsonNode fields = hit.get("fields"); + JsonNode firstName = fields.get("firstName"); + assertThat(firstName.get("highlighted").asBoolean()).isFalse(); + assertThat(firstName.get("value").asText()).isEqualTo("Arthur"); + + JsonNode displayName = fields.get("displayName"); + assertThat(displayName.get("highlighted").asBoolean()).isTrue(); + assertThat(displayName.get("fragments").get(0).asText()).contains("**Arthur**"); + } + + @Test + void shouldBeAbleToMarshalDifferentTypesOfQueryResultToJson() throws IOException { + Instant now = Instant.ofEpochMilli(Instant.now().toEpochMilli()); + try (IndexWriter writer = writer()) { + writer.addDocument(typesDoc(21, 42L, true, now)); + } + + QueryResult result = query(Types.class, "intValue:21"); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + mapper.registerModule(new JavaTimeModule()); + + JsonNode root = mapper.valueToTree(result); + JsonNode fields = root.get("hits").get(0).get("fields"); + assertThat(fields.get("intValue").get("value").asInt()).isEqualTo(21); + assertThat(fields.get("longValue").get("value").asLong()).isEqualTo(42L); + assertThat(fields.get("boolValue").get("value").asBoolean()).isTrue(); + assertThat(fields.get("instantValue").get("value").asText()).isEqualTo(now.toString()); + } + + private QueryResult query(Class type, String queryString) throws IOException { + return query(type, queryString, null, null); + } + + private QueryResult query(Class type, String queryString, Integer start, Integer limit) throws IOException { + try (DirectoryReader reader = DirectoryReader.open(directory)) { + lenient().when(opener.openForRead("default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder( + opener, "default", new StandardAnalyzer() + ); + if (start != null) { + builder.start(start); + } + if (limit != null) { + builder.limit(limit); + } + return builder.execute(type, queryString); + } + } + + private IndexWriter writer() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); + return new IndexWriter(directory, config); + } + + private Document simpleDoc(String content) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + return document; + } + + private Document permissionDoc(String content, String permission) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES)); + return document; + } + + private Document repositoryDoc(String content, String repository) { + Document document = new Document(); + document.add(new TextField("content", content, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Simple.class.getName(), Field.Store.YES)); + document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES)); + return document; + } + + private Document inetOrgPersonDoc(String firstName, String lastName, String displayName, String carLicense) { + Document document = new Document(); + document.add(new TextField("firstName", firstName, Field.Store.YES)); + document.add(new TextField("lastName", lastName, Field.Store.YES)); + document.add(new TextField("displayName", displayName, Field.Store.YES)); + document.add(new TextField("carLicense", carLicense, Field.Store.YES)); + document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, InetOrgPerson.class.getName(), Field.Store.YES)); + return document; + } + + private Document personDoc(String lastName) { + Document document = new Document(); + document.add(new TextField("lastName", lastName, Field.Store.YES)); + document.add(new StringField(FieldNames.TYPE, Person.class.getName(), Field.Store.YES)); + return document; + } + + private Document typesDoc(int intValue, long longValue, boolean boolValue, Instant instantValue) { + Document document = new Document(); + document.add(new IntPoint("intValue", intValue)); + document.add(new StoredField("intValue", intValue)); + document.add(new LongPoint("longValue", longValue)); + document.add(new StoredField("longValue", longValue)); + document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES)); + document.add(new LongPoint("instantValue", instantValue.toEpochMilli())); + document.add(new StoredField("instantValue", instantValue.toEpochMilli())); + document.add(new StringField(FieldNames.TYPE, Types.class.getName(), Field.Store.YES)); + return document; + } + + static class Types { + + @Indexed + private Integer intValue; + @Indexed + private long longValue; + @Indexed + private boolean boolValue; + @Indexed + private Instant instantValue; + + } + + static class Person { + + @Indexed(defaultQuery = true) + private String lastName; + } + + static class InetOrgPerson extends Person { + + @Indexed(defaultQuery = true, boost = 2f) + private String firstName; + + @Indexed(defaultQuery = true, highlighted = true) + private String displayName; + + @Indexed + private String carLicense; + } + + static class Simple { + @Indexed(defaultQuery = true) + private String content; + } + +}