Context sensitive search (#2102)

Extend global search to search context-sensitive in repositories and namespaces.
This commit is contained in:
Eduard Heimbuch
2022-08-04 11:29:05 +02:00
parent 6c82142643
commit 550ebefd93
34 changed files with 1061 additions and 308 deletions

View File

@@ -150,7 +150,7 @@ public class IndexDtoGenerator extends HalAppenderMapper {
builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}")));
builder.array(searchLinks());
builder.single(link("searchableTypes", resourceLinks.search().searchableTypes()));
builder.single(link("searchableTypes", resourceLinks.searchableTypes().searchableTypes()));
if (!Strings.isNullOrEmpty(configuration.getAlertsUrl())) {
builder.single(link("alerts", resourceLinks.alerts().get()));

View File

@@ -24,21 +24,29 @@
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Link;
import de.otto.edison.hal.Links;
import sonia.scm.repository.NamespacePermissions;
import sonia.scm.search.SearchEngine;
import sonia.scm.search.SearchableType;
import javax.inject.Inject;
import java.util.List;
import java.util.stream.Collectors;
import static de.otto.edison.hal.Link.link;
import static de.otto.edison.hal.Link.linkBuilder;
import static de.otto.edison.hal.Links.linkingTo;
class NamespaceToNamespaceDtoMapper {
private final ResourceLinks links;
private final SearchEngine searchEngine;
@Inject
NamespaceToNamespaceDtoMapper(ResourceLinks links) {
NamespaceToNamespaceDtoMapper(ResourceLinks links, SearchEngine searchEngine) {
this.links = links;
this.searchEngine = searchEngine;
}
NamespaceDto map(String namespace) {
@@ -51,6 +59,18 @@ class NamespaceToNamespaceDtoMapper {
linkingTo
.single(link("permissions", links.namespacePermission().all(namespace)));
}
linkingTo.array(searchLinks(namespace));
linkingTo.single(link("searchableTypes", links.searchableTypes().searchableTypesForNamespace(namespace)));
return new NamespaceDto(namespace, linkingTo.build());
}
private List<Link> searchLinks(String namespace) {
return searchEngine.getSearchableTypes().stream()
.filter(SearchableType::limitableToNamespace)
.map(SearchableType::getName)
.map(typeName ->
linkBuilder("search", links.search().queryForNamespace(typeName, namespace)).withName(typeName).build()
)
.collect(Collectors.toList());
}
}

View File

@@ -45,15 +45,19 @@ import sonia.scm.repository.api.Command;
import sonia.scm.repository.api.RepositoryService;
import sonia.scm.repository.api.RepositoryServiceFactory;
import sonia.scm.repository.api.ScmProtocol;
import sonia.scm.search.SearchEngine;
import sonia.scm.search.SearchableType;
import sonia.scm.web.EdisonHalAppender;
import sonia.scm.web.api.RepositoryToHalMapper;
import javax.inject.Inject;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static de.otto.edison.hal.Embedded.embeddedBuilder;
import static de.otto.edison.hal.Link.link;
import static de.otto.edison.hal.Link.linkBuilder;
import static de.otto.edison.hal.Links.linkingTo;
import static java.util.stream.Collectors.toList;
@@ -74,6 +78,8 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper<Reposit
private HealthCheckService healthCheckService;
@Inject
private SCMContextProvider contextProvider;
@Inject
private SearchEngine searchEngine;
abstract HealthCheckFailureDto toDto(HealthCheckFailure failure);
@@ -165,6 +171,8 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper<Reposit
if (RepositoryPermissions.healthCheck(repository).isPermitted() && !healthCheckService.checkRunning(repository)) {
linksBuilder.single(link("runHealthCheck", resourceLinks.repository().runHealthCheck(repository.getNamespace(), repository.getName())));
}
linksBuilder.single(link("searchableTypes", resourceLinks.searchableTypes().searchableTypesForRepository(repository.getNamespace(), repository.getName())));
linksBuilder.array(searchLinks(repository.getNamespace(), repository.getName()));
Embedded.Builder embeddedBuilder = embeddedBuilder();
applyEnrichers(new EdisonHalAppender(linksBuilder, embeddedBuilder), repository);
@@ -174,6 +182,16 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper<Reposit
return repositoryDto;
}
private List<Link> searchLinks(String namespace, String name) {
return searchEngine.getSearchableTypes().stream()
.filter(SearchableType::limitableToRepository)
.map(SearchableType::getName)
.map(typeName ->
linkBuilder("search", resourceLinks.search().queryForRepository(namespace, name, typeName)).withName(typeName).build()
)
.collect(Collectors.toList());
}
private boolean isRenameNamespacePossible() {
for (NamespaceStrategy strategy : strategies) {
if (strategy.getClass().getSimpleName().equals(scmConfiguration.getNamespaceStrategy())) {

View File

@@ -1176,15 +1176,44 @@ class ResourceLinks {
private final LinkBuilder searchLinkBuilder;
SearchLinks(ScmPathInfo pathInfo) {
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class);
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class, SearchResource.SearchEndpoints.class);
}
public String query(String type) {
return searchLinkBuilder.method("query").parameters(type).href();
return searchLinkBuilder.method("query").parameters().method("globally").parameters(type).href();
}
public String queryForNamespace(String namespace, String type) {
return searchLinkBuilder.method("query").parameters().method("forNamespace").parameters(type, namespace).href();
}
public String queryForRepository(String namespace, String name, String type) {
return searchLinkBuilder.method("query").parameters().method("forRepository").parameters(namespace, name, type).href();
}
}
public SearchableTypesLinks searchableTypes() {
return new SearchableTypesLinks(accessScmPathInfoStore().get());
}
public static class SearchableTypesLinks {
private final LinkBuilder searchLinkBuilder;
SearchableTypesLinks(ScmPathInfo pathInfo) {
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class, SearchResource.SearchableTypesEndpoints.class);
}
public String searchableTypes() {
return searchLinkBuilder.method("searchableTypes").parameters().href();
return searchLinkBuilder.method("searchableTypes").parameters().method("globally").parameters().href();
}
public String searchableTypesForNamespace(String namespace) {
return searchLinkBuilder.method("searchableTypes").parameters().method("forNamespace").parameters(namespace).href();
}
public String searchableTypesForRepository(String namespace, String name) {
return searchLinkBuilder.method("searchableTypes").parameters().method("forRepository").parameters(namespace, name).href();
}
}

View File

@@ -24,6 +24,7 @@
package sonia.scm.api.v2.resources;
import io.swagger.v3.oas.annotations.Parameter;
import lombok.Getter;
import javax.validation.constraints.Max;
@@ -37,7 +38,7 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
@Getter
public class SearchParameters {
class SearchParameters {
@Context
private UriInfo uriInfo;
@@ -45,26 +46,57 @@ public class SearchParameters {
@NotNull
@Size(min = 2)
@QueryParam("q")
@Parameter(
name = "q",
description = "The search expression",
required = true,
example = "query"
)
private String query;
@Min(0)
@QueryParam("page")
@DefaultValue("0")
@Parameter(
name = "page",
description = "The requested page number of the search results (zero based, defaults to 0)"
)
private int page = 0;
@Min(1)
@Max(100)
@QueryParam("pageSize")
@DefaultValue("10")
@Parameter(
name = "pageSize",
description = "The maximum number of results per page (defaults to 10)"
)
private int pageSize = 10;
@PathParam("type")
@Parameter(
name = "type",
description = "The type to search for",
example = "repository"
)
private String type;
@QueryParam("countOnly")
@Parameter(
name = "countOnly",
description = "If set to 'true', no results will be returned, only the count of hits and the page count"
)
private boolean countOnly = false;
String getSelfLink() {
return uriInfo.getAbsolutePath().toASCIIString();
}
String getNamespace() {
return null;
}
String getRepositoryName() {
return null;
}
}

View File

@@ -0,0 +1,41 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import io.swagger.v3.oas.annotations.Parameter;
import lombok.Getter;
import javax.ws.rs.PathParam;
@Getter
class SearchParametersLimitedToNamespace extends SearchParameters {
@PathParam("namespace")
@Parameter(
name = "namespace",
description = "The namespace the search will be limited to"
)
private String namespace;
}

View File

@@ -0,0 +1,48 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import io.swagger.v3.oas.annotations.Parameter;
import lombok.Getter;
import javax.ws.rs.PathParam;
@Getter
class SearchParametersLimitedToRepository extends SearchParameters {
@PathParam("namespace")
@Parameter(
name = "namespace",
description = "The namespace of the repository the search will be limited to"
)
private String namespace;
@PathParam("name")
@Parameter(
name = "name",
description = "The name of the repository the search will be limited to"
)
private String repositoryName;
}

View File

@@ -24,6 +24,7 @@
package sonia.scm.api.v2.resources;
import com.google.common.base.Strings;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
@@ -31,9 +32,14 @@ import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.repository.NamespaceAndName;
import sonia.scm.repository.Repository;
import sonia.scm.repository.RepositoryManager;
import sonia.scm.search.QueryBuilder;
import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
import sonia.scm.search.SearchableType;
import sonia.scm.web.VndMediaType;
import javax.inject.Inject;
@@ -41,9 +47,12 @@ import javax.validation.Valid;
import javax.ws.rs.BeanParam;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@Path(SearchResource.PATH)
@@ -57,23 +66,22 @@ public class SearchResource {
private final SearchEngine engine;
private final QueryResultMapper queryResultMapper;
private final SearchableTypeMapper searchableTypeMapper;
private final RepositoryManager repositoryManager;
@Inject
public SearchResource(SearchEngine engine, QueryResultMapper mapper, SearchableTypeMapper searchableTypeMapper) {
public SearchResource(SearchEngine engine, QueryResultMapper mapper, SearchableTypeMapper searchableTypeMapper, RepositoryManager repositoryManager) {
this.engine = engine;
this.queryResultMapper = mapper;
this.searchableTypeMapper = searchableTypeMapper;
this.repositoryManager = repositoryManager;
}
@Path("query")
public SearchEndpoints query() {
return new SearchEndpoints();
}
@GET
@Path("query/{type}")
@Produces(VndMediaType.QUERY_RESULT)
@Operation(
summary = "Query result",
description = "Returns a collection of matched hits.",
tags = "Search",
operationId = "search_query"
)
@ApiResponse(
responseCode = "200",
description = "success",
@@ -90,39 +98,95 @@ public class SearchResource {
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
))
@Parameter(
name = "query",
description = "The search expression",
required = true
)
@Parameter(
name = "page",
description = "The requested page number of the search results (zero based, defaults to 0)"
)
@Parameter(
name = "pageSize",
description = "The maximum number of results per page (defaults to 10)"
)
@Parameter(
name = "countOnly",
description = "If set to 'true', no results will be returned, only the count of hits and the page count"
)
public QueryResultDto query(@Valid @BeanParam SearchParameters params) {
if (params.isCountOnly()) {
return count(params);
public class SearchEndpoints {
@GET
@Path("{type}")
@Operation(
summary = "Global query result",
description = "Returns a collection of matched hits.",
tags = "Search",
operationId = "search_query"
)
public QueryResultDto globally(@Valid @BeanParam SearchParameters params) {
if (params.isCountOnly()) {
return count(params);
}
return search(params);
}
@GET
@Path("{namespace}/{type}")
@Operation(
summary = "Query result for a namespace",
description = "Returns a collection of matched hits limited to the namespace.",
tags = "Search",
operationId = "search_query_for_namespace"
)
public QueryResultDto forNamespace(@Valid @BeanParam SearchParametersLimitedToNamespace params) {
if (params.isCountOnly()) {
return count(params);
}
return search(params);
}
@GET
@Path("{namespace}/{name}/{type}")
@Operation(
summary = "Query result for a repository",
description = "Returns a collection of matched hits limited to the repository specified by namespace and name.",
tags = "Search",
operationId = "search_query_for_repository"
)
public QueryResultDto forRepository(@Valid @BeanParam SearchParametersLimitedToRepository params) {
if (params.isCountOnly()) {
return count(params);
}
return search(params);
}
private QueryResultDto search(SearchParameters params) {
QueryBuilder<Object> queryBuilder = engine.forType(params.getType())
.search()
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize());
filterByContext(params, queryBuilder);
return queryResultMapper.map(params, queryBuilder.execute(params.getQuery()));
}
private QueryResultDto count(SearchParameters params) {
QueryBuilder<Object> queryBuilder = engine.forType(params.getType())
.search();
filterByContext(params, queryBuilder);
QueryCountResult result = queryBuilder.count(params.getQuery());
return queryResultMapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList()));
}
private void filterByContext(SearchParameters params, QueryBuilder<Object> queryBuilder) {
if (!Strings.isNullOrEmpty(params.getNamespace())) {
if (!Strings.isNullOrEmpty(params.getRepositoryName())) {
Repository repository = repositoryManager.get(new NamespaceAndName(params.getNamespace(), params.getRepositoryName()));
queryBuilder.filter(repository);
} else {
repositoryManager.getAll().stream()
.filter(r -> r.getNamespace().equals(params.getNamespace()))
.forEach(queryBuilder::filter);
}
}
}
return search(params);
}
@GET
@Path("searchableTypes")
public SearchableTypesEndpoints searchableTypes() {
return new SearchableTypesEndpoints();
}
@Produces(VndMediaType.SEARCHABLE_TYPE_COLLECTION)
@Operation(
summary = "Searchable types",
description = "Returns a collection of all searchable types.",
tags = "Search",
operationId = "searchable_types"
)
@ApiResponse(
responseCode = "200",
description = "success",
@@ -138,26 +202,67 @@ public class SearchResource {
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
))
public Collection<SearchableTypeDto> searchableTypes() {
return engine.getSearchableTypes().stream().map(searchableTypeMapper::map).collect(Collectors.toList());
public class SearchableTypesEndpoints {
@GET
@Path("")
@Operation(
summary = "Globally searchable types",
description = "Returns a collection of all searchable types.",
tags = "Search",
operationId = "searchable_types"
)
public Collection<SearchableTypeDto> globally() {
return getTypes(t -> true);
}
@GET
@Path("{namespace}")
@Operation(
summary = "Searchable types in a namespace",
description = "Returns a collection of all searchable types when scoped to a namespace.",
tags = "Search",
operationId = "searchable_types_for_namespace"
)
public Collection<SearchableTypeDto> forNamespace(
@Parameter(
name = "namespace",
description = "The namespace to get the types for"
)
@PathParam("namespace") String namespace) {
return getTypes(SearchableType::limitableToNamespace);
}
@GET
@Path("{namespace}/{name}")
@Operation(
summary = "Searchable types in a repository",
description = "Returns a collection of all searchable types when scoped to a repository.",
tags = "Search",
operationId = "searchable_types_for_repository"
)
public Collection<SearchableTypeDto> forRepository(
@Parameter(
name = "namespace",
description = "The namespace of the repository to get the types for"
)
@PathParam("namespace")
String namespace,
@Parameter(
name = "name",
description = "The name of the repository to get the types for"
)
@PathParam("name")
String name
) {
return getTypes(SearchableType::limitableToRepository);
}
private List<SearchableTypeDto> getTypes(Predicate<SearchableType> predicate) {
return engine.getSearchableTypes().stream()
.filter(predicate)
.map(searchableTypeMapper::map)
.collect(Collectors.toList());
}
}
private QueryResultDto search(SearchParameters params) {
QueryResult result = engine.forType(params.getType())
.search()
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
.execute(params.getQuery());
return queryResultMapper.map(params, result);
}
private QueryResultDto count(SearchParameters params) {
QueryCountResult result = engine.forType(params.getType())
.search()
.count(params.getQuery());
return queryResultMapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList()));
}
}

View File

@@ -37,10 +37,12 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Supplier;
import static java.util.Collections.singleton;
import static sonia.scm.search.FieldNames.ID;
import static sonia.scm.search.FieldNames.PERMISSION;
@@ -153,15 +155,15 @@ class LuceneIndex<T> implements Index<T>, AutoCloseable {
private class LuceneDeleteBy implements DeleteBy {
private final Map<Class<?>, String> map = new HashMap<>();
private final Map<Class<?>, Collection<String>> map = new HashMap<>();
private LuceneDeleteBy(Class<?> type, String id) {
map.put(type, id);
map.put(type, singleton(id));
}
@Override
public DeleteBy and(Class<?> type, String id) {
map.put(type, id);
map.put(type, singleton(id));
return this;
}

View File

@@ -50,6 +50,8 @@ public class LuceneSearchableType implements SearchableType {
Map<String, Float> boosts;
Map<String, PointsConfig> pointsConfig;
TypeConverter typeConverter;
boolean repositoryScoped;
boolean namespaceScoped;
public LuceneSearchableType(Class<?> type, @Nonnull IndexedType annotation, List<LuceneSearchableField> fields) {
this.type = type;
@@ -60,6 +62,8 @@ public class LuceneSearchableType implements SearchableType {
this.boosts = boosts(fields);
this.pointsConfig = pointsConfig(fields);
this.typeConverter = TypeConverters.create(type);
this.repositoryScoped = annotation.repositoryScoped();
this.namespaceScoped = annotation.namespaceScoped();
}
public Optional<String> getPermission() {
@@ -106,4 +110,14 @@ public class LuceneSearchableType implements SearchableType {
public Collection<LuceneSearchableField> getAllFields() {
return Collections.unmodifiableCollection(fields);
}
@Override
public boolean limitableToRepository() {
return repositoryScoped;
}
@Override
public boolean limitableToNamespace() {
return repositoryScoped || namespaceScoped;
}
}

View File

@@ -25,13 +25,16 @@
package sonia.scm.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import java.util.Collection;
import java.util.Map;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
import static org.apache.lucene.search.BooleanClause.Occur.SHOULD;
final class Queries {
@@ -39,7 +42,7 @@ final class Queries {
}
static Query filter(Query query, QueryBuilder.QueryParams params) {
Map<Class<?>, String> filters = params.getFilters();
Map<Class<?>, Collection<String>> filters = params.getFilters();
if (!filters.isEmpty()) {
BooleanQuery.Builder builder = builder(filters);
builder.add(query, MUST);
@@ -48,15 +51,21 @@ final class Queries {
return query;
}
static Query filterQuery(Map<Class<?>, String> filters) {
static Query filterQuery(Map<Class<?>, Collection<String>> filters) {
return builder(filters).build();
}
private static BooleanQuery.Builder builder(Map<Class<?>, String> filters) {
private static BooleanQuery.Builder builder(Map<Class<?>, Collection<String>> filters) {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (Map.Entry<Class<?>, String> e : filters.entrySet()) {
Term term = createTerm(e.getKey(), e.getValue());
builder.add(new TermQuery(term), MUST);
for (Map.Entry<Class<?>, Collection<String>> e : filters.entrySet()) {
BooleanQuery.Builder filterBuilder = new BooleanQuery.Builder();
e.getValue().forEach(
value -> {
Term term = createTerm(e.getKey(), value);
filterBuilder.add(new TermQuery(term), SHOULD);
}
);
builder.add(new BooleanClause(filterBuilder.build(), MUST));
}
return builder;
}