Add detailed search result ui (#1738)

Add a dedicated search page with more results and different types.
Users and groups are now indexed along with repositories.

Co-authored-by: René Pfeuffer <rene.pfeuffer@cloudogu.com>
This commit is contained in:
Sebastian Sdorra
2021-07-28 11:19:00 +02:00
committed by GitHub
parent ad6000722d
commit 91fec0f478
60 changed files with 2665 additions and 517 deletions

View File

@@ -35,6 +35,7 @@ import lombok.Setter;
public class QueryResultDto extends CollectionDto {
private Class<?> type;
private long totalHits;
QueryResultDto(Links links, Embedded embedded) {
super(links, embedded);

View File

@@ -61,6 +61,9 @@ public class SearchParameters {
@PathParam("type")
private String type;
@QueryParam("countOnly")
private boolean countOnly = false;
String getSelfLink() {
return uriInfo.getAbsolutePath().toASCIIString();
}

View File

@@ -32,6 +32,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryCountResult;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
import sonia.scm.web.VndMediaType;
@@ -42,6 +43,7 @@ import javax.ws.rs.BeanParam;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import java.util.Collections;
@Path(SearchResource.PATH)
@OpenAPIDefinition(tags = {
@@ -98,7 +100,18 @@ public class SearchResource {
name = "pageSize",
description = "The maximum number of results per page (defaults to 10)"
)
@Parameter(
name = "countOnly",
description = "If set to 'true', no results will be returned, only the count of hits and the page count"
)
public QueryResultDto query(@Valid @BeanParam SearchParameters params) {
if (params.isCountOnly()) {
return count(params);
}
return search(params);
}
private QueryResultDto search(SearchParameters params) {
QueryResult result = engine.search(IndexNames.DEFAULT)
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
@@ -107,4 +120,11 @@ public class SearchResource {
return mapper.map(params, result);
}
private QueryResultDto count(SearchParameters params) {
QueryCountResult result = engine.search(IndexNames.DEFAULT)
.count(params.getType(), params.getQuery());
return mapper.map(params, new QueryResult(result.getTotalHits(), result.getType(), Collections.emptyList()));
}
}

View File

@@ -0,0 +1,116 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.group;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import javax.inject.Inject;
import javax.inject.Singleton;
@Extension
@Singleton
public class GroupIndexer implements Indexer<Group> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting
static final int VERSION = 1;
private final GroupManager groupManager;
private final IndexQueue indexQueue;
@Inject
public GroupIndexer(GroupManager groupManager, IndexQueue indexQueue) {
this.groupManager = groupManager;
this.indexQueue = indexQueue;
}
@Override
public Class<Group> getType() {
return Group.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Override
public int getVersion() {
return VERSION;
}
@Subscribe(async = false)
public void handleEvent(GroupEvent event) {
new HandlerEventIndexSyncer<>(this).handleEvent(event);
}
@Override
public Updater<Group> open() {
return new GroupIndexUpdater(groupManager, indexQueue.getQueuedIndex(INDEX));
}
public static class GroupIndexUpdater implements Updater<Group> {
private final GroupManager groupManager;
private final Index index;
private GroupIndexUpdater(GroupManager groupManager, Index index) {
this.groupManager = groupManager;
this.index = index;
}
@Override
public void store(Group group) {
index.store(Id.of(group), GroupPermissions.read(group).asShiroString(), group);
}
@Override
public void delete(Group group) {
index.delete(Id.of(group), Group.class);
}
@Override
public void reIndexAll() {
index.deleteByType(Group.class);
for (Group group : groupManager.getAll()) {
store(group);
}
}
@Override
public void close() {
index.close();
}
}
}

View File

@@ -1,140 +0,0 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexLog;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.web.security.AdministrationContext;
import sonia.scm.web.security.PrivilegedAction;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.Optional;
@Extension
@Singleton
public class IndexUpdateListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class);
@VisibleForTesting
static final int INDEX_VERSION = 2;
private final AdministrationContext administrationContext;
private final IndexQueue queue;
private final IndexLogStore indexLogStore;
@Inject
public IndexUpdateListener(AdministrationContext administrationContext, IndexQueue queue, IndexLogStore indexLogStore) {
this.administrationContext = administrationContext;
this.queue = queue;
this.indexLogStore = indexLogStore;
}
@Subscribe(async = false)
public void handleEvent(RepositoryEvent event) {
HandlerEventType type = event.getEventType();
if (type.isPost()) {
updateIndex(type, event.getItem());
}
}
private void updateIndex(HandlerEventType type, Repository repository) {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
if (type == HandlerEventType.DELETE) {
index.deleteByRepository(repository.getId());
} else {
store(index, repository);
}
}
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
Optional<IndexLog> indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class);
if (indexLog.isPresent()) {
int version = indexLog.get().getVersion();
if (version < INDEX_VERSION) {
LOG.debug("repository index {} is older then {}, start reindexing of all repositories", version, INDEX_VERSION);
indexAll();
}
} else {
LOG.debug("could not find log entry for repository index, start reindexing of all repositories");
indexAll();
}
}
private void indexAll() {
administrationContext.runAsAdmin(ReIndexAll.class);
indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION);
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
// we have nothing to destroy
}
private static void store(Index index, Repository repository) {
index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository);
}
static class ReIndexAll implements PrivilegedAction {
private final RepositoryManager repositoryManager;
private final IndexQueue queue;
@Inject
public ReIndexAll(RepositoryManager repositoryManager, IndexQueue queue) {
this.repositoryManager = repositoryManager;
this.queue = queue;
}
@Override
public void run() {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
// delete v1 types
index.deleteByTypeName(Repository.class.getName());
for (Repository repository : repositoryManager.getAll()) {
store(index, repository);
}
}
}
}
}

View File

@@ -0,0 +1,119 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import javax.inject.Inject;
import javax.inject.Singleton;
@Singleton
@Extension
public class RepositoryIndexer implements Indexer<Repository> {
@VisibleForTesting
static final int VERSION = 2;
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
private final RepositoryManager repositoryManager;
private final IndexQueue indexQueue;
@Inject
public RepositoryIndexer(RepositoryManager repositoryManager, IndexQueue indexQueue) {
this.repositoryManager = repositoryManager;
this.indexQueue = indexQueue;
}
@Override
public int getVersion() {
return VERSION;
}
@Override
public Class<Repository> getType() {
return Repository.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Subscribe(async = false)
public void handleEvent(RepositoryEvent event) {
new HandlerEventIndexSyncer<>(this).handleEvent(event);
}
@Override
public Updater<Repository> open() {
return new RepositoryIndexUpdater(repositoryManager, indexQueue.getQueuedIndex(INDEX));
}
public static class RepositoryIndexUpdater implements Updater<Repository> {
private final RepositoryManager repositoryManager;
private final Index index;
public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index index) {
this.repositoryManager = repositoryManager;
this.index = index;
}
@Override
public void store(Repository repository) {
index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository);
}
@Override
public void delete(Repository repository) {
index.deleteByRepository(repository.getId());
}
@Override
public void reIndexAll() {
// v1 used the whole classname as type
index.deleteByTypeName(Repository.class.getName());
index.deleteByType(Repository.class);
for (Repository repository : repositoryManager.getAll()) {
store(repository);
}
}
@Override
public void close() {
index.close();
}
}
}

View File

@@ -0,0 +1,92 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.plugin.Extension;
import sonia.scm.web.security.AdministrationContext;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.Optional;
import java.util.Set;
@Singleton
@Extension
@SuppressWarnings("rawtypes")
public class IndexBootstrapListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(IndexBootstrapListener.class);
private final AdministrationContext administrationContext;
private final IndexLogStore indexLogStore;
private final Set<Indexer> indexers;
@Inject
public IndexBootstrapListener(AdministrationContext administrationContext, IndexLogStore indexLogStore, Set<Indexer> indexers) {
this.administrationContext = administrationContext;
this.indexLogStore = indexLogStore;
this.indexers = indexers;
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
for (Indexer indexer : indexers) {
bootstrap(indexer);
}
}
private void bootstrap(Indexer indexer) {
Optional<IndexLog> indexLog = indexLogStore.get(indexer.getIndex(), indexer.getType());
if (indexLog.isPresent()) {
int version = indexLog.get().getVersion();
if (version < indexer.getVersion()) {
LOG.debug("index version {} is older then {}, start reindexing of all {}", version, indexer.getVersion(), indexer.getType());
indexAll(indexer);
}
} else {
LOG.debug("could not find log entry for {} index, start reindexing", indexer.getType());
indexAll(indexer);
}
}
private void indexAll(Indexer indexer) {
administrationContext.runAsAdmin(() -> {
try (Indexer.Updater updater = indexer.open()) {
updater.reIndexAll();
}
});
indexLogStore.log(indexer.getIndex(), indexer.getType(), indexer.getVersion());
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
// nothing to destroy here
}
}

View File

@@ -38,6 +38,7 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TotalHitCountCollector;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.shiro.SecurityUtils;
@@ -70,8 +71,25 @@ public class LuceneQueryBuilder extends QueryBuilder {
return resolver.resolveClassByName(typeName);
}
@Override
protected QueryCountResult count(QueryParams queryParams) {
TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
return search(
queryParams, totalHitCountCollector,
(searcher, type, query) -> new QueryCountResult(type.getType(), totalHitCountCollector.getTotalHits())
);
}
@Override
protected QueryResult execute(QueryParams queryParams) {
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
return search(queryParams, topScoreCollector, (searcher, searchableType, query) -> {
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
});
}
private <T> T search(QueryParams queryParams, Collector collector, ResultBuilder<T> resultBuilder) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString());
LuceneSearchableType searchableType = resolver.resolve(queryParams.getType());
@@ -87,12 +105,9 @@ public class LuceneQueryBuilder extends QueryBuilder {
try (IndexReader reader = opener.openForRead(indexName)) {
IndexSearcher searcher = new IndexSearcher(reader);
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
Collector collector = new PermissionAwareCollector(reader, topScoreCollector);
searcher.search(query, collector);
searcher.search(query, new PermissionAwareCollector(reader, collector));
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
return resultBuilder.create(searcher, searchableType, parsedQuery);
} catch (IOException e) {
throw new SearchEngineException("failed to search index", e);
} catch (InvalidTokenOffsetsException e) {
@@ -155,4 +170,9 @@ public class LuceneQueryBuilder extends QueryBuilder {
}
return queryString;
}
@FunctionalInterface
private interface ResultBuilder<T> {
T create(IndexSearcher searcher, LuceneSearchableType searchableType, Query query) throws IOException, InvalidTokenOffsetsException;
}
}

View File

@@ -82,6 +82,7 @@ public class QueryResultFactory {
}
return new Hit(document.get(FieldNames.ID), scoreDoc.score, fields);
}
private Optional<Hit.Field> field(Document document, SearchableField field) throws IOException, InvalidTokenOffsetsException {
Object value = field.value(document);
if (value != null) {

View File

@@ -0,0 +1,116 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.user;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import sonia.scm.plugin.Extension;
import sonia.scm.search.HandlerEventIndexSyncer;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.Indexer;
import javax.inject.Inject;
import javax.inject.Singleton;
@Extension
@Singleton
public class UserIndexer implements Indexer<User> {
@VisibleForTesting
static final String INDEX = IndexNames.DEFAULT;
@VisibleForTesting
static final int VERSION = 1;
private final UserManager userManager;
private final IndexQueue queue;
@Inject
public UserIndexer(UserManager userManager, IndexQueue queue) {
this.userManager = userManager;
this.queue = queue;
}
@Override
public Class<User> getType() {
return User.class;
}
@Override
public String getIndex() {
return INDEX;
}
@Override
public int getVersion() {
return VERSION;
}
@Subscribe(async = false)
public void handleEvent(UserEvent event) {
new HandlerEventIndexSyncer<>(this).handleEvent(event);
}
@Override
public Updater<User> open() {
return new UserIndexUpdater(userManager, queue.getQueuedIndex(INDEX));
}
public static class UserIndexUpdater implements Updater<User> {
private final UserManager userManager;
private final Index index;
private UserIndexUpdater(UserManager userManager, Index index) {
this.userManager = userManager;
this.index = index;
}
@Override
public void store(User user) {
index.store(Id.of(user), UserPermissions.read(user).asShiroString(), user);
}
@Override
public void delete(User user) {
index.delete(Id.of(user), User.class);
}
@Override
public void reIndexAll() {
index.deleteByType(User.class);
for (User user : userManager.getAll()) {
store(user);
}
}
@Override
public void close() {
index.close();
}
}
}