Nemcio/SCM-Manager
1
0
Fork 0
mirror of https://github.com/scm-manager/scm-manager.git synced 2026-01-19 14:02:13 +01:00
Code Issues Packages Projects Releases Activity

Add search engine and quick search for repositories (#1727)

Browse Source 
Add a powerful search engine based on lucene to the scm-manager api.
The api can be used to index objects, simply by annotating them and add them to an index.
The first indexed object is the repository which could queried by quick search in the header.
This commit is contained in:
Sebastian Sdorra
2021-07-14 11:49:38 +02:00
committed by GitHub
parent ce4b869a7a
commit e321133ff7
88 changed files with 6052 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scm-webapp/src/main/java/sonia/scm/api/v2/resources/CollectionDto.java
View File

@@ -21,7 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
@@ -31,6 +31,7 @@ import lombok.Getter;
import lombok.Setter;
@Getter @Setter
@SuppressWarnings("squid:S2160") // we do not need equals for dto
class CollectionDto extends HalRepresentation {
private int page;

49
scm-webapp/src/main/java/sonia/scm/api/v2/resources/HitDto.java Normal file
View File

@@ -0,0 +1,49 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.HalRepresentation;
import de.otto.edison.hal.Links;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import sonia.scm.search.Hit;
import java.util.Map;
@Getter
@Setter
@NoArgsConstructor
@SuppressWarnings("java:S2160") // we do not need this for dto
public class HitDto extends HalRepresentation {
private float score;
private Map<String, Hit.Field> fields;
public HitDto(Links links, Embedded embedded) {
super(links, embedded);
}
}

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/IndexDtoGenerator.java
View File

@@ -131,6 +131,8 @@ public class IndexDtoGenerator extends HalAppenderMapper {
builder.single(link("namespaceStrategies", resourceLinks.namespaceStrategies().self()));
builder.single(link("repositoryRoles", resourceLinks.repositoryRoleCollection().self()));
builder.single(link("importLog", resourceLinks.repository().importLog("IMPORT_LOG_ID").replace("IMPORT_LOG_ID", "{logId}")));
builder.single(link("search", resourceLinks.search().search()));
} else {
builder.single(link("login", resourceLinks.authentication().jsonLogin()));
}

2
scm-webapp/src/main/java/sonia/scm/api/v2/resources/MapperModule.java
View File

@@ -91,5 +91,7 @@ public class MapperModule extends AbstractModule {
bind(RepositoryExportInformationToDtoMapper.class).to(Mappers.getMapperClass(RepositoryExportInformationToDtoMapper.class));
bind(RepositoryImportDtoToRepositoryImportParametersMapper.class).to(Mappers.getMapperClass(RepositoryImportDtoToRepositoryImportParametersMapper.class));
bind(QueryResultMapper.class).to(Mappers.getMapperClass(QueryResultMapper.class));
}
}

42
scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultDto.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.Links;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@SuppressWarnings("squid:S2160") // we do not need equals for dto
public class QueryResultDto extends CollectionDto {
private Class<?> type;
QueryResultDto(Links links, Embedded embedded) {
super(links, embedded);
}
}

120
scm-webapp/src/main/java/sonia/scm/api/v2/resources/QueryResultMapper.java Normal file
View File

@@ -0,0 +1,120 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import com.damnhandy.uri.template.UriTemplate;
import de.otto.edison.hal.Embedded;
import de.otto.edison.hal.Links;
import de.otto.edison.hal.paging.NumberedPaging;
import de.otto.edison.hal.paging.PagingRel;
import org.mapstruct.AfterMapping;
import org.mapstruct.Context;
import org.mapstruct.Mapper;
import org.mapstruct.MappingTarget;
import org.mapstruct.ObjectFactory;
import sonia.scm.search.Hit;
import sonia.scm.search.QueryResult;
import sonia.scm.web.EdisonHalAppender;
import javax.annotation.Nonnull;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
import static com.damnhandy.uri.template.UriTemplate.fromTemplate;
import static de.otto.edison.hal.Links.linkingTo;
import static de.otto.edison.hal.paging.NumberedPaging.zeroBasedNumberedPaging;
@Mapper
public abstract class QueryResultMapper extends HalAppenderMapper {
public abstract QueryResultDto map(@Context SearchParameters params, QueryResult result);
@AfterMapping
void setPageValues(@MappingTarget QueryResultDto dto, QueryResult result, @Context SearchParameters params) {
int totalHits = (int) result.getTotalHits();
dto.setPageTotal(computePageTotal(totalHits, params.getPageSize()));
dto.setPage(params.getPage());
}
@Nonnull
@ObjectFactory
QueryResultDto createDto(@Context SearchParameters params, QueryResult result) {
int totalHits = (int) result.getTotalHits();
Links.Builder links = links(params, totalHits);
Embedded.Builder embedded = hits(result);
applyEnrichers(new EdisonHalAppender(links, embedded), result);
return new QueryResultDto(links.build(), embedded.build());
}
@Nonnull
private QueryResultDto createDto(SearchParameters params, QueryResult result, int totalHits) {
Links.Builder links = links(params, totalHits);
Embedded.Builder embedded = hits(result);
applyEnrichers(new EdisonHalAppender(links, embedded), result);
return new QueryResultDto(links.build(), embedded.build());
}
private Links.Builder links(SearchParameters params, int totalHits) {
NumberedPaging paging = zeroBasedNumberedPaging(params.getPage(), params.getPageSize(), totalHits);
UriTemplate uriTemplate = fromTemplate(params.getSelfLink() + "{?q,page,pageSize}");
uriTemplate.set("q", params.getQuery());
return linkingTo()
.with(paging.links(
uriTemplate,
EnumSet.allOf(PagingRel.class))
);
}
@Nonnull
private Embedded.Builder hits(QueryResult result) {
List<HitDto> hits = result.getHits()
.stream()
.map(hit -> map(result, hit))
.collect(Collectors.toList());
return Embedded.embeddedBuilder().with("hits", hits);
}
@ObjectFactory
protected HitDto createHitDto(@Context QueryResult queryResult, Hit hit) {
Links.Builder links = linkingTo();
Embedded.Builder embedded = Embedded.embeddedBuilder();
applyEnrichers(new EdisonHalAppender(links, embedded), hit, queryResult);
return new HitDto(links.build(), embedded.build());
}
private int computePageTotal(int totalHits, int pageSize) {
if (totalHits % pageSize > 0) {
return totalHits / pageSize + 1;
} else {
return totalHits / pageSize;
}
}
protected abstract HitDto map(@Context QueryResult queryResult, Hit hit);
}

17
scm-webapp/src/main/java/sonia/scm/api/v2/resources/ResourceLinks.java
View File

@@ -1113,6 +1113,23 @@ class ResourceLinks {
}
}
public SearchLinks search() {
return new SearchLinks(scmPathInfoStore.get());
}
public static class SearchLinks {
private final LinkBuilder searchLinkBuilder;
SearchLinks(ScmPathInfo pathInfo) {
this.searchLinkBuilder = new LinkBuilder(pathInfo, SearchResource.class);
}
public String search() {
return searchLinkBuilder.method("search").parameters().href();
}
}
public InitialAdminAccountLinks initialAdminAccount() {
return new InitialAdminAccountLinks(new LinkBuilder(scmPathInfoStore.get(), InitializationResource.class, AdminAccountStartupResource.class));
}

61
scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchParameters.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import lombok.Getter;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.Size;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
@Getter
public class SearchParameters {
@Context
private UriInfo uriInfo;
@Size(min = 2)
@QueryParam("q")
private String query;
@Min(0)
@QueryParam("page")
@DefaultValue("0")
private int page = 0;
@Min(1)
@Max(100)
@QueryParam("pageSize")
@DefaultValue("10")
private int pageSize = 10;
String getSelfLink() {
return uriInfo.getAbsolutePath().toASCIIString();
}
}

111
scm-webapp/src/main/java/sonia/scm/api/v2/resources/SearchResource.java Normal file
View File

@@ -0,0 +1,111 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.api.v2.resources;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import sonia.scm.repository.Repository;
import sonia.scm.search.IndexNames;
import sonia.scm.search.QueryResult;
import sonia.scm.search.SearchEngine;
import sonia.scm.web.VndMediaType;
import javax.inject.Inject;
import javax.validation.Valid;
import javax.ws.rs.BeanParam;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@Path(SearchResource.PATH)
@OpenAPIDefinition(tags = {
@Tag(name = "Search", description = "Search related endpoints")
})
public class SearchResource {
static final String PATH = "v2/search";
private final SearchEngine engine;
private final QueryResultMapper mapper;
@Inject
public SearchResource(SearchEngine engine, QueryResultMapper mapper) {
this.engine = engine;
this.mapper = mapper;
}
@GET
@Path("")
@Produces(VndMediaType.QUERY_RESULT)
@Operation(
summary = "Query result",
description = "Returns a collection of matched hits.",
tags = "Search",
operationId = "search_query"
)
@ApiResponse(
responseCode = "200",
description = "success",
content = @Content(
mediaType = VndMediaType.QUERY_RESULT,
schema = @Schema(implementation = QueryResultDto.class)
)
)
@ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials")
@ApiResponse(
responseCode = "500",
description = "internal server error",
content = @Content(
mediaType = VndMediaType.ERROR_TYPE,
schema = @Schema(implementation = ErrorDto.class)
))
@Parameter(
name = "query",
description = "The search expression",
required = true
)
@Parameter(
name = "page",
description = "The requested page number of the search results (zero based, defaults to 0)"
)
@Parameter(
name = "pageSize",
description = "The maximum number of results per page (defaults to 10)"
)
public QueryResultDto search(@Valid @BeanParam SearchParameters params) {
QueryResult result = engine.search(IndexNames.DEFAULT)
.start(params.getPage() * params.getPageSize())
.limit(params.getPageSize())
.execute(Repository.class, params.getQuery());
return mapper.map(params, result);
}
}

11
scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java
View File

@@ -98,6 +98,12 @@ import sonia.scm.repository.xml.XmlRepositoryDAO;
import sonia.scm.repository.xml.XmlRepositoryRoleDAO;
import sonia.scm.schedule.CronScheduler;
import sonia.scm.schedule.Scheduler;
import sonia.scm.search.DefaultIndexLogStore;
import sonia.scm.search.DefaultIndexQueue;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexQueue;
import sonia.scm.search.LuceneSearchEngine;
import sonia.scm.search.SearchEngine;
import sonia.scm.security.AccessTokenCookieIssuer;
import sonia.scm.security.AuthorizationChangedEventProducer;
import sonia.scm.security.ConfigurableLoginAttemptHandler;
@@ -279,6 +285,11 @@ class ScmServletModule extends ServletModule {
bind(NotificationSender.class).to(DefaultNotificationSender.class);
bind(InitializationFinisher.class).to(DefaultInitializationFinisher.class);
// bind search stuff
bind(IndexQueue.class, DefaultIndexQueue.class);
bind(SearchEngine.class, LuceneSearchEngine.class);
bind(IndexLogStore.class, DefaultIndexLogStore.class);
}
private <T> void bind(Class<T> clazz, Class<? extends T> defaultImplementation) {

128
scm-webapp/src/main/java/sonia/scm/repository/IndexUpdateListener.java Normal file
View File

@@ -0,0 +1,128 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.repository;
import com.github.legman.Subscribe;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import sonia.scm.HandlerEventType;
import sonia.scm.plugin.Extension;
import sonia.scm.search.Id;
import sonia.scm.search.Index;
import sonia.scm.search.IndexLog;
import sonia.scm.search.IndexLogStore;
import sonia.scm.search.IndexNames;
import sonia.scm.search.IndexQueue;
import sonia.scm.web.security.AdministrationContext;
import sonia.scm.web.security.PrivilegedAction;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.util.Optional;
@Extension
@Singleton
public class IndexUpdateListener implements ServletContextListener {
private static final Logger LOG = LoggerFactory.getLogger(IndexUpdateListener.class);
@VisibleForTesting
static final int INDEX_VERSION = 1;
private final AdministrationContext administrationContext;
private final IndexQueue queue;
private final IndexLogStore indexLogStore;
@Inject
public IndexUpdateListener(AdministrationContext administrationContext, IndexQueue queue, IndexLogStore indexLogStore) {
this.administrationContext = administrationContext;
this.queue = queue;
this.indexLogStore = indexLogStore;
}
@Subscribe(async = false)
public void handleEvent(RepositoryEvent event) {
HandlerEventType type = event.getEventType();
if (type.isPost()) {
updateIndex(type, event.getItem());
}
}
private void updateIndex(HandlerEventType type, Repository repository) {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
if (type == HandlerEventType.DELETE) {
index.deleteByRepository(repository.getId());
} else {
store(index, repository);
}
}
}
@Override
public void contextInitialized(ServletContextEvent servletContextEvent) {
Optional<IndexLog> indexLog = indexLogStore.get(IndexNames.DEFAULT, Repository.class);
if (!indexLog.isPresent()) {
LOG.debug("could not find log entry for repository index, start reindexing of all repositories");
administrationContext.runAsAdmin(ReIndexAll.class);
indexLogStore.log(IndexNames.DEFAULT, Repository.class, INDEX_VERSION);
}
}
@Override
public void contextDestroyed(ServletContextEvent servletContextEvent) {
// we have nothing to destroy
}
private static void store(Index index, Repository repository) {
index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository);
}
static class ReIndexAll implements PrivilegedAction {
private final RepositoryManager repositoryManager;
private final IndexQueue queue;
@Inject
public ReIndexAll(RepositoryManager repositoryManager, IndexQueue queue) {
this.repositoryManager = repositoryManager;
this.queue = queue;
}
@Override
public void run() {
try (Index index = queue.getQueuedIndex(IndexNames.DEFAULT)) {
for (Repository repository : repositoryManager.getAll()) {
store(index, repository);
}
}
}
}
}

62
scm-webapp/src/main/java/sonia/scm/search/AnalyzerFactory.java Normal file
View File

@@ -0,0 +1,62 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.de.GermanAnalyzer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.es.SpanishAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import javax.annotation.Nonnull;
public class AnalyzerFactory {
@Nonnull
public Analyzer create(IndexOptions options) {
if (options.getType() == IndexOptions.Type.NATURAL_LANGUAGE) {
return createNaturalLanguageAnalyzer(options.getLocale().getLanguage());
}
return createDefaultAnalyzer();
}
private Analyzer createDefaultAnalyzer() {
return new StandardAnalyzer();
}
private Analyzer createNaturalLanguageAnalyzer(String lang) {
switch (lang) {
case "en":
return new EnglishAnalyzer();
case "de":
return new GermanAnalyzer();
case "es":
return new SpanishAnalyzer();
default:
return createDefaultAnalyzer();
}
}
}

59
scm-webapp/src/main/java/sonia/scm/search/DefaultIndexLogStore.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import sonia.scm.store.DataStore;
import sonia.scm.store.DataStoreFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.Optional;
@Singleton
public class DefaultIndexLogStore implements IndexLogStore {
private final DataStore<IndexLog> dataStore;
@Inject
public DefaultIndexLogStore(DataStoreFactory dataStoreFactory) {
this.dataStore = dataStoreFactory.withType(IndexLog.class).withName("index-log").build();
}
@Override
public void log(String index,Class<?> type, int version) {
String id = id(index, type);
dataStore.put(id, new IndexLog(version));
}
private String id(String index, Class<?> type) {
return index + "_" + type.getName();
}
@Override
public Optional<IndexLog> get(String index, Class<?> type) {
String id = id(index, type);
return dataStore.getOptional(id);
}
}

77
scm-webapp/src/main/java/sonia/scm/search/DefaultIndexQueue.java Normal file
View File

@@ -0,0 +1,77 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.annotations.VisibleForTesting;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
@Singleton
public class DefaultIndexQueue implements IndexQueue, Closeable {
private final ExecutorService executor = Executors.newSingleThreadExecutor();
private final AtomicLong size = new AtomicLong(0);
private final SearchEngine searchEngine;
@Inject
public DefaultIndexQueue(SearchEngine searchEngine) {
this.searchEngine = searchEngine;
}
@Override
public Index getQueuedIndex(String name, IndexOptions indexOptions) {
return new QueuedIndex(this, name, indexOptions);
}
public SearchEngine getSearchEngine() {
return searchEngine;
}
void enqueue(IndexQueueTaskWrapper task) {
size.incrementAndGet();
executor.execute(() -> {
task.run();
size.decrementAndGet();
});
}
@VisibleForTesting
long getSize() {
return size.get();
}
@Override
public void close() throws IOException {
executor.shutdown();
}
}

144
scm-webapp/src/main/java/sonia/scm/search/DocumentConverter.java Normal file
View File

@@ -0,0 +1,144 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import javax.inject.Singleton;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Collections.emptySet;
@Singleton
public class DocumentConverter {
private final Map<Class<?>, TypeConverter> typeConverter = new ConcurrentHashMap<>();
Document convert(Object object) {
TypeConverter converter = typeConverter.computeIfAbsent(object.getClass(), this::createTypeConverter);
try {
return converter.convert(object);
} catch (IllegalAccessException | InvocationTargetException ex) {
throw new SearchEngineException("failed to create document", ex);
}
}
private TypeConverter createTypeConverter(Class<?> type) {
List<FieldConverter> fieldConverters = new ArrayList<>();
collectFields(fieldConverters, type);
return new TypeConverter(fieldConverters);
}
private void collectFields(List<FieldConverter> fieldConverters, Class<?> type) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(fieldConverters, parent);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
IndexableFieldFactory fieldFactory = IndexableFields.create(field, indexed);
Method getter = findGetter(type, field);
fieldConverters.add(new FieldConverter(field, getter, indexed, fieldFactory));
}
}
}
private Method findGetter(Class<?> type, Field field) {
String name = createGetterName(field);
try {
return type.getMethod(name);
} catch (NoSuchMethodException ex) {
throw new NonReadableFieldException("could not find getter for field", ex);
}
}
private String createGetterName(Field field) {
String fieldName = field.getName();
String prefix = "get";
if (field.getType() == Boolean.TYPE) {
prefix = "is";
}
return prefix + fieldName.substring(0, 1).toUpperCase(Locale.ENGLISH) + fieldName.substring(1);
}
private static class TypeConverter {
private final List<FieldConverter> fieldConverters;
private TypeConverter(List<FieldConverter> fieldConverters) {
this.fieldConverters = fieldConverters;
}
public Document convert(Object object) throws IllegalAccessException, InvocationTargetException {
Document document = new Document();
for (FieldConverter fieldConverter : fieldConverters) {
for (IndexableField field : fieldConverter.convert(object)) {
document.add(field);
}
}
return document;
}
}
private static class FieldConverter {
private final Method getter;
private final IndexableFieldFactory fieldFactory;
private final String name;
private FieldConverter(Field field, Method getter, Indexed indexed, IndexableFieldFactory fieldFactory) {
this.getter = getter;
this.fieldFactory = fieldFactory;
this.name = createName(field, indexed);
}
private String createName(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (Strings.isNullOrEmpty(nameFromAnnotation)) {
return field.getName();
}
return nameFromAnnotation;
}
Iterable<IndexableField> convert(Object object) throws IllegalAccessException, InvocationTargetException {
Object value = getter.invoke(object);
if (value != null) {
return fieldFactory.create(name, value);
}
return emptySet();
}
}
}

35
scm-webapp/src/main/java/sonia/scm/search/FieldNames.java Normal file
View File

@@ -0,0 +1,35 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
final class FieldNames {
private FieldNames(){}
static final String UID = "_uid";
static final String ID = "_id";
static final String TYPE = "_type";
static final String REPOSITORY = "_repository";
static final String PERMISSION = "_permission";
}

65
scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java Normal file
View File

@@ -0,0 +1,65 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import sonia.scm.SCMContextProvider;
import javax.inject.Inject;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class IndexOpener {
private final Path directory;
private final AnalyzerFactory analyzerFactory;
@Inject
public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) {
directory = context.resolve(Paths.get("index"));
this.analyzerFactory = analyzerFactory;
}
public IndexReader openForRead(String name) throws IOException {
return DirectoryReader.open(directory(name));
}
public IndexWriter openForWrite(String name, IndexOptions options) throws IOException {
IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(options));
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(name), config);
}
private Directory directory(String name) throws IOException {
return FSDirectory.open(directory.resolve(name));
}
}

32
scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
@FunctionalInterface
public interface IndexQueueTask {
void updateIndex(Index index);
}

56
scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java Normal file
View File

@@ -0,0 +1,56 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class IndexQueueTaskWrapper implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class);
private final SearchEngine searchEngine;
private final String indexName;
private final IndexOptions options;
private final Iterable<IndexQueueTask> tasks;
IndexQueueTaskWrapper(SearchEngine searchEngine, String indexName, IndexOptions options, Iterable<IndexQueueTask> tasks) {
this.searchEngine = searchEngine;
this.indexName = indexName;
this.options = options;
this.tasks = tasks;
}
@Override
public void run() {
try (Index index = searchEngine.getOrCreate(this.indexName, options)) {
for (IndexQueueTask task : tasks) {
task.updateIndex(index);
}
} catch (Exception e) {
LOG.warn("failure during execution of index task for index {}", indexName, e);
}
}
}

32
scm-webapp/src/main/java/sonia/scm/search/IndexableFieldFactory.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.IndexableField;
@FunctionalInterface
interface IndexableFieldFactory {
Iterable<IndexableField> create(String name, Object value);
}

172
scm-webapp/src/main/java/sonia/scm/search/IndexableFields.java Normal file
View File

@@ -0,0 +1,172 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
import java.text.DecimalFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static java.util.Collections.singleton;
import static sonia.scm.search.TypeCheck.isBoolean;
import static sonia.scm.search.TypeCheck.isInstant;
import static sonia.scm.search.TypeCheck.isInteger;
import static sonia.scm.search.TypeCheck.isLong;
class IndexableFields {
private IndexableFields() {
}
static PointsConfig pointConfig(Field field) {
Class<?> type = field.getType();
if (isLong(type) || isInstant(type)) {
return new PointsConfig(new DecimalFormat(), Long.class);
} else if (isInteger(type)) {
return new PointsConfig(new DecimalFormat(), Integer.class);
}
return null;
}
static IndexableFieldFactory create(Field field, Indexed indexed) {
Class<?> fieldType = field.getType();
Indexed.Type indexType = indexed.type();
if (fieldType == String.class) {
return new StringFieldFactory(indexType);
} else if (isLong(fieldType)) {
return new LongFieldFactory(indexType);
} else if (isInteger(fieldType)) {
return new IntegerFieldFactory(indexType);
} else if (isBoolean(fieldType)) {
return new BooleanFieldFactory(indexType);
} else if (isInstant(fieldType)) {
return new InstantFieldFactory(indexType);
} else {
throw new UnsupportedTypeOfFieldException(fieldType, field.getName());
}
}
private static class StringFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private StringFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
String stringValue = (String) value;
if (type.isTokenized()) {
return singleton(new TextField(name, stringValue, Store.YES));
} else if (type.isSearchable()) {
return singleton(new StringField(name, stringValue, Store.YES));
} else {
return singleton(new StoredField(name, stringValue));
}
}
}
private static class LongFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private LongFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Long longValue = (Long) value;
List<IndexableField> fields = new ArrayList<>();
if (type.isSearchable()) {
fields.add(new LongPoint(name, longValue));
}
fields.add(new StoredField(name, longValue));
return Collections.unmodifiableList(fields);
}
}
private static class IntegerFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private IntegerFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Integer integerValue = (Integer) value;
List<IndexableField> fields = new ArrayList<>();
if (type.isSearchable()) {
fields.add(new IntPoint(name, integerValue));
}
fields.add(new StoredField(name, integerValue));
return Collections.unmodifiableList(fields);
}
}
private static class BooleanFieldFactory implements IndexableFieldFactory {
private final Indexed.Type type;
private BooleanFieldFactory(Indexed.Type type) {
this.type = type;
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Boolean booleanValue = (Boolean) value;
if (type.isSearchable()) {
return singleton(new StringField(name, booleanValue.toString(), Store.YES));
} else {
return singleton(new StoredField(name, booleanValue.toString()));
}
}
}
private static class InstantFieldFactory extends LongFieldFactory {
private InstantFieldFactory(Indexed.Type type) {
super(type);
}
@Override
public Iterable<IndexableField> create(String name, Object value) {
Instant instant = (Instant) value;
return super.create(name, instant.toEpochMilli());
}
}
}

109
scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java Normal file
View File

@@ -0,0 +1,109 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import java.io.IOException;
import static sonia.scm.search.FieldNames.*;
public class LuceneIndex implements Index {
private final DocumentConverter converter;
private final IndexWriter writer;
LuceneIndex(DocumentConverter converter, IndexWriter writer) {
this.converter = converter;
this.writer = writer;
}
@Override
public void store(Id id, String permission, Object object) {
String uid = createUid(id, object.getClass());
Document document = converter.convert(object);
try {
field(document, UID, uid);
field(document, ID, id.getValue());
id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository));
field(document, TYPE, object.getClass().getName());
if (!Strings.isNullOrEmpty(permission)) {
field(document, PERMISSION, permission);
}
writer.updateDocument(new Term(UID, uid), document);
} catch (IOException e) {
throw new SearchEngineException("failed to add document to index", e);
}
}
private String createUid(Id id, Class<?> type) {
return id.asString() + "/" + type.getName();
}
private void field(Document document, String type, String name) {
document.add(new StringField(type, name, Field.Store.YES));
}
@Override
public void delete(Id id, Class<?> type) {
try {
writer.deleteDocuments(new Term(UID, createUid(id, type)));
} catch (IOException e) {
throw new SearchEngineException("failed to delete document from index", e);
}
}
@Override
public void deleteByRepository(String repository) {
try {
writer.deleteDocuments(new Term(REPOSITORY, repository));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + repository + " from index", ex);
}
}
@Override
public void deleteByType(Class<?> type) {
try {
writer.deleteDocuments(new Term(TYPE, type.getName()));
} catch (IOException ex) {
throw new SearchEngineException("failed to delete documents by repository " + type + " from index", ex);
}
}
@Override
public void close() {
try {
writer.close();
} catch (IOException e) {
throw new SearchEngineException("failed to close index writer", e);
}
}
}

149
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java Normal file
View File

@@ -0,0 +1,149 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class LuceneQueryBuilder extends QueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class);
private static final Map<Class<?>, SearchableType> CACHE = new ConcurrentHashMap<>();
private final IndexOpener opener;
private final String indexName;
private final Analyzer analyzer;
LuceneQueryBuilder(IndexOpener opener, String indexName, Analyzer analyzer) {
this.opener = opener;
this.indexName = indexName;
this.analyzer = analyzer;
}
@Override
protected QueryResult execute(QueryParams queryParams) {
String queryString = Strings.nullToEmpty(queryParams.getQueryString());
SearchableType searchableType = CACHE.computeIfAbsent(queryParams.getType(), SearchableTypes::create);
Query query = Queries.filter(createQuery(searchableType, queryParams, queryString), queryParams);
if (LOG.isDebugEnabled()) {
LOG.debug("execute lucene query: {}", query);
}
try (IndexReader reader = opener.openForRead(indexName)) {
IndexSearcher searcher = new IndexSearcher(reader);
TopScoreDocCollector topScoreCollector = createTopScoreCollector(queryParams);
Collector collector = new PermissionAwareCollector(reader, topScoreCollector);
searcher.search(query, collector);
QueryResultFactory resultFactory = new QueryResultFactory(analyzer, searcher, searchableType, query);
return resultFactory.create(getTopDocs(queryParams, topScoreCollector));
} catch (IOException e) {
throw new SearchEngineException("failed to search index", e);
} catch (InvalidTokenOffsetsException e) {
throw new SearchEngineException("failed to highlight results", e);
}
}
@Nonnull
private TopScoreDocCollector createTopScoreCollector(QueryParams queryParams) {
return TopScoreDocCollector.create(queryParams.getStart() + queryParams.getLimit(), Integer.MAX_VALUE);
}
private TopDocs getTopDocs(QueryParams queryParams, TopScoreDocCollector topScoreCollector) {
return topScoreCollector.topDocs(queryParams.getStart(), queryParams.getLimit());
}
private Query createQuery(SearchableType searchableType, QueryParams queryParams, String queryString) {
try {
if (queryString.contains(":")) {
return createExpertQuery(searchableType, queryParams);
}
return createBestGuessQuery(searchableType, queryParams);
} catch (QueryNodeException ex) {
throw new QueryParseException(queryString, "failed to parse query", ex);
}
}
private Query createExpertQuery(SearchableType searchableType, QueryParams queryParams) throws QueryNodeException {
StandardQueryParser parser = new StandardQueryParser(analyzer);
parser.setPointsConfigMap(searchableType.getPointsConfig());
return parser.parse(queryParams.getQueryString(), "");
}
public Query createBestGuessQuery(SearchableType searchableType, QueryBuilder.QueryParams queryParams) {
String[] fieldNames = searchableType.getFieldNames();
if (fieldNames == null || fieldNames.length == 0) {
throw new NoDefaultQueryFieldsFoundException(searchableType.getType());
}
BooleanQuery.Builder builder = new BooleanQuery.Builder();
for (String fieldName : fieldNames) {
Term term = new Term(fieldName, appendWildcardIfNotAlreadyUsed(queryParams));
WildcardQuery query = new WildcardQuery(term);
Float boost = searchableType.getBoosts().get(fieldName);
if (boost != null) {
builder.add(new BoostQuery(query, boost), BooleanClause.Occur.SHOULD);
} else {
builder.add(query, BooleanClause.Occur.SHOULD);
}
}
return builder.build();
}
@Nonnull
private String appendWildcardIfNotAlreadyUsed(QueryParams queryParams) {
String queryString = queryParams.getQueryString().toLowerCase(Locale.ENGLISH);
if (!queryString.contains("?") && !queryString.contains("*")) {
queryString += "*";
}
return queryString;
}
}

44
scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java Normal file
View File

@@ -0,0 +1,44 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import javax.inject.Inject;
public class LuceneQueryBuilderFactory {
private final IndexOpener indexOpener;
private final AnalyzerFactory analyzerFactory;
@Inject
public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) {
this.indexOpener = indexOpener;
this.analyzerFactory = analyzerFactory;
}
public LuceneQueryBuilder create(String name, IndexOptions options) {
return new LuceneQueryBuilder(indexOpener, name, analyzerFactory.create(options));
}
}

57
scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java Normal file
View File

@@ -0,0 +1,57 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import javax.inject.Inject;
import java.io.IOException;
public class LuceneSearchEngine implements SearchEngine {
private final IndexOpener indexOpener;
private final DocumentConverter converter;
private final LuceneQueryBuilderFactory queryBuilderFactory;
@Inject
public LuceneSearchEngine(IndexOpener indexOpener, DocumentConverter converter, LuceneQueryBuilderFactory queryBuilderFactory) {
this.indexOpener = indexOpener;
this.converter = converter;
this.queryBuilderFactory = queryBuilderFactory;
}
@Override
public Index getOrCreate(String name, IndexOptions options) {
try {
return new LuceneIndex(converter, indexOpener.openForWrite(name, options));
} catch (IOException ex) {
throw new SearchEngineException("failed to open index", ex);
}
}
@Override
public QueryBuilder search(String name, IndexOptions options) {
return queryBuilderFactory.create(name, options);
}
}

31
scm-webapp/src/main/java/sonia/scm/search/NonReadableFieldException.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
public class NonReadableFieldException extends SearchEngineException {
public NonReadableFieldException(String message, Throwable cause) {
super(message, cause);
}
}

88
scm-webapp/src/main/java/sonia/scm/search/PermissionAwareCollector.java Normal file
View File

@@ -0,0 +1,88 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
import org.apache.shiro.SecurityUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
public class PermissionAwareCollector implements Collector {
private static final String FIELD_PERMISSION = "_permission";
private static final Set<String> FIELDS = Collections.singleton(FIELD_PERMISSION);
private final IndexReader reader;
private final Collector delegate;
public PermissionAwareCollector(IndexReader reader, Collector delegate) {
this.reader = reader;
this.delegate = delegate;
}
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
return new PermissionAwareLeafCollector(delegate.getLeafCollector(context), context.docBase);
}
@Override
public ScoreMode scoreMode() {
return delegate.scoreMode();
}
private class PermissionAwareLeafCollector implements LeafCollector {
private final LeafCollector delegate;
private final int docBase;
private PermissionAwareLeafCollector(LeafCollector delegate, int docBase) {
this.delegate = delegate;
this.docBase = docBase;
}
@Override
public void setScorer(Scorable scorer) throws IOException {
this.delegate.setScorer(scorer);
}
@Override
public void collect(int doc) throws IOException {
Document document = reader.document(docBase + doc, FIELDS);
String permission = document.get(FIELD_PERMISSION);
if (Strings.isNullOrEmpty(permission) || SecurityUtils.getSubject().isPermitted(permission)) {
this.delegate.collect(doc);
}
}
}
}

54
scm-webapp/src/main/java/sonia/scm/search/Queries.java Normal file
View File

@@ -0,0 +1,54 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import static org.apache.lucene.search.BooleanClause.Occur.MUST;
final class Queries {
private Queries() {
}
private static Query typeQuery(Class<?> type) {
return new TermQuery(new Term(FieldNames.TYPE, type.getName()));
}
private static Query repositoryQuery(String repositoryId) {
return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId));
}
static Query filter(Query query, QueryBuilder.QueryParams params) {
BooleanQuery.Builder builder = new BooleanQuery.Builder()
.add(query, MUST)
.add(typeQuery(params.getType()), MUST);
params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST));
return builder.build();
}
}

103
scm-webapp/src/main/java/sonia/scm/search/QueryResultFactory.java Normal file
View File

@@ -0,0 +1,103 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static java.util.Optional.empty;
import static java.util.Optional.of;
public class QueryResultFactory {
private final Analyzer analyzer;
private final Highlighter highlighter;
private final IndexSearcher searcher;
private final SearchableType searchableType;
public QueryResultFactory(Analyzer analyzer, IndexSearcher searcher, SearchableType searchableType, Query query) {
this.analyzer = analyzer;
this.searcher = searcher;
this.searchableType = searchableType;
this.highlighter = createHighlighter(query);
}
private Highlighter createHighlighter(Query query) {
return new Highlighter(
new SimpleHTMLFormatter("**", "**"),
new QueryScorer(query)
);
}
public QueryResult create(TopDocs topDocs) throws IOException, InvalidTokenOffsetsException {
List<Hit> hits = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
hits.add(createHit(scoreDoc));
}
return new QueryResult(topDocs.totalHits.value, searchableType.getType(), hits);
}
private Hit createHit(ScoreDoc scoreDoc) throws IOException, InvalidTokenOffsetsException {
Document document = searcher.doc(scoreDoc.doc);
Map<String, Hit.Field> fields = new HashMap<>();
for (SearchableField field : searchableType.getFields()) {
field(document, field).ifPresent(f -> fields.put(field.getName(), f));
}
return new Hit(document.get(FieldNames.ID), scoreDoc.score, fields);
}
private Optional<Hit.Field> field(Document document, SearchableField field) throws IOException, InvalidTokenOffsetsException {
Object value = field.value(document);
if (value != null) {
if (field.isHighlighted()) {
String[] fragments = createFragments(field, value.toString());
if (fragments.length > 0) {
return of(new Hit.HighlightedField(fragments));
}
}
return of(new Hit.ValueField(value));
}
return empty();
}
private String[] createFragments(SearchableField field, String value) throws InvalidTokenOffsetsException, IOException {
return highlighter.getBestFragments(analyzer, field.getName(), value, 5);
}
}

71
scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java Normal file
View File

@@ -0,0 +1,71 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import java.util.ArrayList;
import java.util.List;
public class QueuedIndex implements Index {
private final DefaultIndexQueue queue;
private final String indexName;
private final IndexOptions indexOptions;
private final List<IndexQueueTask> tasks = new ArrayList<>();
QueuedIndex(DefaultIndexQueue queue, String indexName, IndexOptions indexOptions) {
this.queue = queue;
this.indexName = indexName;
this.indexOptions = indexOptions;
}
@Override
public void store(Id id, String permission, Object object) {
tasks.add(index -> index.store(id, permission, object));
}
@Override
public void delete(Id id, Class<?> type) {
tasks.add(index -> index.delete(id, type));
}
@Override
public void deleteByRepository(String repository) {
tasks.add(index -> index.deleteByRepository(repository));
}
@Override
public void deleteByType(Class<?> type) {
tasks.add(index -> index.deleteByType(type));
}
@Override
public void close() {
IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper(
queue.getSearchEngine(), indexName, indexOptions, tasks
);
queue.enqueue(wrappedTask);
}
}

66
scm-webapp/src/main/java/sonia/scm/search/SearchableField.java Normal file
View File

@@ -0,0 +1,66 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import com.google.common.base.Strings;
import lombok.Getter;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
@Getter
class SearchableField {
private final String name;
private final Class<?> type;
private final ValueExtractor valueExtractor;
private final float boost;
private final boolean defaultQuery;
private final boolean highlighted;
private final PointsConfig pointsConfig;
SearchableField(Field field, Indexed indexed) {
this.name = name(field, indexed);
this.type = field.getType();
this.valueExtractor = ValueExtractors.create(name, type);
this.boost = indexed.boost();
this.defaultQuery = indexed.defaultQuery();
this.highlighted = indexed.highlighted();
this.pointsConfig = IndexableFields.pointConfig(field);
}
Object value(Document document) {
return valueExtractor.extract(document);
}
private String name(Field field, Indexed indexed) {
String nameFromAnnotation = indexed.name();
if (!Strings.isNullOrEmpty(nameFromAnnotation)) {
return nameFromAnnotation;
}
return field.getName();
}
}

51
scm-webapp/src/main/java/sonia/scm/search/SearchableType.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import lombok.Value;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Value
public class SearchableType {
Class<?> type;
String[] fieldNames;
Map<String,Float> boosts;
Map<String, PointsConfig> pointsConfig;
List<SearchableField> fields;
SearchableType(Class<?> type, String[] fieldNames, Map<String, Float> boosts, Map<String, PointsConfig> pointsConfig, List<SearchableField> fields) {
this.type = type;
this.fieldNames = fieldNames;
this.boosts = Collections.unmodifiableMap(boosts);
this.pointsConfig = Collections.unmodifiableMap(pointsConfig);
this.fields = Collections.unmodifiableList(fields);
}
}

82
scm-webapp/src/main/java/sonia/scm/search/SearchableTypes.java Normal file
View File

@@ -0,0 +1,82 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
final class SearchableTypes {
private static final float DEFAULT_BOOST = 1f;
private SearchableTypes() {
}
static SearchableType create(Class<?> type) {
List<SearchableField> fields = new ArrayList<>();
collectFields(type, fields);
return createSearchableType(type, fields);
}
private static SearchableType createSearchableType(Class<?> type, List<SearchableField> fields) {
String[] fieldsNames = fields.stream()
.filter(SearchableField::isDefaultQuery)
.map(SearchableField::getName)
.toArray(String[]::new);
Map<String, Float> boosts = new HashMap<>();
Map<String, PointsConfig> pointsConfig = new HashMap<>();
for (SearchableField field : fields) {
if (field.isDefaultQuery() && field.getBoost() != DEFAULT_BOOST) {
boosts.put(field.getName(), field.getBoost());
}
PointsConfig config = field.getPointsConfig();
if (config != null) {
pointsConfig.put(field.getName(), config);
}
}
return new SearchableType(type, fieldsNames, boosts, pointsConfig, fields);
}
private static void collectFields(Class<?> type, List<SearchableField> fields) {
Class<?> parent = type.getSuperclass();
if (parent != null) {
collectFields(parent, fields);
}
for (Field field : type.getDeclaredFields()) {
Indexed indexed = field.getAnnotation(Indexed.class);
if (indexed != null) {
fields.add(new SearchableField(field, indexed));
}
}
}
}

53
scm-webapp/src/main/java/sonia/scm/search/TypeCheck.java Normal file
View File

@@ -0,0 +1,53 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import java.time.Instant;
final class TypeCheck {
private TypeCheck() {
}
public static boolean isLong(Class<?> type) {
return type == Long.TYPE || type == Long.class;
}
public static boolean isInteger(Class<?> type) {
return type == Integer.TYPE || type == Integer.class;
}
public static boolean isBoolean(Class<?> type) {
return type == Boolean.TYPE || type == Boolean.class;
}
public static boolean isInstant(Class<?> type) {
return type == Instant.class;
}
public static boolean isString(Class<?> type) {
return type == String.class;
}
}

31
scm-webapp/src/main/java/sonia/scm/search/UnsupportedTypeOfFieldException.java Normal file
View File

@@ -0,0 +1,31 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
public class UnsupportedTypeOfFieldException extends SearchEngineException {
public UnsupportedTypeOfFieldException(Class<?> type, String field) {
super("type " + type + " of " + field + " is unsupported.");
}
}

32
scm-webapp/src/main/java/sonia/scm/search/ValueExtractor.java Normal file
View File

@@ -0,0 +1,32 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.document.Document;
@FunctionalInterface
public interface ValueExtractor {
Object extract(Document document);
}

95
scm-webapp/src/main/java/sonia/scm/search/ValueExtractors.java Normal file
View File

@@ -0,0 +1,95 @@
/*
* MIT License
*
* Copyright (c) 2020-present Cloudogu GmbH and Contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package sonia.scm.search;
import org.apache.lucene.index.IndexableField;
import javax.annotation.Nonnull;
import java.time.Instant;
final class ValueExtractors {
private ValueExtractors() {
}
static ValueExtractor create(String name, Class<?> type) {
if (TypeCheck.isString(type)) {
return stringExtractor(name);
} else if (TypeCheck.isLong(type)) {
return longExtractor(name);
} else if (TypeCheck.isInteger(type)) {
return integerExtractor(name);
} else if (TypeCheck.isBoolean(type)) {
return booleanExtractor(name);
} else if (TypeCheck.isInstant(type)) {
return instantExtractor(name);
} else {
throw new UnsupportedTypeOfFieldException(type, name);
}
}
@Nonnull
private static ValueExtractor stringExtractor(String name) {
return doc -> doc.get(name);
}
@Nonnull
private static ValueExtractor instantExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return Instant.ofEpochMilli(field.numericValue().longValue());
}
return null;
};
}
@Nonnull
private static ValueExtractor booleanExtractor(String name) {
return doc -> Boolean.parseBoolean(doc.get(name));
}
@Nonnull
private static ValueExtractor integerExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return field.numericValue().intValue();
}
return null;
};
}
@Nonnull
private static ValueExtractor longExtractor(String name) {
return doc -> {
IndexableField field = doc.getField(name);
if (field != null) {
return field.numericValue().longValue();
}
return null;
};
}
}