From 0a26741ebd63add2a2ec4e17b29ae62758804760 Mon Sep 17 00:00:00 2001 From: Sebastian Sdorra Date: Wed, 25 Aug 2021 15:40:11 +0200 Subject: [PATCH] One index per type and parallel indexing (#1781) Before this change the search uses a single index which distinguishes types (repositories, users, etc.) with a field (_type). But it has turned out that this could lead to problems, in particular if different types have the same field and uses different analyzers for those fields. The following links show even more problems of a combined index: https://www.elastic.co/blog/index-vs-type https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html With this change every type becomes its own index and the SearchEngine gets an api to modify multiple indices at once to remove all documents from all indices, which are related to a specific repository, for example. The search uses another new api to coordinate the indexing, the central work queue. The central work queue is able to coordinate long-running or resource intensive tasks. It is able to run tasks in parallel, but can also run tasks which targets the same resources in sequence. The queue is also persistent and can restore queued tasks after restart. Co-authored-by: Konstantin Schaper --- gradle/changelog/one_index_per_type.yaml | 8 + .../scm/search/HandlerEventIndexSyncer.java | 21 +- .../src/main/java/sonia/scm/search/Index.java | 64 +-- .../java/sonia/scm/search/IndexDetails.java | 47 ++ .../java/sonia/scm/search/IndexOptions.java | 3 +- .../main/java/sonia/scm/search/IndexTask.java | 51 ++ .../main/java/sonia/scm/search/Indexer.java | 66 +-- .../java/sonia/scm/search/SearchEngine.java | 116 ++++- .../scm/search/SerializableIndexTask.java | 40 ++ .../java/sonia/scm/work/CentralWorkQueue.java | 174 +++++++ .../scm/work/NonPersistableTaskException.java | 39 ++ .../src/main/java/sonia/scm/work/Task.java | 38 ++ .../java/sonia/scm/store/StoreConstants.java | 22 +- .../sonia/scm/store/InMemoryBlobStore.java | 4 +- .../java/sonia/scm/web/RestDispatcher.java | 1 - .../ui-webapp/src/containers/OmniSearch.tsx | 2 +- scm-ui/ui-webapp/src/search/RepositoryHit.tsx | 2 +- .../java/sonia/scm/group/GroupIndexer.java | 60 +-- .../lifecycle/modules/ScmServletModule.java | 4 + .../scm/repository/RepositoryIndexer.java | 71 +-- .../java/sonia/scm/search/FieldNames.java | 2 - .../scm/search/IndexBootstrapListener.java | 23 +- .../java/sonia/scm/search/IndexManager.java | 139 ++++++ .../java/sonia/scm/search/IndexParams.java | 16 +- .../java/sonia/scm/search/LuceneIndex.java | 100 ++-- .../sonia/scm/search/LuceneIndexDetails.java | 43 ++ .../sonia/scm/search/LuceneIndexFactory.java | 47 +- .../sonia/scm/search/LuceneIndexTask.java | 76 +++ ...per.java => LuceneInjectingIndexTask.java} | 30 +- .../sonia/scm/search/LuceneQueryBuilder.java | 8 +- .../scm/search/LuceneQueryBuilderFactory.java | 8 +- .../sonia/scm/search/LuceneSearchEngine.java | 103 +++- ...xQueue.java => LuceneSimpleIndexTask.java} | 52 +-- .../main/java/sonia/scm/search/Queries.java | 21 +- .../java/sonia/scm/search/QueuedIndex.java | 102 ---- .../sonia/scm/search/SharableIndexWriter.java | 90 ++++ .../java/sonia/scm/security/Impersonator.java | 128 +++++ .../index/RemoveCombinedIndex.java} | 57 ++- .../main/java/sonia/scm/user/UserIndexer.java | 59 ++- .../security/AdministrationContextMarker.java | 6 +- .../DefaultAdministrationContext.java | 197 +------- .../scm/work/DefaultCentralWorkQueue.java | 213 +++++++++ .../Finalizer.java} | 6 +- .../sonia/scm/work/InjectingUnitOfWork.java | 47 ++ .../main/java/sonia/scm/work/Persistence.java | 108 +++++ .../main/java/sonia/scm/work/Resource.java | 66 +++ .../java/sonia/scm/work/SimpleUnitOfWork.java | 48 ++ .../sonia/scm/work/ThreadCountProvider.java | 79 ++++ .../main/java/sonia/scm/work/UnitOfWork.java | 149 ++++++ .../sonia/scm/group/GroupIndexerTest.java | 93 ++-- .../scm/repository/RepositoryIndexerTest.java | 136 +++--- .../search/HandlerEventIndexSyncerTest.java | 33 +- .../search/IndexBootstrapListenerTest.java | 68 +-- .../sonia/scm/search/IndexManagerTest.java | 188 ++++++++ .../sonia/scm/search/IndexOpenerTest.java | 118 ----- .../java/sonia/scm/search/IndexQueueTest.java | 153 ------ .../scm/search/LuceneIndexFactoryTest.java | 87 ++++ .../sonia/scm/search/LuceneIndexTest.java | 234 +++++----- .../search/LuceneInjectingIndexTaskTest.java | 98 ++++ .../scm/search/LuceneQueryBuilderTest.java | 51 +- .../scm/search/LuceneSearchEngineTest.java | 439 ++++++++++++------ .../scm/search/LuceneSimpleIndexTaskTest.java | 83 ++++ .../scm/search/SharableIndexWriterTest.java | 249 ++++++++++ .../sonia/scm/security/ImpersonatorTest.java | 91 ++++ .../update/index/RemoveCombinedIndexTest.java | 91 ++++ .../java/sonia/scm/user/UserIndexerTest.java | 93 ++-- .../DefaultAdministrationContextTest.java | 3 +- .../scm/work/DefaultCentralWorkQueueTest.java | 375 +++++++++++++++ .../java/sonia/scm/work/PersistenceTest.java | 178 +++++++ .../java/sonia/scm/work/ResourceTest.java | 72 +++ .../sonia/scm/work/SimpleUnitOfWorkTest.java | 87 ++++ .../scm/work/ThreadCountProviderTest.java | 80 ++++ 72 files changed, 4536 insertions(+), 1420 deletions(-) create mode 100644 gradle/changelog/one_index_per_type.yaml create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexDetails.java create mode 100644 scm-core/src/main/java/sonia/scm/search/IndexTask.java create mode 100644 scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java create mode 100644 scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java create mode 100644 scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java create mode 100644 scm-core/src/main/java/sonia/scm/work/Task.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/IndexManager.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java rename scm-webapp/src/main/java/sonia/scm/search/{IndexQueueTaskWrapper.java => LuceneInjectingIndexTask.java} (58%) rename scm-webapp/src/main/java/sonia/scm/search/{IndexQueue.java => LuceneSimpleIndexTask.java} (51%) delete mode 100644 scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java create mode 100644 scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java create mode 100644 scm-webapp/src/main/java/sonia/scm/security/Impersonator.java rename scm-webapp/src/main/java/sonia/scm/{search/IndexOpener.java => update/index/RemoveCombinedIndex.java} (51%) create mode 100644 scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java rename scm-webapp/src/main/java/sonia/scm/{search/IndexQueueTask.java => work/Finalizer.java} (92%) create mode 100644 scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java create mode 100644 scm-webapp/src/main/java/sonia/scm/work/Persistence.java create mode 100644 scm-webapp/src/main/java/sonia/scm/work/Resource.java create mode 100644 scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java create mode 100644 scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java create mode 100644 scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java delete mode 100644 scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java delete mode 100644 scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java create mode 100644 scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java diff --git a/gradle/changelog/one_index_per_type.yaml b/gradle/changelog/one_index_per_type.yaml new file mode 100644 index 0000000000..69a05dbc4e --- /dev/null +++ b/gradle/changelog/one_index_per_type.yaml @@ -0,0 +1,8 @@ +- type: Added + description: Central Work Queue for coordinating long-running tasks ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Changed + description: One index per type instead of one index for all types ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Added + description: Api to modify mutliple indices at once ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Changed + description: Use central work queue for all indexing tasks ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) diff --git a/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java b/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java index 4832e3d448..136d29138e 100644 --- a/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java +++ b/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java @@ -24,6 +24,7 @@ package sonia.scm.search; +import com.google.common.annotations.Beta; import sonia.scm.HandlerEventType; import sonia.scm.event.HandlerEvent; @@ -33,11 +34,14 @@ import sonia.scm.event.HandlerEvent; * @param type of indexed item * @since 2.22.0 */ +@Beta public final class HandlerEventIndexSyncer { + private final SearchEngine searchEngine; private final Indexer indexer; - public HandlerEventIndexSyncer(Indexer indexer) { + public HandlerEventIndexSyncer(SearchEngine searchEngine, Indexer indexer) { + this.searchEngine = searchEngine; this.indexer = indexer; } @@ -49,17 +53,16 @@ public final class HandlerEventIndexSyncer { public void handleEvent(HandlerEvent event) { HandlerEventType type = event.getEventType(); if (type.isPost()) { - updateIndex(type, event.getItem()); + SerializableIndexTask task = createTask(type, event.getItem()); + searchEngine.forType(indexer.getType()).update(task); } } - private void updateIndex(HandlerEventType type, T item) { - try (Indexer.Updater updater = indexer.open()) { - if (type == HandlerEventType.DELETE) { - updater.delete(item); - } else { - updater.store(item); - } + private SerializableIndexTask createTask(HandlerEventType type, T item) { + if (type == HandlerEventType.DELETE) { + return indexer.createDeleteTask(item); + } else { + return indexer.createStoreTask(item); } } diff --git a/scm-core/src/main/java/sonia/scm/search/Index.java b/scm-core/src/main/java/sonia/scm/search/Index.java index 5e23016054..3a00fc50b0 100644 --- a/scm-core/src/main/java/sonia/scm/search/Index.java +++ b/scm-core/src/main/java/sonia/scm/search/Index.java @@ -25,6 +25,7 @@ package sonia.scm.search; import com.google.common.annotations.Beta; +import sonia.scm.repository.Repository; /** * Can be used to index objects for full text searches. @@ -32,7 +33,15 @@ import com.google.common.annotations.Beta; * @since 2.21.0 */ @Beta -public interface Index extends AutoCloseable { +public interface Index { + + /** + * Returns details such as name and type of index. + * + * @return details of index + * @since 2.23.0 + */ + IndexDetails getDetails(); /** * Store the given object in the index. @@ -53,12 +62,6 @@ public interface Index extends AutoCloseable { */ Deleter delete(); - /** - * Close index and commit changes. - */ - @Override - void close(); - /** * Deleter provides an api to delete object from index. * @@ -66,27 +69,6 @@ public interface Index extends AutoCloseable { */ interface Deleter { - /** - * Returns an api which allows deletion of objects from the type of this index. - * @return type restricted delete api - */ - ByTypeDeleter byType(); - - /** - * Returns an api which allows deletion of objects of every type. - * @return unrestricted delete api for all types. - */ - AllTypesDeleter allTypes(); - } - - /** - * Delete api for the type of the index. This means, that only entries for this - * type will be deleted. - * - * @since 2.23.0 - */ - interface ByTypeDeleter { - /** * Delete the object with the given id and type from index. * @param id id of object @@ -104,27 +86,15 @@ public interface Index extends AutoCloseable { * @param repositoryId id of repository */ void byRepository(String repositoryId); - } - - /** - * Delete api for the overall index regarding all types. - * - * @since 2.23.0 - */ - interface AllTypesDeleter { /** - * Delete all objects which are related to the given repository from index regardless of their type. - * @param repositoryId repository id + * Delete all objects which are related the given type and repository from index. + * + * @param repository repository */ - void byRepository(String repositoryId); - - /** - * Delete all objects with the given type from index. - * This method is mostly useful if the index type has changed and the old type (in form of a class) - * is no longer available. - * @param typeName type name of objects - */ - void byTypeName(String typeName); + default void byRepository(Repository repository) { + byRepository(repository.getId()); + } } + } diff --git a/scm-core/src/main/java/sonia/scm/search/IndexDetails.java b/scm-core/src/main/java/sonia/scm/search/IndexDetails.java new file mode 100644 index 0000000000..23db92a53c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexDetails.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Details of an index. + * @since 2.23.0 + */ +@Beta +public interface IndexDetails { + + /** + * Returns type of objects which are indexed. + * @return type of objects + */ + Class getType(); + + /** + * Returns the name of the index (e.g. `default`) + * @return name + */ + String getName(); +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java index 4b99655206..21c9cb931a 100644 --- a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java +++ b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java @@ -27,6 +27,7 @@ package sonia.scm.search; import com.google.common.annotations.Beta; import lombok.EqualsAndHashCode; +import java.io.Serializable; import java.util.Locale; /** @@ -36,7 +37,7 @@ import java.util.Locale; */ @Beta @EqualsAndHashCode -public class IndexOptions { +public class IndexOptions implements Serializable { private final Type type; private final Locale locale; diff --git a/scm-core/src/main/java/sonia/scm/search/IndexTask.java b/scm-core/src/main/java/sonia/scm/search/IndexTask.java new file mode 100644 index 0000000000..11b5d9bfab --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexTask.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * A task which updates an index. + * @param type of indexed objects + * @since 2.23.0 + */ +@Beta +@FunctionalInterface +public interface IndexTask { + + /** + * Execute operations on the index. + * @param index index to update + */ + void update(Index index); + + /** + * This method is called after work is committed to the index. + */ + default void afterUpdate() { +// Do nothing + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Indexer.java b/scm-core/src/main/java/sonia/scm/search/Indexer.java index 13c8935927..7e6967d78a 100644 --- a/scm-core/src/main/java/sonia/scm/search/Indexer.java +++ b/scm-core/src/main/java/sonia/scm/search/Indexer.java @@ -24,6 +24,7 @@ package sonia.scm.search; +import com.google.common.annotations.Beta; import sonia.scm.plugin.ExtensionPoint; /** @@ -35,6 +36,7 @@ import sonia.scm.plugin.ExtensionPoint; * @since 2.22.0 * @see HandlerEventIndexSyncer */ +@Beta @ExtensionPoint public interface Indexer { @@ -54,42 +56,48 @@ public interface Indexer { int getVersion(); /** - * Opens the index and return an updater for the given type. - * - * @return updater with open index + * Returns task which re index all items. + * @return task to re index all + * @since 2.23.0 */ - Updater open(); + Class> getReIndexAllTask(); /** - * Updater for index. - * - * @param type to index + * Creates a task which stores the given item in the index. + * @param item item to store + * @return task which stores the item + * @since 2.23.0 */ - interface Updater extends AutoCloseable { + SerializableIndexTask createStoreTask(T item); - /** - * Stores the given item in the index. - * - * @param item item to index - */ - void store(T item); + /** + * Creates a task which deletes the given item from index. + * @param item item to delete + * @return task which deletes the item + * @since 2.23.0 + */ + SerializableIndexTask createDeleteTask(T item); - /** - * Delete the given item from the index - * - * @param item item to delete - */ - void delete(T item); + /** + * Abstract class which builds the foundation for tasks which re-index all items. + * + * @since 2.23.0 + */ + abstract class ReIndexAllTask implements IndexTask { - /** - * Re index all existing items. - */ - void reIndexAll(); + private final IndexLogStore logStore; + private final Class type; + private final int version; - /** - * Close the given index. - */ - void close(); + protected ReIndexAllTask(IndexLogStore logStore, Class type, int version) { + this.logStore = logStore; + this.type = type; + this.version = version; + } + + @Override + public void afterUpdate() { + logStore.defaultIndex().log(type, version); + } } - } diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java index c95c79ef37..1d81422064 100644 --- a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java @@ -25,8 +25,10 @@ package sonia.scm.search; import com.google.common.annotations.Beta; +import sonia.scm.ModelObject; import java.util.Collection; +import java.util.function.Predicate; /** * The {@link SearchEngine} is the main entry point for indexing and searching. @@ -61,6 +63,80 @@ public interface SearchEngine { */ ForType forType(String name); + /** + * Returns an api which allows the modification of multiple indices at once. + * @return api to modify multiple indices + * @since 2.23.0 + */ + ForIndices forIndices(); + + /** + * Api for modifying multiple indices at once. + * @since 2.23.0 + */ + interface ForIndices { + + /** + * This method can be used to filter the indices. + * If no predicate is used the tasks are enqueued for every existing index. + * + * @param predicate predicate to filter indices + * @return {@code this} + */ + ForIndices matching(Predicate predicate); + + /** + * Apply a lock for a specific resource. By default, a lock for the whole index is used. + * If one or more specific resources are locked, than the lock is applied only for those resources + * and tasks which targets other resources of the same index can run in parallel. + * + * @param resource specific resource to lock + * @return {@code this} + */ + ForIndices forResource(String resource); + + /** + * This method is a shortcut for {@link #forResource(String)} with the id of the given resource. + * + * @param resource resource in form of model object + * @return {@code this} + */ + default ForIndices forResource(ModelObject resource) { + return forResource(resource.getId()); + } + + /** + * Specify options for the index. + * If not used the default options will be used. + * @param options index options + * @return {@code this} + * @see IndexOptions#defaults() + */ + ForIndices withOptions(IndexOptions options); + + /** + * Submits the task and execute it for every index + * which are matching the predicate ({@link #matching(Predicate)}. + * The task is executed asynchronous and will be finished some time in the future. + * Note: the task must be serializable because it is submitted to the + * {@link sonia.scm.work.CentralWorkQueue}. + * For more information on task serialization have a look at the + * {@link sonia.scm.work.CentralWorkQueue} documentation. + * + * @param task serializable task for updating multiple indices + */ + void batch(SerializableIndexTask task); + + /** + * Submits the task and executes it for every index + * which are matching the predicate ({@link #matching(Predicate)}. + * The task is executed asynchronously and will finish at some unknown point in the future. + * + * @param task task for updating multiple indices + */ + void batch(Class> task); + } + /** * Search and index api. * @@ -87,10 +163,44 @@ public interface SearchEngine { ForType withIndex(String name); /** - * Returns an index object which provides method to update the search index. - * @return index object + * Apply a lock for a specific resource. By default, a lock for the whole index is used. + * If one or more specific resources are locked, then the lock is applied only for those resources + * and tasks which target other resources of the same index can run in parallel. + * + * @param resource specific resource to lock + * @return {@code this} */ - Index getOrCreate(); + ForType forResource(String resource); + + /** + * This method is a shortcut for {@link #forResource(String)} with the id of the given resource. + * + * @param resource resource in form of model object + * @return {@code this} + */ + default ForType forResource(ModelObject resource) { + return forResource(resource.getId()); + } + + /** + * Submits a task to update the index. + * The task is executed asynchronously and will finish at some unknown point in the future. + * Note: the task must be serializable because it is submitted to the + * {@link sonia.scm.work.CentralWorkQueue}, + * for more information about the task serialization have a look at the + * {@link sonia.scm.work.CentralWorkQueue} documentation. + * + * @param task serializable task for updating the index + */ + void update(SerializableIndexTask task); + + /** + * Submits a task to update the index. + * The task is executed asynchronous and will be finished some time in the future. + * + * @param task task for updating multiple indices + */ + void update(Class> task); /** * Returns a query builder object which can be used to search the index. diff --git a/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java b/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java new file mode 100644 index 0000000000..754d056a0c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java @@ -0,0 +1,40 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.io.Serializable; + +/** + * A serializable version of {@link IndexTask}. + * + * @param type of indexed objects + * @since 2.23.0 + */ +@Beta +@FunctionalInterface +public interface SerializableIndexTask extends IndexTask, Serializable { +} diff --git a/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java b/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java new file mode 100644 index 0000000000..c526fd642e --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java @@ -0,0 +1,174 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; +import sonia.scm.ModelObject; + +import javax.annotation.Nullable; + +/** + * The {@link CentralWorkQueue} provides an api to submit and coordinate long-running or resource intensive tasks. + * + * The tasks are executed in parallel, but if some tasks access the same resource this can become a problem. + * To avoid this problem a task can be enqueued with a lock e.g.: + * + * 
{@code
+ *   queue.append().locks("my-resources").enqueue(MyTask.class)
+ * }
+ * + * No tasks with the same lock will run in parallel. The example above locks a whole group of resources. + * It is possible to assign the lock to a more specific resource by adding the id parameter to the lock e.g.: + * + * 
{@code
+ *   queue.append().locks("my-resources", "42").enqueue(MyTask.class)
+ * }
+ * + * This will ensure, that no task with a lock for the my-resource 42 or for my-resources will run at the + * same time as the task which is enqueued in the example above. + * But this will also allow a task for my-resources with an id other than 42 can run in parallel. + * + * All tasks are executed with the permissions of the user which enqueues the task. + * If the task should run as admin, the {@link Enqueue#runAsAdmin()} method can be used. + * + * Tasks which could not be finished, + * before a restart of shutdown of the server, will be restored and executed on startup. + * In order to achieve the persistence of tasks, + * the enqueued task must be provided as a class or it must be serializable. + * This could become unhandy if the task has parameters and dependencies which must be injected. + * The injected objects should not be serialized with the task. + * In order to avoid this, dependencies should be declared as {@code transient} + * and injected via setters instead of the constructor parameters e.g.: + * + * 

+ *   public class MyTask implements Task {
+ *
+ *     private final Repository repository;
+ *     private transient RepositoryServiceFactory repositoryServiceFactory;
+ *
+ *     public MyTask(Repository repository) {
+ *       this.repository = repository;
+ *     }
+ *
+ *     {@code @}Inject
+ *     public void setRepositoryServiceFactory(RepositoryServiceFactory repositoryServiceFactory) {
+ *       this.repositoryServiceFactory = repositoryServiceFactory;
+ *     }
+ *
+ *     {@code @}Override
+ *     public void run() {
+ *       // do something with the repository and the repositoryServiceFactory
+ *     }
+ *
+ *   }
+ *
+ * + * The {@link CentralWorkQueue} will inject the requested members before the {@code run} method of the task is executed. + * + * @since 2.23.0 + */ +@Beta +public interface CentralWorkQueue { + + /** + * Append a new task to the central work queue. + * The method will return a builder interface to configure how the task will be enqueued. + * + * @return builder api for enqueue a task + */ + Enqueue append(); + + /** + * Returns the count of pending or running tasks. + * + * @return count of pending or running tasks + */ + int getSize(); + + /** + * Builder interface for the enqueueing of a new task. + */ + interface Enqueue { + + /** + * Configure a lock for the given resource type. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @return {@code this} + * @see CentralWorkQueue + */ + Enqueue locks(String resourceType); + + /** + * Configure a lock for the resource with the given type and id. + * Note if the id is {@code null} the whole resource type is locked. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @param id id of resource to lock + * @return {@code this} + * @see CentralWorkQueue + */ + Enqueue locks(String resourceType, @Nullable String id); + + /** + * Configure a lock for the resource with the given type and the id from the given {@link ModelObject}. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @param object which holds the id of the resource to lock + * @return {@code this} + * @see CentralWorkQueue + */ + default Enqueue locks(String resourceType, ModelObject object) { + return locks(resourceType, object.getId()); + } + + /** + * Run the enqueued task with administrator permission. + * + * @return {@code this} + */ + Enqueue runAsAdmin(); + + /** + * Enqueue the given task to {@link CentralWorkQueue}. + * Warning: Ensure that the task is serializable. + * If the task is not serializable an {@link NonPersistableTaskException} will be thrown. + * For more information about the persistence of tasks see class documentation. + * + * @param task serializable task to enqueue + * @see CentralWorkQueue + */ + void enqueue(Task task); + + /** + * Enqueue the given task to {@link CentralWorkQueue}. + * @param task task to enqueue + */ + void enqueue(Class task); + } +} diff --git a/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java b/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java new file mode 100644 index 0000000000..6014171acc --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java @@ -0,0 +1,39 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; + +/** + * Exception thrown when a task is enqueued to the {@link CentralWorkQueue} which cannot persisted. + * + * @since 2.23.0 + */ +@Beta +public final class NonPersistableTaskException extends RuntimeException { + public NonPersistableTaskException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-core/src/main/java/sonia/scm/work/Task.java b/scm-core/src/main/java/sonia/scm/work/Task.java new file mode 100644 index 0000000000..962ae97dae --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/Task.java @@ -0,0 +1,38 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; + +import java.io.Serializable; + +/** + * Serializable task which can be enqueued to the {@link CentralWorkQueue}. + * + * @since 2.23.0 + */ +@Beta +public interface Task extends Runnable, Serializable { +} diff --git a/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java b/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java index aa441e64a4..759e9b98bc 100644 --- a/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java +++ b/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java @@ -21,12 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.store; /** * Store constants for xml implementations. - * + * * @author Sebastian Sdorra */ public class StoreConstants @@ -36,6 +36,24 @@ public class StoreConstants public static final String CONFIG_DIRECTORY_NAME = "config"; + /** + * Name of the parent of data or blob directories. + * @since 2.23.0 + */ + public static final String VARIABLE_DATA_DIRECTORY_NAME = "var"; + + /** + * Name of data directories. + * @since 2.23.0 + */ + public static final String DATA_DIRECTORY_NAME = "data"; + + /** + * Name of blob directories. + * @since 2.23.0 + */ + public static final String BLOG_DIRECTORY_NAME = "data"; + public static final String REPOSITORY_METADATA = "metadata"; public static final String FILE_EXTENSION = ".xml"; diff --git a/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java b/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java index e95742b6ac..01d00d76a8 100644 --- a/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java +++ b/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java @@ -24,6 +24,8 @@ package sonia.scm.store; +import com.google.common.collect.ImmutableList; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -59,7 +61,7 @@ public class InMemoryBlobStore implements BlobStore { @Override public List getAll() { - return blobs; + return ImmutableList.copyOf(blobs); } @Override diff --git a/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java b/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java index a0c3253f72..8170da9c4a 100644 --- a/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java +++ b/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java @@ -28,7 +28,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.shiro.authz.AuthorizationException; import org.apache.shiro.authz.UnauthorizedException; import org.jboss.resteasy.mock.MockDispatcherFactory; -import org.jboss.resteasy.mock.MockHttpRequest; import org.jboss.resteasy.spi.Dispatcher; import org.jboss.resteasy.spi.HttpRequest; import org.jboss.resteasy.spi.HttpResponse; diff --git a/scm-ui/ui-webapp/src/containers/OmniSearch.tsx b/scm-ui/ui-webapp/src/containers/OmniSearch.tsx index 0972ce97d3..0dedf8f18d 100644 --- a/scm-ui/ui-webapp/src/containers/OmniSearch.tsx +++ b/scm-ui/ui-webapp/src/containers/OmniSearch.tsx @@ -99,7 +99,7 @@ const AvatarSection: FC = ({ hit }) => { const name = useStringHitFieldValue(hit, "name"); const type = useStringHitFieldValue(hit, "type"); - const repository = hit._embedded.repository; + const repository = hit._embedded?.repository; if (!namespace || !name || !type || !repository) { return null; } diff --git a/scm-ui/ui-webapp/src/search/RepositoryHit.tsx b/scm-ui/ui-webapp/src/search/RepositoryHit.tsx index 8d30a36ed6..a4eb491f6d 100644 --- a/scm-ui/ui-webapp/src/search/RepositoryHit.tsx +++ b/scm-ui/ui-webapp/src/search/RepositoryHit.tsx @@ -44,7 +44,7 @@ const RepositoryHit: FC = ({ hit }) => { // the embedded repository is only a subset of the repository (RepositoryCoordinates), // so we should use the fields to get more information - const repository = hit._embedded.repository; + const repository = hit._embedded?.repository; if (!namespace || !name || !type || !repository) { return null; } diff --git a/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java b/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java index 38dcd78157..103b1d5638 100644 --- a/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java @@ -30,8 +30,10 @@ import sonia.scm.plugin.Extension; import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class GroupIndexer implements Indexer { @VisibleForTesting static final int VERSION = 1; - private final GroupManager groupManager; private final SearchEngine searchEngine; @Inject - public GroupIndexer(GroupManager groupManager, SearchEngine searchEngine) { - this.groupManager = groupManager; + public GroupIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,47 +62,47 @@ public class GroupIndexer implements Indexer { return VERSION; } - @Subscribe(async = false) - public void handleEvent(GroupEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; } @Override - public Updater open() { - return new GroupIndexUpdater(groupManager, searchEngine.forType(Group.class).getOrCreate()); + public SerializableIndexTask createStoreTask(Group group) { + return index -> store(index, group); } - public static class GroupIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(Group group) { + return index -> index.delete().byId(Id.of(group)); + } + + @Subscribe(async = false) + public void handleEvent(GroupEvent event) { + new HandlerEventIndexSyncer<>(searchEngine, this).handleEvent(event); + } + + public static void store(Index index, Group group) { + index.store(Id.of(group), GroupPermissions.read(group).asShiroString(), group); + } + + public static class ReIndexAll extends ReIndexAllTask { private final GroupManager groupManager; - private final Index index; - private GroupIndexUpdater(GroupManager groupManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, GroupManager groupManager) { + super(logStore, Group.class, VERSION); this.groupManager = groupManager; - this.index = index; } @Override - public void store(Group group) { - index.store(Id.of(group), GroupPermissions.read(group).asShiroString(), group); - } - - @Override - public void delete(Group group) { - index.delete().byType().byId(Id.of(group)); - } - - @Override - public void reIndexAll() { - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (Group group : groupManager.getAll()) { - store(group); + store(index, group); } } - - @Override - public void close() { - index.close(); - } } + } diff --git a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java index b9b82e8289..fa52c95594 100644 --- a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java +++ b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java @@ -134,6 +134,8 @@ import sonia.scm.web.cgi.DefaultCGIExecutorFactory; import sonia.scm.web.filter.LoggingFilter; import sonia.scm.web.security.AdministrationContext; import sonia.scm.web.security.DefaultAdministrationContext; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.DefaultCentralWorkQueue; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; @@ -290,6 +292,8 @@ class ScmServletModule extends ServletModule { bind(SearchEngine.class, LuceneSearchEngine.class); bind(IndexLogStore.class, DefaultIndexLogStore.class); + bind(CentralWorkQueue.class, DefaultCentralWorkQueue.class); + bind(ContentTypeResolver.class).to(DefaultContentTypeResolver.class); } diff --git a/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java b/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java index 1eadd222f3..9016d72b1d 100644 --- a/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java @@ -26,12 +26,14 @@ package sonia.scm.repository; import com.github.legman.Subscribe; import com.google.common.annotations.VisibleForTesting; +import sonia.scm.HandlerEventType; import sonia.scm.plugin.Extension; -import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class RepositoryIndexer implements Indexer { @VisibleForTesting static final int VERSION = 3; - private final RepositoryManager repositoryManager; private final SearchEngine searchEngine; @Inject - public RepositoryIndexer(RepositoryManager repositoryManager, SearchEngine searchEngine) { - this.repositoryManager = repositoryManager; + public RepositoryIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,49 +62,58 @@ public class RepositoryIndexer implements Indexer { return Repository.class; } + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; + } + @Subscribe(async = false) public void handleEvent(RepositoryEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + HandlerEventType type = event.getEventType(); + if (type.isPost()) { + Repository repository = event.getItem(); + if (type == HandlerEventType.DELETE) { + searchEngine.forIndices() + .forResource(repository) + .batch(createDeleteTask(repository)); + } else { + searchEngine.forType(Repository.class) + .update(createStoreTask(repository)); + } + } } @Override - public Updater open() { - return new RepositoryIndexUpdater(repositoryManager, searchEngine.forType(getType()).getOrCreate()); + public SerializableIndexTask createStoreTask(Repository repository) { + return index -> store(index, repository); } - public static class RepositoryIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(Repository repository) { + return index -> index.delete().byRepository(repository); + } + + private static void store(Index index, Repository repository) { + index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); + } + + public static class ReIndexAll extends ReIndexAllTask { private final RepositoryManager repositoryManager; - private final Index index; - public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, RepositoryManager repositoryManager) { + super(logStore, Repository.class, VERSION); this.repositoryManager = repositoryManager; - this.index = index; } @Override - public void store(Repository repository) { - index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); - } - - @Override - public void delete(Repository repository) { - index.delete().allTypes().byRepository(repository.getId()); - } - - @Override - public void reIndexAll() { - // v1 used the whole classname as type - index.delete().allTypes().byTypeName(Repository.class.getName()); - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (Repository repository : repositoryManager.getAll()) { - store(repository); + store(index, repository); } } - - @Override - public void close() { - index.close(); - } } + } diff --git a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java index 9c11baa34c..7681eda584 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java +++ b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java @@ -27,9 +27,7 @@ package sonia.scm.search; final class FieldNames { private FieldNames(){} - static final String UID = "_uid"; static final String ID = "_id"; - static final String TYPE = "_type"; static final String REPOSITORY = "_repository"; static final String PERMISSION = "_permission"; } diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java b/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java index 752fe9619e..923da1ff44 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java @@ -27,7 +27,6 @@ package sonia.scm.search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.plugin.Extension; -import sonia.scm.web.security.AdministrationContext; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,13 +42,13 @@ public class IndexBootstrapListener implements ServletContextListener { private static final Logger LOG = LoggerFactory.getLogger(IndexBootstrapListener.class); - private final AdministrationContext administrationContext; + private final SearchEngine searchEngine; private final IndexLogStore indexLogStore; private final Set indexers; @Inject - public IndexBootstrapListener(AdministrationContext administrationContext, IndexLogStore indexLogStore, Set indexers) { - this.administrationContext = administrationContext; + public IndexBootstrapListener(SearchEngine searchEngine, IndexLogStore indexLogStore, Set indexers) { + this.searchEngine = searchEngine; this.indexLogStore = indexLogStore; this.indexers = indexers; } @@ -65,8 +64,11 @@ public class IndexBootstrapListener implements ServletContextListener { Optional indexLog = indexLogStore.defaultIndex().get(indexer.getType()); if (indexLog.isPresent()) { int version = indexLog.get().getVersion(); - if (version < indexer.getVersion()) { - LOG.debug("index version {} is older then {}, start reindexing of all {}", version, indexer.getVersion(), indexer.getType()); + if (version != indexer.getVersion()) { + LOG.debug( + "index version {} is older then {}, start reindexing of all {}", + version, indexer.getVersion(), indexer.getType() + ); indexAll(indexer); } } else { @@ -75,14 +77,9 @@ public class IndexBootstrapListener implements ServletContextListener { } } + @SuppressWarnings("unchecked") private void indexAll(Indexer indexer) { - administrationContext.runAsAdmin(() -> { - try (Indexer.Updater updater = indexer.open()) { - updater.reIndexAll(); - } - }); - - indexLogStore.defaultIndex().log(indexer.getType(), indexer.getVersion()); + searchEngine.forType(indexer.getType()).update(indexer.getReIndexAllTask()); } @Override diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java b/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java new file mode 100644 index 0000000000..9a6b3ca68a --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java @@ -0,0 +1,139 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Data; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.FSDirectory; +import sonia.scm.SCMContextProvider; +import sonia.scm.plugin.PluginLoader; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.xml.bind.JAXB; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@Singleton +public class IndexManager { + + private final Path directory; + private final AnalyzerFactory analyzerFactory; + private final IndexXml indexXml; + + @Inject + public IndexManager(SCMContextProvider context, PluginLoader pluginLoader, AnalyzerFactory analyzerFactory) { + directory = context.resolve(Paths.get("index")); + this.analyzerFactory = analyzerFactory; + this.indexXml = readIndexXml(pluginLoader.getUberClassLoader()); + } + + private IndexXml readIndexXml(ClassLoader uberClassLoader) { + Path path = directory.resolve("index.xml"); + if (Files.exists(path)) { + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(uberClassLoader); + return JAXB.unmarshal(path.toFile(), IndexXml.class); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } + } + return new IndexXml(); + } + + public Collection all() { + return Collections.unmodifiableSet(indexXml.indices); + } + + public IndexReader openForRead(LuceneSearchableType type, String indexName) throws IOException { + Path path = resolveIndexDirectory(type, indexName); + return DirectoryReader.open(FSDirectory.open(path)); + } + + public IndexWriter openForWrite(IndexParams indexParams) { + IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + + Path path = resolveIndexDirectory(indexParams); + if (!Files.exists(path)) { + store(new LuceneIndexDetails(indexParams.getType(), indexParams.getIndex())); + } + + try { + return new IndexWriter(FSDirectory.open(path), config); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index at " + path, ex); + } + } + + private Path resolveIndexDirectory(IndexParams indexParams) { + return directory.resolve(indexParams.getSearchableType().getName()).resolve(indexParams.getIndex()); + } + + private Path resolveIndexDirectory(LuceneSearchableType searchableType, String indexName) { + return directory.resolve(searchableType.getName()).resolve(indexName); + } + + private synchronized void store(LuceneIndexDetails details) { + if (!indexXml.getIndices().add(details)) { + return; + } + + if (!Files.exists(directory)) { + try { + Files.createDirectory(directory); + } catch (IOException e) { + throw new SearchEngineException("failed to create index directory", e); + } + } + + Path path = directory.resolve("index.xml"); + JAXB.marshal(indexXml, path.toFile()); + } + + @Data + @XmlRootElement(name = "indices") + @XmlAccessorType(XmlAccessType.FIELD) + public static class IndexXml { + + @XmlElement(name = "index") + private Set indices = new HashSet<>(); + + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java b/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java index 2fca1540e3..14f456ab3e 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java @@ -27,10 +27,24 @@ package sonia.scm.search; import lombok.Value; @Value -public class IndexParams { +public class IndexParams implements IndexDetails { String index; LuceneSearchableType searchableType; IndexOptions options; + @Override + public Class getType() { + return searchableType.getType(); + } + + @Override + public String getName() { + return index; + } + + @Override + public String toString() { + return searchableType.getName() + "/" + index; + } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java index 1612dc3fd6..4d201b38af 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java @@ -24,54 +24,71 @@ package sonia.scm.search; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.TermQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; +import java.util.function.Supplier; import static sonia.scm.search.FieldNames.ID; import static sonia.scm.search.FieldNames.PERMISSION; import static sonia.scm.search.FieldNames.REPOSITORY; -import static sonia.scm.search.FieldNames.TYPE; -import static sonia.scm.search.FieldNames.UID; -class LuceneIndex implements Index { +class LuceneIndex implements Index, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(LuceneIndex.class); + + private final IndexDetails details; private final LuceneSearchableType searchableType; - private final IndexWriter writer; + private final SharableIndexWriter writer; - LuceneIndex(LuceneSearchableType searchableType, IndexWriter writer) { - this.searchableType = searchableType; - this.writer = writer; + LuceneIndex(IndexParams params, Supplier writerFactory) { + this.details = params; + this.searchableType = params.getSearchableType(); + this.writer = new SharableIndexWriter(writerFactory); + this.open(); + } + + void open() { + writer.open(); + } + + @VisibleForTesting + SharableIndexWriter getWriter() { + return writer; + } + + @Override + public IndexDetails getDetails() { + return details; } @Override public void store(Id id, String permission, Object object) { - String uid = createUid(id, searchableType); Document document = searchableType.getTypeConverter().convert(object); try { - field(document, UID, uid); - field(document, ID, id.getValue()); + field(document, ID, id.asString()); id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository)); - field(document, TYPE, searchableType.getName()); if (!Strings.isNullOrEmpty(permission)) { field(document, PERMISSION, permission); } - writer.updateDocument(new Term(UID, uid), document); + writer.updateDocument(idTerm(id), document); } catch (IOException e) { throw new SearchEngineException("failed to add document to index", e); } } - private String createUid(Id id, LuceneSearchableType type) { - return id.asString() + "/" + type.getName(); + @Nonnull + private Term idTerm(Id id) { + return new Term(ID, id.asString()); } private void field(Document document, String type, String name) { @@ -94,24 +111,11 @@ class LuceneIndex implements Index { private class LuceneDeleter implements Deleter { - @Override - public ByTypeDeleter byType() { - return new LuceneByTypeDeleter(); - } - - @Override - public AllTypesDeleter allTypes() { - return new LuceneAllTypesDelete(); - } - } - - @SuppressWarnings("java:S1192") - private class LuceneByTypeDeleter implements ByTypeDeleter { - @Override public void byId(Id id) { try { - writer.deleteDocuments(new Term(UID, createUid(id, searchableType))); + long count = writer.deleteDocuments(idTerm(id)); + LOG.debug("delete {} document(s) by id {} from index {}", count, id, details); } catch (IOException e) { throw new SearchEngineException("failed to delete document from index", e); } @@ -120,7 +124,8 @@ class LuceneIndex implements Index { @Override public void all() { try { - writer.deleteDocuments(new Term(TYPE, searchableType.getName())); + long count = writer.deleteAll(); + LOG.debug("deleted all {} documents from index {}", count, details); } catch (IOException ex) { throw new SearchEngineException("failed to delete documents by type " + searchableType.getName() + " from index", ex); } @@ -129,35 +134,16 @@ class LuceneIndex implements Index { @Override public void byRepository(String repositoryId) { try { - BooleanQuery query = new BooleanQuery.Builder() - .add(new TermQuery(new Term(TYPE, searchableType.getName())), BooleanClause.Occur.MUST) - .add(new TermQuery(new Term(REPOSITORY, repositoryId)), BooleanClause.Occur.MUST) - .build(); - writer.deleteDocuments(query); + long count = writer.deleteDocuments(repositoryTerm(repositoryId)); + LOG.debug("deleted {} documents by repository {} from index {}", count, repositoryId, details); } catch (IOException ex) { throw new SearchEngineException("failed to delete documents by repository " + repositoryId + " from index", ex); } } - } - private class LuceneAllTypesDelete implements AllTypesDeleter { - - @Override - public void byRepository(String repositoryId) { - try { - writer.deleteDocuments(new Term(REPOSITORY, repositoryId)); - } catch (IOException ex) { - throw new SearchEngineException("failed to delete all documents by repository " + repositoryId + " from index", ex); - } - } - - @Override - public void byTypeName(String typeName) { - try { - writer.deleteDocuments(new Term(TYPE, typeName)); - } catch (IOException ex) { - throw new SearchEngineException("failed to delete documents by type " + typeName + " from index", ex); - } + @Nonnull + private Term repositoryTerm(String repositoryId) { + return new Term(REPOSITORY, repositoryId); } } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java new file mode 100644 index 0000000000..e4010bc115 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java @@ -0,0 +1,43 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@Data +@XmlRootElement +@NoArgsConstructor +@AllArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class LuceneIndexDetails implements IndexDetails { + private Class type; + private String name; +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java index 103fe63c55..aca51bb430 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java @@ -24,23 +24,50 @@ package sonia.scm.search; -import javax.inject.Inject; -import java.io.IOException; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Singleton +@SuppressWarnings("unchecked") public class LuceneIndexFactory { - private final IndexOpener indexOpener; + private final IndexManager indexManager; + @SuppressWarnings("rawtypes") + private final Map indexes = new ConcurrentHashMap<>(); @Inject - public LuceneIndexFactory(IndexOpener indexOpener) { - this.indexOpener = indexOpener; + public LuceneIndexFactory(IndexManager indexManager) { + this.indexManager = indexManager; } public LuceneIndex create(IndexParams indexParams) { - try { - return new LuceneIndex<>(indexParams.getSearchableType(), indexOpener.openForWrite(indexParams)); - } catch (IOException ex) { - throw new SearchEngineException("failed to open index " + indexParams.getIndex(), ex); - } + return indexes.compute(keyOf(indexParams), (key, index) -> { + if (index != null) { + index.open(); + return index; + } + return new LuceneIndex<>( + indexParams, + () -> indexManager.openForWrite(indexParams) + ); + }); + } + + private IndexKey keyOf(IndexParams indexParams) { + return new IndexKey( + indexParams.getSearchableType().getName(), indexParams.getIndex() + ); + } + + @EqualsAndHashCode + @AllArgsConstructor + private static class IndexKey { + String type; + String name; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java new file mode 100644 index 0000000000..589ec39945 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java @@ -0,0 +1,76 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.Injector; + +import javax.inject.Inject; +import java.io.Serializable; + +public abstract class LuceneIndexTask implements Runnable, Serializable { + + private final Class type; + private final String indexName; + private final IndexOptions options; + + private transient LuceneIndexFactory indexFactory; + private transient SearchableTypeResolver searchableTypeResolver; + private transient Injector injector; + + protected LuceneIndexTask(IndexParams params) { + this.type = params.getSearchableType().getType(); + this.indexName = params.getIndex(); + this.options = params.getOptions(); + } + + @Inject + public void setIndexFactory(LuceneIndexFactory indexFactory) { + this.indexFactory = indexFactory; + } + + @Inject + public void setSearchableTypeResolver(SearchableTypeResolver searchableTypeResolver) { + this.searchableTypeResolver = searchableTypeResolver; + } + + @Inject + public void setInjector(Injector injector) { + this.injector = injector; + } + + public abstract IndexTask task(Injector injector); + + @SuppressWarnings({"unchecked", "rawtypes"}) + public void run() { + LuceneSearchableType searchableType = searchableTypeResolver.resolve(type); + IndexTask task = task(injector); + try (LuceneIndex index = indexFactory.create(new IndexParams(indexName, searchableType, options))) { + task.update(index); + } + task.afterUpdate(); + } + + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java similarity index 58% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java rename to scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java index 2a7c152f97..a62feb5b60 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java @@ -24,31 +24,21 @@ package sonia.scm.search; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.inject.Injector; +import sonia.scm.work.Task; -public final class IndexQueueTaskWrapper implements Runnable { +@SuppressWarnings("rawtypes") +public class LuceneInjectingIndexTask extends LuceneIndexTask implements Task { - private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class); + private final Class taskClass; - private final LuceneIndexFactory indexFactory; - private final IndexParams indexParams; - private final Iterable> tasks; - - IndexQueueTaskWrapper(LuceneIndexFactory indexFactory, IndexParams indexParams, Iterable> tasks) { - this.indexFactory = indexFactory; - this.indexParams = indexParams; - this.tasks = tasks; + LuceneInjectingIndexTask(IndexParams params, Class taskClass) { + super(params); + this.taskClass = taskClass; } @Override - public void run() { - try (Index index = indexFactory.create(indexParams)) { - for (IndexQueueTask task : tasks) { - task.updateIndex(index); - } - } catch (Exception e) { - LOG.warn("failure during execution of index task for index {}", indexParams.getIndex(), e); - } + public IndexTask task(Injector injector) { + return injector.getInstance(taskClass); } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java index 35b030bb35..7ac161bc46 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java @@ -54,12 +54,12 @@ public class LuceneQueryBuilder extends QueryBuilder { private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class); - private final IndexOpener opener; + private final IndexManager opener; private final LuceneSearchableType searchableType; private final String indexName; private final Analyzer analyzer; - LuceneQueryBuilder(IndexOpener opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) { + LuceneQueryBuilder(IndexManager opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) { this.opener = opener; this.indexName = indexName; this.searchableType = searchableType; @@ -88,11 +88,11 @@ public class LuceneQueryBuilder extends QueryBuilder { String queryString = Strings.nullToEmpty(queryParams.getQueryString()); Query parsedQuery = createQuery(searchableType, queryParams, queryString); - Query query = Queries.filter(parsedQuery, searchableType, queryParams); + Query query = Queries.filter(parsedQuery, queryParams); if (LOG.isDebugEnabled()) { LOG.debug("execute lucene query: {}", query); } - try (IndexReader reader = opener.openForRead(indexName)) { + try (IndexReader reader = opener.openForRead(searchableType, indexName)) { IndexSearcher searcher = new IndexSearcher(reader); searcher.search(query, new PermissionAwareCollector(reader, collector)); diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java index c3e20036bd..b543634a1d 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java @@ -28,18 +28,18 @@ import javax.inject.Inject; public class LuceneQueryBuilderFactory { - private final IndexOpener indexOpener; + private final IndexManager indexManager; private final AnalyzerFactory analyzerFactory; @Inject - public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) { - this.indexOpener = indexOpener; + public LuceneQueryBuilderFactory(IndexManager indexManager, AnalyzerFactory analyzerFactory) { + this.indexManager = indexManager; this.analyzerFactory = analyzerFactory; } public LuceneQueryBuilder create(IndexParams indexParams) { return new LuceneQueryBuilder<>( - indexOpener, + indexManager, indexParams.getIndex(), indexParams.getSearchableType(), analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions()) diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java index 7a7c5c9b23..9c71f76402 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java @@ -24,24 +24,34 @@ package sonia.scm.search; +import com.google.common.base.Joiner; import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.Subject; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.CentralWorkQueue.Enqueue; +import sonia.scm.work.Task; import javax.inject.Inject; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; public class LuceneSearchEngine implements SearchEngine { + private final IndexManager indexManager; private final SearchableTypeResolver resolver; - private final IndexQueue indexQueue; private final LuceneQueryBuilderFactory queryBuilderFactory; + private final CentralWorkQueue centralWorkQueue; @Inject - public LuceneSearchEngine(SearchableTypeResolver resolver, IndexQueue indexQueue, LuceneQueryBuilderFactory queryBuilderFactory) { + public LuceneSearchEngine(IndexManager indexManager, SearchableTypeResolver resolver, LuceneQueryBuilderFactory queryBuilderFactory, CentralWorkQueue centralWorkQueue) { + this.indexManager = indexManager; this.resolver = resolver; - this.indexQueue = indexQueue; this.queryBuilderFactory = queryBuilderFactory; + this.centralWorkQueue = centralWorkQueue; } @Override @@ -67,11 +77,79 @@ public class LuceneSearchEngine implements SearchEngine { return new LuceneForType<>(searchableType); } + private void enqueue(LuceneSearchableType searchableType, String index, List resources, Task task) { + Enqueue enqueuer = centralWorkQueue.append(); + + String resourceName = Joiner.on('-').join(searchableType.getName(), index, "index"); + if (resources.isEmpty()) { + enqueuer.locks(resourceName); + } else { + for (String resource : resources) { + enqueuer.locks(resourceName, resource); + } + } + + enqueuer.runAsAdmin().enqueue(task); + } + + @Override + public ForIndices forIndices() { + return new LuceneForIndices(); + } + + class LuceneForIndices implements ForIndices { + + private final List resources = new ArrayList<>(); + private Predicate predicate = details -> true; + private IndexOptions options = IndexOptions.defaults(); + + @Override + public ForIndices matching(Predicate predicate) { + this.predicate = predicate; + return this; + } + + @Override + public ForIndices withOptions(IndexOptions options) { + this.options = options; + return this; + } + + @Override + public ForIndices forResource(String resource) { + this.resources.add(resource); + return this; + } + + @Override + public void batch(SerializableIndexTask task) { + exec(params -> batch(params, new LuceneSimpleIndexTask(params, task))); + } + + @Override + public void batch(Class> task) { + exec(params -> batch(params, new LuceneInjectingIndexTask(params, task))); + } + + private void exec(Consumer consumer) { + indexManager.all() + .stream() + .filter(predicate) + .map(details -> new IndexParams(details.getName(), resolver.resolve(details.getType()), options)) + .forEach(consumer); + } + + private void batch(IndexParams params, Task task) { + LuceneSearchEngine.this.enqueue(params.getSearchableType(), params.getIndex(), resources, task); + } + } + class LuceneForType implements ForType { private final LuceneSearchableType searchableType; private IndexOptions options = IndexOptions.defaults(); private String index = "default"; + private final List resources = new ArrayList<>(); private LuceneForType(LuceneSearchableType searchableType) { this.searchableType = searchableType; @@ -94,8 +172,23 @@ public class LuceneSearchEngine implements SearchEngine { } @Override - public Index getOrCreate() { - return indexQueue.getQueuedIndex(params()); + public ForType forResource(String resource) { + resources.add(resource); + return this; + } + + @Override + public void update(Class> task) { + enqueue(new LuceneInjectingIndexTask(params(), task)); + } + + @Override + public void update(SerializableIndexTask task) { + enqueue(new LuceneSimpleIndexTask(params(), task)); + } + + private void enqueue(Task task) { + LuceneSearchEngine.this.enqueue(searchableType, index, resources, task); } @Override diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java similarity index 51% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java rename to scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java index a8d7de863c..c12be9d9e9 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java @@ -24,53 +24,21 @@ package sonia.scm.search; -import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Injector; +import sonia.scm.work.Task; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; +public final class LuceneSimpleIndexTask extends LuceneIndexTask implements Task { -@Singleton -public class IndexQueue implements Closeable { + private final SerializableIndexTask task; - private final ExecutorService executor = Executors.newSingleThreadExecutor(); - - private final AtomicLong size = new AtomicLong(0); - - private final LuceneIndexFactory indexFactory; - - @Inject - public IndexQueue(LuceneIndexFactory indexFactory) { - this.indexFactory = indexFactory; - } - - public Index getQueuedIndex(IndexParams indexParams) { - return new QueuedIndex<>(this, indexParams); - } - - public LuceneIndexFactory getIndexFactory() { - return indexFactory; - } - - void enqueue(IndexQueueTaskWrapper task) { - size.incrementAndGet(); - executor.execute(() -> { - task.run(); - size.decrementAndGet(); - }); - } - - @VisibleForTesting - long getSize() { - return size.get(); + LuceneSimpleIndexTask(IndexParams params, SerializableIndexTask task) { + super(params); + this.task = task; } @Override - public void close() throws IOException { - executor.shutdown(); + public IndexTask task(Injector injector) { + injector.injectMembers(task); + return task; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/Queries.java b/scm-webapp/src/main/java/sonia/scm/search/Queries.java index 6ec016fb65..af7a1d8953 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/Queries.java +++ b/scm-webapp/src/main/java/sonia/scm/search/Queries.java @@ -29,6 +29,8 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import java.util.Optional; + import static org.apache.lucene.search.BooleanClause.Occur.MUST; final class Queries { @@ -36,19 +38,18 @@ final class Queries { private Queries() { } - private static Query typeQuery(LuceneSearchableType type) { - return new TermQuery(new Term(FieldNames.TYPE, type.getName())); - } - private static Query repositoryQuery(String repositoryId) { return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId)); } - static Query filter(Query query, LuceneSearchableType searchableType, QueryBuilder.QueryParams params) { - BooleanQuery.Builder builder = new BooleanQuery.Builder() - .add(query, MUST) - .add(typeQuery(searchableType), MUST); - params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST)); - return builder.build(); + static Query filter(Query query, QueryBuilder.QueryParams params) { + Optional repositoryId = params.getRepositoryId(); + if (repositoryId.isPresent()) { + return new BooleanQuery.Builder() + .add(query, MUST) + .add(repositoryQuery(repositoryId.get()), MUST) + .build(); + } + return query; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java deleted file mode 100644 index 70ccaef7f2..0000000000 --- a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import java.util.ArrayList; -import java.util.List; - -class QueuedIndex implements Index { - - private final IndexQueue queue; - private final IndexParams indexParams; - private final List> tasks = new ArrayList<>(); - - QueuedIndex(IndexQueue queue, IndexParams indexParams) { - this.queue = queue; - this.indexParams = indexParams; - } - - @Override - public void store(Id id, String permission, T object) { - tasks.add(index -> index.store(id, permission, object)); - } - - @Override - public Deleter delete() { - return new QueueDeleter(); - } - - @Override - public void close() { - IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper<>( - queue.getIndexFactory(), indexParams, tasks - ); - queue.enqueue(wrappedTask); - } - - private class QueueDeleter implements Deleter { - - @Override - public ByTypeDeleter byType() { - return new QueueByTypeDeleter(); - } - - @Override - public AllTypesDeleter allTypes() { - return new QueueAllTypesDeleter(); - } - } - - private class QueueByTypeDeleter implements ByTypeDeleter { - - @Override - public void byId(Id id) { - tasks.add(index -> index.delete().byType().byId(id)); - } - - @Override - public void all() { - tasks.add(index -> index.delete().byType().all()); - } - - @Override - public void byRepository(String repositoryId) { - tasks.add(index -> index.delete().byType().byRepository(repositoryId)); - } - } - - private class QueueAllTypesDeleter implements AllTypesDeleter { - - @Override - public void byRepository(String repositoryId) { - tasks.add(index -> index.delete().allTypes().byRepository(repositoryId)); - } - - @Override - public void byTypeName(String typeName) { - tasks.add(index -> index.delete().allTypes().byTypeName(typeName)); - } - } -} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java b/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java new file mode 100644 index 0000000000..769430b1d1 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java @@ -0,0 +1,90 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.function.Supplier; + +class SharableIndexWriter { + + private static final Logger LOG = LoggerFactory.getLogger(SharableIndexWriter.class); + + private int usageCounter = 0; + + private final Supplier writerFactory; + private IndexWriter writer; + + SharableIndexWriter(Supplier writerFactory) { + this.writerFactory = writerFactory; + } + + synchronized void open() { + usageCounter++; + if (usageCounter == 1) { + LOG.trace("open writer, because usage increased from zero to one"); + writer = writerFactory.get(); + } else { + LOG.trace("new task is using the writer, counter is now at {}", usageCounter); + } + } + + @VisibleForTesting + int getUsageCounter() { + return usageCounter; + } + + void updateDocument(Term term, Document document) throws IOException { + writer.updateDocument(term, document); + } + + long deleteDocuments(Term term) throws IOException { + return writer.deleteDocuments(term); + } + + long deleteAll() throws IOException { + return writer.deleteAll(); + } + + synchronized void close() throws IOException { + usageCounter--; + if (usageCounter == 0) { + LOG.trace("no one seems to use index any longer, closing underlying writer"); + writer.close(); + writer = null; + } else if (usageCounter > 0) { + LOG.trace("index is still used by {} task(s), commit work but keep writer open", usageCounter); + writer.commit(); + } else { + LOG.warn("index is already closed"); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java b/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java new file mode 100644 index 0000000000..0b4b691cd3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.security; + +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.Subject; +import org.apache.shiro.subject.support.SubjectThreadState; +import org.apache.shiro.util.ThreadContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +/** + * Impersonator allows the usage of scm-manager api in the context of another user. + * + * @since 2.23.0 + */ +public final class Impersonator { + + private static final Logger LOG = LoggerFactory.getLogger(Impersonator.class); + + private final SecurityManager securityManager; + + @Inject + public Impersonator(SecurityManager securityManager) { + this.securityManager = securityManager; + } + + public Session impersonate(PrincipalCollection principal) { + Subject subject = createSubject(principal); + if (ThreadContext.getSecurityManager() != null) { + return new WebImpersonator(subject); + } + return new NonWebImpersonator(securityManager, subject); + } + + private Subject createSubject(PrincipalCollection principal) { + return new Subject.Builder(securityManager) + .authenticated(true) + .principals(principal) + .buildSubject(); + } + + public interface Session extends AutoCloseable { + void close(); + } + + private static class WebImpersonator implements Session { + + private final Subject subject; + private final Subject previousSubject; + + private WebImpersonator(Subject subject) { + this.subject = subject; + this.previousSubject = SecurityUtils.getSubject(); + bind(); + } + + private void bind() { + LOG.debug("user {} start impersonate session as {}", previousSubject.getPrincipal(), subject.getPrincipal()); + + + // do not use runas, because we want only bind the session to this thread. + // Runas could affect other threads. + ThreadContext.bind(this.subject); + } + + @Override + public void close() { + LOG.debug("release impersonate session from user {} to {}", previousSubject.getPrincipal(), subject.getPrincipal()); + ThreadContext.bind(previousSubject); + } + + } + + private static class NonWebImpersonator implements Session { + + private final SecurityManager securityManager; + private final SubjectThreadState state; + private final Subject subject; + + private NonWebImpersonator(SecurityManager securityManager, Subject subject) { + this.securityManager = securityManager; + this.state = new SubjectThreadState(subject); + this.subject = subject; + bind(); + } + + private void bind() { + LOG.debug("start impersonate session as user {}", subject.getPrincipal()); + SecurityUtils.setSecurityManager(securityManager); + state.bind(); + } + + @Override + public void close() { + LOG.debug("release impersonate session of {}", subject.getPrincipal()); + state.restore(); + SecurityUtils.setSecurityManager(null); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java b/scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java similarity index 51% rename from scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java rename to scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java index 2e84eb4333..38ae345d08 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java +++ b/scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java @@ -22,44 +22,59 @@ * SOFTWARE. */ -package sonia.scm.search; +package sonia.scm.update.index; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import sonia.scm.SCMContextProvider; +import sonia.scm.migration.UpdateStep; +import sonia.scm.plugin.Extension; +import sonia.scm.util.IOUtil; +import sonia.scm.version.Version; +import javax.annotation.Nonnull; import javax.inject.Inject; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -public class IndexOpener { +import static sonia.scm.store.StoreConstants.DATA_DIRECTORY_NAME; +import static sonia.scm.store.StoreConstants.VARIABLE_DATA_DIRECTORY_NAME; - private final Path directory; - private final AnalyzerFactory analyzerFactory; +@Extension +public class RemoveCombinedIndex implements UpdateStep { + + private final SCMContextProvider contextProvider; @Inject - public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) { - directory = context.resolve(Paths.get("index")); - this.analyzerFactory = analyzerFactory; + public RemoveCombinedIndex(SCMContextProvider contextProvider) { + this.contextProvider = contextProvider; } - public IndexReader openForRead(String name) throws IOException { - return DirectoryReader.open(directory(name)); + @Override + public void doUpdate() throws IOException { + Path index = contextProvider.resolve(Paths.get("index")); + if (Files.exists(index)) { + IOUtil.delete(index.toFile()); + } + + Path indexLog = contextProvider.resolve(indexLogPath()); + if (Files.exists(indexLog)) { + IOUtil.delete(indexLog.toFile()); + } } - public IndexWriter openForWrite(IndexParams indexParams) throws IOException { - IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())); - config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory(indexParams.getIndex()), config); + @Nonnull + private Path indexLogPath() { + return Paths.get(VARIABLE_DATA_DIRECTORY_NAME).resolve(DATA_DIRECTORY_NAME).resolve("index-log"); } - private Directory directory(String name) throws IOException { - return FSDirectory.open(directory.resolve(name)); + @Override + public Version getTargetVersion() { + return Version.parse("2.0.0"); } + @Override + public String getAffectedDataType() { + return "sonia.scm.index"; + } } diff --git a/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java b/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java index 4fd2271c63..383945e714 100644 --- a/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java @@ -30,8 +30,10 @@ import sonia.scm.plugin.Extension; import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class UserIndexer implements Indexer { @VisibleForTesting static final int VERSION = 1; - private final UserManager userManager; private final SearchEngine searchEngine; @Inject - public UserIndexer(UserManager userManager, SearchEngine searchEngine) { - this.userManager = userManager; + public UserIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,47 +62,46 @@ public class UserIndexer implements Indexer { return VERSION; } - @Subscribe(async = false) - public void handleEvent(UserEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; } @Override - public Updater open() { - return new UserIndexUpdater(userManager, searchEngine.forType(User.class).getOrCreate()); + public SerializableIndexTask createStoreTask(User user) { + return index -> store(index, user); } - public static class UserIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(User item) { + return index -> index.delete().byId(Id.of(item)); + } + + @Subscribe(async = false) + public void handleEvent(UserEvent event) { + new HandlerEventIndexSyncer<>(searchEngine, this).handleEvent(event); + } + + private static void store(Index index, User user) { + index.store(Id.of(user), UserPermissions.read(user).asShiroString(), user); + } + + public static class ReIndexAll extends ReIndexAllTask { private final UserManager userManager; - private final Index index; - private UserIndexUpdater(UserManager userManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, UserManager userManager) { + super(logStore, User.class, VERSION); this.userManager = userManager; - this.index = index; } @Override - public void store(User user) { - index.store(Id.of(user), UserPermissions.read(user).asShiroString(), user); - } - - @Override - public void delete(User user) { - index.delete().byType().byId(Id.of(user)); - } - - @Override - public void reIndexAll() { - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (User user : userManager.getAll()) { - store(user); + store(index, user); } } - - @Override - public void close() { - index.close(); - } } } diff --git a/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java b/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java index 1ad6876b70..40218c89f0 100644 --- a/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java +++ b/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java @@ -21,10 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.web.security; -final class AdministrationContextMarker { +import java.io.Serializable; + +final class AdministrationContextMarker implements Serializable { static final AdministrationContextMarker MARKER = new AdministrationContextMarker(); diff --git a/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java b/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java index dc24498094..e0de27a917 100644 --- a/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java +++ b/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java @@ -24,223 +24,70 @@ package sonia.scm.web.security; -//~--- non-JDK imports -------------------------------------------------------- - import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; -import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.PrincipalCollection; import org.apache.shiro.subject.SimplePrincipalCollection; -import org.apache.shiro.subject.Subject; -import org.apache.shiro.subject.support.SubjectThreadState; -import org.apache.shiro.util.ThreadContext; -import org.apache.shiro.util.ThreadState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sonia.scm.SCMContext; import sonia.scm.security.Authentications; -import sonia.scm.security.Role; +import sonia.scm.security.Impersonator; import sonia.scm.user.User; import sonia.scm.util.AssertUtil; //~--- JDK imports ------------------------------------------------------------ /** - * * @author Sebastian Sdorra */ @Singleton -public class DefaultAdministrationContext implements AdministrationContext -{ +public class DefaultAdministrationContext implements AdministrationContext { - /** Field description */ private static final User SYSTEM_ACCOUNT = new User( Authentications.PRINCIPAL_SYSTEM, "SCM-Manager System Account", null ); - - - /** Field description */ static final String REALM = "AdminRealm"; - /** the logger for DefaultAdministrationContext */ - private static final Logger logger = - LoggerFactory.getLogger(DefaultAdministrationContext.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultAdministrationContext.class); - //~--- constructors --------------------------------------------------------- + private final Injector injector; + private final Impersonator impersonator; + private final PrincipalCollection adminPrincipal; - /** - * Constructs ... - * - * - * @param injector - * @param securityManager - */ @Inject - public DefaultAdministrationContext(Injector injector, - org.apache.shiro.mgt.SecurityManager securityManager) - { + public DefaultAdministrationContext(Injector injector, Impersonator impersonator) { this.injector = injector; - this.securityManager = securityManager; - - principalCollection = createAdminCollection(SYSTEM_ACCOUNT); + this.impersonator = impersonator; + this.adminPrincipal = createAdminPrincipal(); } - //~--- methods -------------------------------------------------------------- - - /** - * Method description - * - * - * @param action - */ - @Override - public void runAsAdmin(PrivilegedAction action) - { - AssertUtil.assertIsNotNull(action); - - if (ThreadContext.getSecurityManager() != null) - { - doRunAsInWebSessionContext(action); - } - else - { - doRunAsInNonWebSessionContext(action); - } - - } - - /** - * Method description - * - * - * @param actionClass - */ - @Override - public void runAsAdmin(Class actionClass) - { - PrivilegedAction action = injector.getInstance(actionClass); - - runAsAdmin(action); - } - - /** - * Method description - * - * - * @param adminUser - * - * @return - */ - private PrincipalCollection createAdminCollection(User adminUser) - { + public static PrincipalCollection createAdminPrincipal() { SimplePrincipalCollection collection = new SimplePrincipalCollection(); - collection.add(adminUser.getId(), REALM); - collection.add(adminUser, REALM); + collection.add(SYSTEM_ACCOUNT.getId(), REALM); + collection.add(SYSTEM_ACCOUNT, REALM); collection.add(AdministrationContextMarker.MARKER, REALM); return collection; } - /** - * Method description - * - * - * @return - */ - private Subject createAdminSubject() - { - //J- - return new Subject.Builder(securityManager) - .authenticated(true) - .principals(principalCollection) - .buildSubject(); - //J+ - } - - private void doRunAsInNonWebSessionContext(PrivilegedAction action) { - logger.trace("bind shiro security manager to current thread"); - - try { - SecurityUtils.setSecurityManager(securityManager); - - Subject subject = createAdminSubject(); - ThreadState state = new SubjectThreadState(subject); - - state.bind(); - try - { - logger.debug("execute action {} in administration context", action.getClass().getName()); - - action.run(); - } finally { - logger.trace("restore current thread state"); - state.restore(); - } - } finally { - SecurityUtils.setSecurityManager(null); - } - } - - /** - * Method description - * - * - * @param action - */ - private void doRunAsInWebSessionContext(PrivilegedAction action) - { - Subject subject = SecurityUtils.getSubject(); - - String principal = (String) subject.getPrincipal(); - - if (logger.isInfoEnabled()) - { - String username; - - if (subject.hasRole(Role.USER)) - { - username = principal; - } - else - { - username = SCMContext.USER_ANONYMOUS; - } - - logger.debug("user {} executes {} as admin", username, action.getClass().getName()); - } - - Subject adminSubject = createAdminSubject(); - - // do not use runas, because we want only execute this action in this - // thread as administrator. Runas could affect other threads - - ThreadContext.bind(adminSubject); - - try - { + @Override + public void runAsAdmin(PrivilegedAction action) { + AssertUtil.assertIsNotNull(action); + LOG.debug("execute action {} in administration context", action.getClass().getName()); + try (Impersonator.Session session = impersonator.impersonate(adminPrincipal)) { action.run(); } - finally - { - logger.debug("release administration context for user {}/{}", principal, - subject.getPrincipal()); - ThreadContext.bind(subject); - } } - //~--- fields --------------------------------------------------------------- + @Override + public void runAsAdmin(Class actionClass) { + PrivilegedAction action = injector.getInstance(actionClass); + runAsAdmin(action); + } - /** Field description */ - private final Injector injector; - - /** Field description */ - private final org.apache.shiro.mgt.SecurityManager securityManager; - - /** Field description */ - private PrincipalCollection principalCollection; } diff --git a/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java b/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java new file mode 100644 index 0000000000..b5802a17a8 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java @@ -0,0 +1,213 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.subject.PrincipalCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.metrics.Metrics; +import sonia.scm.web.security.DefaultAdministrationContext; + +import javax.annotation.Nullable; +import javax.inject.Singleton; +import java.io.Closeable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntSupplier; + +@Singleton +public class DefaultCentralWorkQueue implements CentralWorkQueue, Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultCentralWorkQueue.class); + + private final List queue = new ArrayList<>(); + private final List lockedResources = new ArrayList<>(); + private final AtomicInteger size = new AtomicInteger(); + private final AtomicLong order = new AtomicLong(); + + private final Injector injector; + private final Persistence persistence; + private final ExecutorService executor; + private final MeterRegistry meterRegistry; + + @Inject + public DefaultCentralWorkQueue(Injector injector, Persistence persistence, MeterRegistry meterRegistry) { + this(injector, persistence, meterRegistry, new ThreadCountProvider()); + } + + @VisibleForTesting + DefaultCentralWorkQueue(Injector injector, Persistence persistence, MeterRegistry meterRegistry, IntSupplier threadCountProvider) { + this.injector = injector; + this.persistence = persistence; + this.executor = createExecutorService(meterRegistry, threadCountProvider.getAsInt()); + this.meterRegistry = meterRegistry; + + loadFromDisk(); + } + + private static ExecutorService createExecutorService(MeterRegistry registry, int threadCount) { + ExecutorService executorService = Executors.newFixedThreadPool( + threadCount, + new ThreadFactoryBuilder() + .setNameFormat("CentralWorkQueue-%d") + .build() + ); + Metrics.executor(registry, executorService, "CentralWorkQueue", "fixed"); + return executorService; + } + + @Override + public Enqueue append() { + return new DefaultEnqueue(); + } + + @Override + public int getSize() { + return size.get(); + } + + @Override + public void close() { + executor.shutdown(); + } + + private void loadFromDisk() { + for (UnitOfWork unitOfWork : persistence.loadAll()) { + unitOfWork.restore(order.incrementAndGet()); + append(unitOfWork); + } + run(); + } + + private synchronized void append(UnitOfWork unitOfWork) { + persistence.store(unitOfWork); + int queueSize = size.incrementAndGet(); + queue.add(unitOfWork); + LOG.debug("add task {} to queue, queue size is now {}", unitOfWork, queueSize); + } + + private synchronized void run() { + Iterator iterator = queue.iterator(); + while (iterator.hasNext()) { + UnitOfWork unitOfWork = iterator.next(); + if (isRunnable(unitOfWork)) { + run(unitOfWork); + iterator.remove(); + } else { + unitOfWork.blocked(); + } + } + } + + private void run(UnitOfWork unitOfWork) { + lockedResources.addAll(unitOfWork.getLocks()); + unitOfWork.init(injector, this::finalizeWork, meterRegistry); + LOG.trace("pass task {} to executor", unitOfWork); + executor.execute(unitOfWork); + } + + private synchronized void finalizeWork(UnitOfWork unitOfWork) { + for (Resource lock : unitOfWork.getLocks()) { + lockedResources.remove(lock); + } + persistence.remove(unitOfWork); + + int queueSize = size.decrementAndGet(); + LOG.debug("finish task, queue size is now {}", queueSize); + + run(); + } + + private boolean isRunnable(UnitOfWork unitOfWork) { + for (Resource resource : unitOfWork.getLocks()) { + for (Resource lock : lockedResources) { + if (resource.isBlockedBy(lock)) { + LOG.trace("skip {}, because resource {} is locked by {}", unitOfWork, resource, lock); + return false; + } + } + } + return true; + } + + private class DefaultEnqueue implements Enqueue { + + private final Set locks = new HashSet<>(); + private boolean runAsAdmin = false; + + @Override + public Enqueue locks(String resourceType) { + locks.add(new Resource(resourceType)); + return this; + } + + @Override + public Enqueue locks(String resource, @Nullable String id) { + locks.add(new Resource(resource, id)); + return this; + } + + @Override + public Enqueue runAsAdmin() { + this.runAsAdmin = true; + return this; + } + + @Override + public void enqueue(Task task) { + appendAndRun(new SimpleUnitOfWork(order.incrementAndGet(), principal(), locks, task)); + } + + @Override + public void enqueue(Class task) { + appendAndRun(new InjectingUnitOfWork(order.incrementAndGet(), principal(), locks, task)); + } + + private PrincipalCollection principal() { + if (runAsAdmin) { + return DefaultAdministrationContext.createAdminPrincipal(); + } + return SecurityUtils.getSubject().getPrincipals(); + } + + private synchronized void appendAndRun(UnitOfWork unitOfWork) { + append(unitOfWork); + run(); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java b/scm-webapp/src/main/java/sonia/scm/work/Finalizer.java similarity index 92% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java rename to scm-webapp/src/main/java/sonia/scm/work/Finalizer.java index f1ac704782..832caa3c6e 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java +++ b/scm-webapp/src/main/java/sonia/scm/work/Finalizer.java @@ -22,11 +22,11 @@ * SOFTWARE. */ -package sonia.scm.search; +package sonia.scm.work; @FunctionalInterface -public interface IndexQueueTask { +interface Finalizer { - void updateIndex(Index index); + void finalizeWork(UnitOfWork unitOfWork); } diff --git a/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java new file mode 100644 index 0000000000..4d5e442a68 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.Injector; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; + +import java.util.Set; + +@EqualsAndHashCode(callSuper = true) +class InjectingUnitOfWork extends UnitOfWork { + + private final Class task; + + InjectingUnitOfWork(long order, PrincipalCollection principal, Set locks, Class task) { + super(order, principal, locks); + this.task = task; + } + + @Override + protected Runnable task(Injector injector) { + return injector.getInstance(task); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/Persistence.java b/scm-webapp/src/main/java/sonia/scm/work/Persistence.java new file mode 100644 index 0000000000..e736f4c1d0 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/Persistence.java @@ -0,0 +1,108 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import org.apache.commons.io.input.ClassLoaderObjectInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.plugin.PluginLoader; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.store.BlobStoreFactory; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +class Persistence { + + private static final Logger LOG = LoggerFactory.getLogger(Persistence.class); + private static final String STORE_NAME = "central-work-queue"; + + private final ClassLoader classLoader; + private final BlobStore store; + + @Inject + public Persistence(PluginLoader pluginLoader, BlobStoreFactory storeFactory) { + this(pluginLoader.getUberClassLoader(), storeFactory.withName(STORE_NAME).build()); + } + + @VisibleForTesting + Persistence(ClassLoader classLoader, BlobStore store) { + this.classLoader = classLoader; + this.store = store; + } + + Collection loadAll() { + List chunks = new ArrayList<>(); + for (Blob blob : store.getAll()) { + load(blob).ifPresent(chunkOfWork -> { + chunkOfWork.assignStorageId(null); + chunks.add(chunkOfWork); + }); + store.remove(blob); + } + Collections.sort(chunks); + return chunks; + } + + private Optional load(Blob blob) { + try (ObjectInputStream stream = new ClassLoaderObjectInputStream(classLoader, blob.getInputStream())) { + Object o = stream.readObject(); + if (o instanceof UnitOfWork) { + return Optional.of((UnitOfWork) o); + } else { + LOG.error("loaded object is not a instance of {}: {}", UnitOfWork.class, o); + } + } catch (IOException | ClassNotFoundException ex) { + LOG.error("failed to load task from store", ex); + } + return Optional.empty(); + } + + void store(UnitOfWork unitOfWork) { + Blob blob = store.create(); + try (ObjectOutputStream outputStream = new ObjectOutputStream(blob.getOutputStream())) { + outputStream.writeObject(unitOfWork); + blob.commit(); + + unitOfWork.assignStorageId(blob.getId()); + } catch (IOException ex) { + throw new NonPersistableTaskException("Failed to persist task", ex); + } + } + + void remove(UnitOfWork unitOfWork) { + unitOfWork.getStorageId().ifPresent(store::remove); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/Resource.java b/scm-webapp/src/main/java/sonia/scm/work/Resource.java new file mode 100644 index 0000000000..dec2c7fabb --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/Resource.java @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import lombok.EqualsAndHashCode; + +import javax.annotation.Nullable; +import java.io.Serializable; + +@EqualsAndHashCode +final class Resource implements Serializable { + + private final String name; + @Nullable + private final String id; + + Resource(String name) { + this.name = name; + this.id = null; + } + + Resource(String name, @Nullable String id) { + this.name = name; + this.id = id; + } + + boolean isBlockedBy(Resource resource) { + if (name.equals(resource.name)) { + if (id != null && resource.id != null) { + return id.equals(resource.id); + } + return true; + } + return false; + } + + @Override + public String toString() { + if (id != null) { + return name + ":" + id; + } + return name; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java new file mode 100644 index 0000000000..f21a07a6fe --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.Injector; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; + +import java.util.Set; + +@EqualsAndHashCode(callSuper = true) +class SimpleUnitOfWork extends UnitOfWork { + + private final Task task; + + SimpleUnitOfWork(long order, PrincipalCollection principal, Set locks, Task task) { + super(order, principal, locks); + this.task = task; + } + + @Override + protected Task task(Injector injector) { + injector.injectMembers(task); + return task; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java b/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java new file mode 100644 index 0000000000..282f1843d3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java @@ -0,0 +1,79 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.IntSupplier; + +public class ThreadCountProvider implements IntSupplier { + + private static final Logger LOG = LoggerFactory.getLogger(ThreadCountProvider.class); + + @VisibleForTesting + static final String PROPERTY = "scm.central-work-queue.workers"; + + private final IntSupplier cpuCountProvider; + + public ThreadCountProvider() { + this(() -> Runtime.getRuntime().availableProcessors()); + } + + @VisibleForTesting + ThreadCountProvider(IntSupplier cpuCountProvider) { + this.cpuCountProvider = cpuCountProvider; + } + + @Override + public int getAsInt() { + Integer systemProperty = Integer.getInteger(PROPERTY); + if (systemProperty == null) { + LOG.debug("derive worker count from cpu count"); + return deriveFromCPUCount(); + } + if (isInvalid(systemProperty)) { + LOG.warn( + "system property {} contains a invalid value {}, fall back and derive worker count from cpu count", + PROPERTY, systemProperty + ); + return deriveFromCPUCount(); + } + return systemProperty; + } + + private boolean isInvalid(int value) { + return value <= 0 || value > 64; + } + + private int deriveFromCPUCount() { + int cpus = cpuCountProvider.getAsInt(); + if (cpus > 1) { + return 4; + } + return 2; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java new file mode 100644 index 0000000000..4ccd5b4b41 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java @@ -0,0 +1,149 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.inject.Injector; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.security.Impersonator; +import sonia.scm.security.Impersonator.Session; + +import java.io.Serializable; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@EqualsAndHashCode +abstract class UnitOfWork implements Runnable, Serializable, Comparable { + + @VisibleForTesting + static final String METRIC_EXECUTION = "cwq.task.execution.duration"; + + @VisibleForTesting + static final String METRIC_WAIT = "cwq.task.wait.duration"; + + private static final Logger LOG = LoggerFactory.getLogger(UnitOfWork.class); + + private long order; + private int blockCount = 0; + private int restoreCount = 0; + private final Set locks; + private final PrincipalCollection principal; + + private transient Finalizer finalizer; + private transient Runnable task; + private transient MeterRegistry meterRegistry; + private transient Impersonator impersonator; + + private transient long createdAt; + private transient String storageId; + + protected UnitOfWork(long order, PrincipalCollection principal, Set locks) { + this.order = order; + this.principal = principal; + this.locks = locks; + this.createdAt = System.nanoTime(); + } + + public long getOrder() { + return order; + } + + public void restore(long newOrderId) { + this.order = newOrderId; + this.createdAt = System.nanoTime(); + this.restoreCount++; + } + + public int getRestoreCount() { + return restoreCount; + } + + public void blocked() { + blockCount++; + } + + public void assignStorageId(String storageId) { + this.storageId = storageId; + } + + public Optional getStorageId() { + return Optional.ofNullable(storageId); + } + + public Set getLocks() { + return locks; + } + + void init(Injector injector, Finalizer finalizer, MeterRegistry meterRegistry) { + this.task = task(injector); + this.finalizer = finalizer; + this.meterRegistry = meterRegistry; + this.impersonator = injector.getInstance(Impersonator.class); + } + + protected abstract Runnable task(Injector injector); + + @Override + public void run() { + Stopwatch sw = Stopwatch.createStarted(); + Timer.Sample sample = Timer.start(meterRegistry); + try (Session session = impersonator.impersonate(principal)) { + task.run(); + LOG.debug("task {} finished successful after {}", task, sw.stop()); + } catch (Exception ex) { + LOG.error("task {} failed after {}", task, sw.stop(), ex); + } finally { + sample.stop(createExecutionTimer()); + createWaitTimer().record(System.nanoTime() - createdAt, TimeUnit.NANOSECONDS); + finalizer.finalizeWork(this); + } + } + + private Timer createExecutionTimer() { + return Timer.builder(METRIC_EXECUTION) + .description("Central work queue task execution duration") + .tags("task", task.getClass().getName()) + .register(meterRegistry); + } + + private Timer createWaitTimer() { + return Timer.builder(METRIC_WAIT) + .description("Central work queue task wait duration") + .tags("task", task.getClass().getName(), "restores", String.valueOf(restoreCount), "blocked", String.valueOf(blockCount)) + .register(meterRegistry); + } + + @Override + public int compareTo(UnitOfWork o) { + return Long.compare(order, o.order); + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java index bf013ed823..099a7966d1 100644 --- a/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java @@ -24,20 +24,23 @@ package sonia.scm.group; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +57,19 @@ class GroupIndexerTest { @InjectMocks private GroupIndexer indexer; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + + private final Group astronauts = new Group("xml", "astronauts"); + private final Group planetCreators = new Group("xml", "planet-creators"); + @Test void shouldReturnClass() { assertThat(indexer.getType()).isEqualTo(Group.class); @@ -64,58 +80,47 @@ class GroupIndexerTest { assertThat(indexer.getVersion()).isEqualTo(GroupIndexer.VERSION); } - @Nested - class UpdaterTests { - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(GroupIndexer.ReIndexAll.class); + } - private final Group group = new Group("xml", "astronauts"); + @Test + void shouldCreateGroup() { + indexer.createStoreTask(astronauts).update(index); - @BeforeEach - void open() { - when(searchEngine.forType(Group.class).getOrCreate()).thenReturn(index); - } + verify(index).store(Id.of(astronauts), GroupPermissions.read(astronauts).asShiroString(), astronauts); + } - @Test - void shouldStore() { - indexer.open().store(group); + @Test + void shouldDeleteGroup() { + indexer.createDeleteTask(astronauts).update(index); - verify(index).store(Id.of(group), "group:read:astronauts", group); - } + verify(index.delete()).byId(Id.of(astronauts)); + } - @Test - void shouldDeleteById() { - indexer.open().delete(group); + @Test + void shouldReIndexAll() { + when(groupManager.getAll()).thenReturn(Arrays.asList(astronauts, planetCreators)); - verify(index.delete().byType()).byId(Id.of(group)); - } + GroupIndexer.ReIndexAll reIndexAll = new GroupIndexer.ReIndexAll(indexLogStore, groupManager); + reIndexAll.update(index); - @Test - void shouldReIndexAll() { - when(groupManager.getAll()).thenReturn(singletonList(group)); + verify(index.delete()).all(); + verify(index).store(Id.of(astronauts), GroupPermissions.read(astronauts).asShiroString(), astronauts); + verify(index).store(Id.of(planetCreators), GroupPermissions.read(planetCreators).asShiroString(), planetCreators); + } - indexer.open().reIndexAll(); + @Test + void shouldHandleEvents() { + GroupEvent event = new GroupEvent(HandlerEventType.DELETE, astronauts); - verify(index.delete().byType()).all(); - verify(index).store(Id.of(group), "group:read:astronauts", group); - } + indexer.handleEvent(event); - @Test - void shouldHandleEvent() { - GroupEvent event = new GroupEvent(HandlerEventType.DELETE, group); - - indexer.handleEvent(event); - - verify(index.delete().byType()).byId(Id.of(group)); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + verify(searchEngine.forType(Group.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byId(Id.of(astronauts)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java index acdb64c7c8..7114296c69 100644 --- a/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java @@ -24,20 +24,25 @@ package sonia.scm.repository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +59,15 @@ class RepositoryIndexerTest { @InjectMocks private RepositoryIndexer indexer; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + @Test void shouldReturnRepositoryClass() { assertThat(indexer.getType()).isEqualTo(Repository.class); @@ -64,61 +78,65 @@ class RepositoryIndexerTest { assertThat(indexer.getVersion()).isEqualTo(RepositoryIndexer.VERSION); } - @Nested - class UpdaterTests { - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; - - private Repository repository; - - @BeforeEach - void open() { - when(searchEngine.forType(Repository.class).getOrCreate()).thenReturn(index); - repository = new Repository(); - repository.setId("42"); - } - - @Test - void shouldStoreRepository() { - indexer.open().store(repository); - - verify(index).store(Id.of(repository), "repository:read:42", repository); - } - - @Test - void shouldDeleteByRepository() { - indexer.open().delete(repository); - - verify(index.delete().allTypes()).byRepository("42"); - } - - @Test - void shouldReIndexAll() { - when(repositoryManager.getAll()).thenReturn(singletonList(repository)); - - indexer.open().reIndexAll(); - - verify(index.delete().allTypes()).byTypeName(Repository.class.getName()); - verify(index.delete().byType()).all(); - - verify(index).store(Id.of(repository), "repository:read:42", repository); - } - - @Test - void shouldHandleEvent() { - RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, repository); - - indexer.handleEvent(event); - - verify(index.delete().allTypes()).byRepository("42"); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(RepositoryIndexer.ReIndexAll.class); } + + @Test + void shouldCreateRepository() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + + indexer.createStoreTask(heartOfGold).update(index); + + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + } + + @Test + void shouldDeleteRepository() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + + indexer.createDeleteTask(heartOfGold).update(index); + + verify(index.delete()).byRepository(heartOfGold); + } + + @Test + void shouldReIndexAll() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle = RepositoryTestData.create42Puzzle(); + when(repositoryManager.getAll()).thenReturn(Arrays.asList(heartOfGold, puzzle)); + + RepositoryIndexer.ReIndexAll reIndexAll = new RepositoryIndexer.ReIndexAll(indexLogStore, repositoryManager); + reIndexAll.update(index); + + verify(index.delete()).all(); + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + verify(index).store(Id.of(puzzle), RepositoryPermissions.read(puzzle).asShiroString(), puzzle); + } + + @Test + void shouldHandleDeleteEvents() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, heartOfGold); + + indexer.handleEvent(event); + + verify(searchEngine.forIndices().forResource(heartOfGold)).batch(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byRepository(heartOfGold); + } + + @Test + void shouldHandleUpdateEvents() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.CREATE, heartOfGold); + + indexer.handleEvent(event); + + verify(searchEngine.forType(Repository.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + } + } diff --git a/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java b/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java index 360788bf22..411a9d693e 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java @@ -24,10 +24,12 @@ package sonia.scm.search; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.Answers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; @@ -35,6 +37,7 @@ import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryEvent; import sonia.scm.repository.RepositoryTestData; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -42,18 +45,23 @@ import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class HandlerEventIndexSyncerTest { + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; + @Mock private Indexer indexer; - @Mock - private Indexer.Updater updater; + @BeforeEach + void setUpIndexer() { + lenient().when(indexer.getType()).thenReturn(Repository.class); + } @ParameterizedTest @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.MATCH_ANY, names = "BEFORE_.*") void shouldIgnoreBeforeEvents(HandlerEventType type) { RepositoryEvent event = new RepositoryEvent(type, RepositoryTestData.create42Puzzle()); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); verifyNoInteractions(indexer); } @@ -61,28 +69,29 @@ class HandlerEventIndexSyncerTest { @ParameterizedTest @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.INCLUDE, names = {"CREATE", "MODIFY"}) void shouldStore(HandlerEventType type) { - when(indexer.open()).thenReturn(updater); + SerializableIndexTask store = index -> {}; Repository puzzle = RepositoryTestData.create42Puzzle(); + when(indexer.createStoreTask(puzzle)).thenReturn(store); + RepositoryEvent event = new RepositoryEvent(type, puzzle); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); - verify(updater).store(puzzle); - verify(updater).close(); + verify(searchEngine.forType(Repository.class)).update(store); } @Test void shouldDelete() { - when(indexer.open()).thenReturn(updater); + SerializableIndexTask delete = index -> {}; Repository puzzle = RepositoryTestData.create42Puzzle(); + when(indexer.createDeleteTask(puzzle)).thenReturn(delete); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, puzzle); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); - - verify(updater).delete(puzzle); - verify(updater).close(); + verify(searchEngine.forType(Repository.class)).update(delete); } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java index b6222c9687..a91d11a05f 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java @@ -32,8 +32,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.group.Group; import sonia.scm.repository.Repository; import sonia.scm.user.User; -import sonia.scm.web.security.AdministrationContext; -import sonia.scm.web.security.PrivilegedAction; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -42,47 +40,39 @@ import java.util.HashSet; import java.util.Optional; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class IndexBootstrapListenerTest { - @Mock - private AdministrationContext administrationContext; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; @Mock(answer = Answers.RETURNS_DEEP_STUBS) private IndexLogStore indexLogStore; @Test void shouldReIndexWithoutLog() { - mockAdminContext(); Indexer indexer = indexer(Repository.class, 1); - Indexer.Updater updater = updater(indexer); mockEmptyIndexLog(Repository.class); doInitialization(indexer); - verify(updater).reIndexAll(); - verify(updater).close(); - verify(indexLogStore.defaultIndex()).log(Repository.class, 1); + verify(searchEngine.forType(Repository.class)).update(RepositoryReIndexAllTask.class); } @Test void shouldReIndexIfVersionWasUpdated() { - mockAdminContext(); Indexer indexer = indexer(User.class, 2); - Indexer.Updater updater = updater(indexer); mockIndexLog(User.class, 1); doInitialization(indexer); - verify(updater).reIndexAll(); - verify(updater).close(); - verify(indexLogStore.defaultIndex()).log(User.class, 2); + verify(searchEngine.forType(User.class)).update(UserReIndexAllTask.class); } @Test @@ -92,7 +82,7 @@ class IndexBootstrapListenerTest { mockIndexLog(Group.class, 3); doInitialization(indexer); - verify(indexer, never()).open(); + verifyNoInteractions(searchEngine); } private void mockIndexLog(Class type, int version) { @@ -107,13 +97,6 @@ class IndexBootstrapListenerTest { when(indexLogStore.defaultIndex().get(type)).thenReturn(Optional.ofNullable(indexLog)); } - private void mockAdminContext() { - doAnswer(ic -> { - PrivilegedAction action = ic.getArgument(0); - action.run(); - return null; - }).when(administrationContext).runAsAdmin(any(PrivilegedAction.class)); - } @SuppressWarnings("rawtypes") private void doInitialization(Indexer... indexers) { @@ -125,7 +108,7 @@ class IndexBootstrapListenerTest { @SuppressWarnings("rawtypes") private IndexBootstrapListener listener(Indexer... indexers) { return new IndexBootstrapListener( - administrationContext, indexLogStore, new HashSet<>(Arrays.asList(indexers)) + searchEngine, indexLogStore, new HashSet<>(Arrays.asList(indexers)) ); } @@ -133,15 +116,38 @@ class IndexBootstrapListenerTest { private Indexer indexer(Class type, int version) { Indexer indexer = mock(Indexer.class); when(indexer.getType()).thenReturn(type); - when(indexer.getVersion()).thenReturn(version); + lenient().when(indexer.getVersion()).thenReturn(version); + lenient().when(indexer.getReIndexAllTask()).thenAnswer(ic -> { + if (type == User.class) { + return UserReIndexAllTask.class; + } + return RepositoryReIndexAllTask.class; + }); return indexer; } - @SuppressWarnings("unchecked") - private Indexer.Updater updater(Indexer indexer) { - Indexer.Updater updater = mock(Indexer.Updater.class); - when(indexer.open()).thenReturn(updater); - return updater; + public static class RepositoryReIndexAllTask extends Indexer.ReIndexAllTask { + + public RepositoryReIndexAllTask(IndexLogStore logStore, Class type, int version) { + super(logStore, type, version); + } + + @Override + public void update(Index index) { + + } + } + + public static class UserReIndexAllTask extends Indexer.ReIndexAllTask { + + public UserReIndexAllTask(IndexLogStore logStore, Class type, int version) { + super(logStore, type, version); + } + + @Override + public void update(Index index) { + + } } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java new file mode 100644 index 0000000000..c3b9cd5e6d --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java @@ -0,0 +1,188 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.core.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.assertj.core.api.AssertionsForClassTypes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; +import sonia.scm.plugin.PluginLoader; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexManagerTest { + + private Path directory; + + @Mock + private AnalyzerFactory analyzerFactory; + + @Mock + private LuceneSearchableType searchableType; + + @Mock + private SCMContextProvider context; + + @Mock + private PluginLoader pluginLoader; + + private IndexManager indexManager; + + @BeforeEach + void createIndexWriterFactory(@TempDir Path tempDirectory) { + this.directory = tempDirectory; + when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory.resolve("index")); + when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); + when(pluginLoader.getUberClassLoader()).thenReturn(IndexManagerTest.class.getClassLoader()); + indexManager = new IndexManager(context, pluginLoader, analyzerFactory); + } + + @Test + void shouldCreateNewIndex() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("index").resolve("songs").resolve("new-index")).exists(); + } + + @Test + void shouldCreateNewIndexForEachType() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "new-index")) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("index").resolve("songs").resolve("new-index")).exists(); + assertThat(directory.resolve("index").resolve("lyrics").resolve("new-index")).exists(); + } + + @Test + void shouldReturnAllCreatedIndices() throws IOException { + try (IndexWriter writer = open(Songs.class, "special")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "awesome")) { + addDoc(writer, "Trillian"); + } + + assertThat(indexManager.all()) + .anySatisfy(details -> { + assertThat(details.getType()).isEqualTo(Songs.class); + assertThat(details.getName()).isEqualTo("special"); + }) + .anySatisfy(details -> { + assertThat(details.getType()).isEqualTo(Lyrics.class); + assertThat(details.getName()).isEqualTo("awesome"); + }); + } + + @Test + void shouldRestoreIndicesOnCreation() throws IOException { + try (IndexWriter writer = open(Songs.class, "special")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "awesome")) { + addDoc(writer, "Trillian"); + } + + assertThat(new IndexManager(context, pluginLoader, analyzerFactory).all()) + .anySatisfy(details -> { + AssertionsForClassTypes.assertThat(details.getType()).isEqualTo(Songs.class); + AssertionsForClassTypes.assertThat(details.getName()).isEqualTo("special"); + }) + .anySatisfy(details -> { + AssertionsForClassTypes.assertThat(details.getType()).isEqualTo(Lyrics.class); + AssertionsForClassTypes.assertThat(details.getName()).isEqualTo("awesome"); + }); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private IndexWriter open(Class type, String indexName) throws IOException { + lenient().when(searchableType.getType()).thenReturn(type); + when(searchableType.getName()).thenReturn(type.getSimpleName().toLowerCase(Locale.ENGLISH)); + return indexManager.openForWrite(new IndexParams(indexName, searchableType, IndexOptions.defaults())); + } + + @Test + void shouldOpenExistingIndex() throws IOException { + try (IndexWriter writer = open(Songs.class, "reused")) { + addDoc(writer, "Dent"); + } + try (IndexWriter writer = open(Songs.class, "reused")) { + assertThat(writer.getFieldNames()).contains("hitchhiker"); + } + } + + @Test + void shouldUseAnalyzerFromFactory() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); + } + } + + @Test + void shouldOpenIndexForRead() throws IOException { + try (IndexWriter writer = open(Songs.class, "idx-for-read")) { + addDoc(writer, "Dent"); + } + + try (IndexReader reader = indexManager.openForRead(searchableType, "idx-for-read")) { + assertThat(reader.numDocs()).isOne(); + } + } + + private void addDoc(IndexWriter writer, String name) throws IOException { + Document doc = new Document(); + doc.add(new TextField("hitchhiker", name, Field.Store.YES)); + writer.addDocument(doc); + } + + public static class Songs { + } + + public static class Lyrics { + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java deleted file mode 100644 index 929842104f..0000000000 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import org.apache.lucene.analysis.core.SimpleAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import sonia.scm.SCMContextProvider; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class IndexOpenerTest { - - private Path directory; - - @Mock - private AnalyzerFactory analyzerFactory; - - @Mock - private LuceneSearchableType searchableType; - - private IndexOpener indexOpener; - - @BeforeEach - void createIndexWriterFactory(@TempDir Path tempDirectory) { - this.directory = tempDirectory; - SCMContextProvider context = mock(SCMContextProvider.class); - when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory); - when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); - indexOpener = new IndexOpener(context, analyzerFactory); - } - - @Test - void shouldCreateNewIndex() throws IOException { - try (IndexWriter writer = open("new-index")) { - addDoc(writer, "Trillian"); - } - assertThat(directory.resolve("new-index")).exists(); - } - - private IndexWriter open(String index) throws IOException { - return indexOpener.openForWrite(new IndexParams(index, searchableType, IndexOptions.defaults())); - } - - @Test - void shouldOpenExistingIndex() throws IOException { - try (IndexWriter writer = open("reused")) { - addDoc(writer, "Dent"); - } - try (IndexWriter writer = open("reused")) { - assertThat(writer.getFieldNames()).contains("hitchhiker"); - } - } - - @Test - void shouldUseAnalyzerFromFactory() throws IOException { - try (IndexWriter writer = open("new-index")) { - assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); - } - } - - @Test - void shouldOpenIndexForRead() throws IOException { - try (IndexWriter writer = open("idx-for-read")) { - addDoc(writer, "Dent"); - } - - try (IndexReader reader = indexOpener.openForRead("idx-for-read")) { - assertThat(reader.numDocs()).isOne(); - } - } - - private void addDoc(IndexWriter writer, String name) throws IOException { - Document doc = new Document(); - doc.add(new TextField("hitchhiker", name, Field.Store.YES)); - writer.addDocument(doc); - } - -} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java deleted file mode 100644 index 660b97bf21..0000000000 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import lombok.Value; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.in; -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class IndexQueueTest { - - private Directory directory; - - private IndexQueue queue; - - @BeforeEach - void createQueue() throws IOException { - directory = new ByteBuffersDirectory(); - IndexOpener opener = mock(IndexOpener.class); - when(opener.openForWrite(any(IndexParams.class))).thenAnswer(ic -> { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory, config); - }); - - LuceneIndexFactory indexFactory = new LuceneIndexFactory(opener); - queue = new IndexQueue(indexFactory); - } - - @AfterEach - void closeQueue() throws IOException { - queue.close(); - directory.close(); - } - - @Test - void shouldWriteToIndex() throws Exception { - try (Index index = getIndex(Account.class)) { - index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan")); - index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent")); - } - assertDocCount(2); - } - - private Index getIndex(Class type) { - SearchableTypeResolver resolver = new SearchableTypeResolver(type); - LuceneSearchableType searchableType = resolver.resolve(type); - IndexParams indexParams = new IndexParams("default", searchableType, IndexOptions.defaults()); - return queue.getQueuedIndex(indexParams); - } - - @Test - void shouldWriteMultiThreaded() throws Exception { - ExecutorService executorService = Executors.newFixedThreadPool(4); - for (int i = 0; i < 20; i++) { - executorService.execute(new IndexNumberTask(i)); - } - executorService.execute(() -> { - try (Index index = getIndex(IndexedNumber.class)) { - index.delete().byType().byId(Id.of(String.valueOf(12))); - } - }); - executorService.shutdown(); - - assertDocCount(19); - } - - private void assertDocCount(int expectedCount) throws IOException { - // wait until all tasks are finished - await().until(() -> queue.getSize() == 0); - try (DirectoryReader reader = DirectoryReader.open(directory)) { - assertThat(reader.numDocs()).isEqualTo(expectedCount); - } - } - - @Value - @IndexedType - public static class Account { - @Indexed - String username; - @Indexed - String firstName; - @Indexed - String lastName; - } - - @Value - @IndexedType - public static class IndexedNumber { - @Indexed - int value; - } - - public class IndexNumberTask implements Runnable { - - private final int number; - - public IndexNumberTask(int number) { - this.number = number; - } - - @Override - public void run() { - try (Index index = getIndex(IndexedNumber.class)) { - index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number)); - } - } - } - -} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java new file mode 100644 index 0000000000..dbca039857 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.user.User; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class LuceneIndexFactoryTest { + + @Mock + private IndexManager indexManager; + + @InjectMocks + private LuceneIndexFactory indexFactory; + + @Test + void shouldCallOpenOnCreation() { + LuceneIndex index = indexFactory.create(params("default", Repository.class)); + assertThat(index.getWriter().getUsageCounter()).isOne(); + } + + @Test + void shouldCallOpenOnReturn() { + indexFactory.create(params("default", Repository.class)); + indexFactory.create(params("default", Repository.class)); + LuceneIndex index = indexFactory.create(params("default", Repository.class)); + assertThat(index.getWriter().getUsageCounter()).isEqualTo(3); + } + + @Test + @SuppressWarnings("AssertBetweenInconvertibleTypes") + void shouldReturnDifferentIndexForDifferentTypes() { + LuceneIndex repository = indexFactory.create(params("default", Repository.class)); + LuceneIndex user = indexFactory.create(params("default", User.class)); + + assertThat(repository).isNotSameAs(user); + } + + @Test + void shouldReturnDifferentIndexForDifferentIndexNames() { + LuceneIndex def = indexFactory.create(params("default", Repository.class)); + LuceneIndex other = indexFactory.create(params("other", Repository.class)); + + assertThat(def).isNotSameAs(other); + } + + @Test + void shouldReturnSameIndex() { + LuceneIndex one = indexFactory.create(params("default", Repository.class)); + LuceneIndex two = indexFactory.create(params("default", Repository.class)); + assertThat(one).isSameAs(two); + } + + private IndexParams params(String indexName, Class type) { + return new IndexParams(indexName, SearchableTypes.create(type), IndexOptions.defaults()); + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java index 701536ca31..f39e997f72 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java @@ -27,7 +27,6 @@ package sonia.scm.search; import com.google.errorprone.annotations.CanIgnoreReturnValue; import lombok.Value; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -39,12 +38,26 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryTestData; +import sonia.scm.repository.RepositoryType; import java.io.IOException; +import java.util.function.Supplier; import static org.assertj.core.api.Assertions.assertThat; -import static sonia.scm.search.FieldNames.*; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; +import static sonia.scm.search.FieldNames.ID; +import static sonia.scm.search.FieldNames.PERMISSION; +import static sonia.scm.search.FieldNames.REPOSITORY; class LuceneIndexTest { @@ -77,15 +90,6 @@ class LuceneIndexTest { assertHits("value", "content", 1); } - @Test - void shouldStoreUidOfObject() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Awesome content which should be indexed")); - } - - assertHits(UID, "one/storable", 1); - } - @Test void shouldStoreIdOfObject() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { @@ -104,15 +108,6 @@ class LuceneIndexTest { assertHits(REPOSITORY, "4211", 1); } - @Test - void shouldStoreTypeOfObject() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some other text")); - } - - assertHits(TYPE, "storable", 1); - } - @Test void shouldDeleteById() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { @@ -120,131 +115,54 @@ class LuceneIndexTest { } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(ONE); + index.delete().byId(ONE); } assertHits(ID, "one", 0); } @Test - void shouldDeleteAllByType() throws IOException { + void shouldDeleteByIdAndRepository() throws IOException { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle42 = RepositoryTestData.createHeartOfGold(); try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("content")); - index.store(Id.of("two"), null, new Storable("content")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(Id.of("three"), null, new OtherStorable("content")); + index.store(ONE.withRepository(heartOfGold), null, new Storable("content")); + index.store(ONE.withRepository(puzzle42), null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().all(); + index.delete().byId(ONE.withRepository(heartOfGold)); } assertHits("value", "content", 1); } @Test - void shouldDeleteByIdAnyType() throws IOException { + void shouldDeleteAll() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE, null, new OtherStorable("Some other text")); + index.store(ONE, null, new Storable("content")); + index.store(TWO, null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(ONE); + index.delete().all(); } - assertHits(ID, "one", 1); - ScoreDoc[] docs = assertHits(ID, "one", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get("value")).isEqualTo("Some other text"); - } - - @Test - void shouldDeleteByIdAndRepository() throws IOException { - Id withRepository = ONE.withRepository("4211"); - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some other text")); - index.store(withRepository, null, new Storable("New stuff")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(withRepository); - } - - ScoreDoc[] docs = assertHits(ID, "one", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get("value")).isEqualTo("Some other text"); + assertHits("value", "content", 0); } @Test void shouldDeleteByRepository() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("Some other text")); - index.store(ONE.withRepository("4212"), null, new Storable("New stuff")); + index.store(ONE.withRepository("4211"), null, new Storable("content")); + index.store(TWO.withRepository("4212"), null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byRepository("4212"); + index.delete().byRepository("4212"); } - assertHits(ID, "one", 1); - } - - @Test - void shouldDeleteByRepositoryAndType() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("some text")); - index.store(TWO.withRepository("4211"), null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE.withRepository("4211"), null, new OtherStorable("some text")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byRepository("4211"); - } - - ScoreDoc[] docs = assertHits("value", "text", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get(TYPE)).isEqualTo("otherStorable"); - } - - @Test - void shouldDeleteAllByRepository() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("some text")); - index.store(TWO.withRepository("4211"), null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE.withRepository("4211"), null, new OtherStorable("some text")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().allTypes().byRepository("4211"); - } - - assertHits("value", "text", 0); - } - - @Test - void shouldDeleteAllByTypeName() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("some text")); - index.store(TWO, null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.delete().allTypes().byTypeName("storable"); - } - - assertHits("value", "text", 0); + assertHits("value", "content", 1); } @Test @@ -256,12 +174,69 @@ class LuceneIndexTest { assertHits(PERMISSION, "repo:4211:read", 1); } - private Document doc(int doc) throws IOException { - try (DirectoryReader reader = DirectoryReader.open(directory)) { - return reader.document(doc); + @Test + void shouldReturnDetails() { + try (LuceneIndex index = createIndex(Storable.class)) { + IndexDetails details = index.getDetails(); + assertThat(details.getType()).isEqualTo(Storable.class); + assertThat(details.getName()).isEqualTo("default"); } } + @Nested + @ExtendWith(MockitoExtension.class) + class ExceptionTests { + + @Mock + private IndexWriter writer; + + private LuceneIndex index; + + @BeforeEach + void setUpIndex() { + index = createIndex(Storable.class, () -> writer); + } + + @Test + void shouldThrowSearchEngineExceptionOnStore() throws IOException { + when(writer.updateDocument(any(), any())).thenThrow(new IOException("failed to store")); + + Storable storable = new Storable("Some other text"); + assertThrows(SearchEngineException.class, () -> index.store(ONE, null, storable)); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteById() throws IOException { + when(writer.deleteDocuments(any(Term.class))).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, () -> deleter.byId(ONE)); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteAll() throws IOException { + when(writer.deleteAll()).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, deleter::all); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteByRepository() throws IOException { + when(writer.deleteDocuments(any(Term.class))).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, () -> deleter.byRepository("42")); + } + + @Test + void shouldThrowSearchEngineExceptionOnClose() throws IOException { + doThrow(new IOException("failed to delete")).when(writer).close(); + assertThrows(SearchEngineException.class, () -> index.close()); + } + + } + @CanIgnoreReturnValue private ScoreDoc[] assertHits(String field, String value, int expectedHits) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { @@ -272,15 +247,25 @@ class LuceneIndexTest { } } - private LuceneIndex createIndex(Class type) throws IOException { - SearchableTypeResolver resolver = new SearchableTypeResolver(type); - return new LuceneIndex<>(resolver.resolve(type), createWriter()); + private LuceneIndex createIndex(Class type) { + return createIndex(type, this::createWriter); } - private IndexWriter createWriter() throws IOException { + private LuceneIndex createIndex(Class type, Supplier writerFactor) { + SearchableTypeResolver resolver = new SearchableTypeResolver(type); + return new LuceneIndex<>( + new IndexParams("default", resolver.resolve(type), IndexOptions.defaults()), writerFactor + ); + } + + private IndexWriter createWriter() { IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory, config); + try { + return new IndexWriter(directory, config); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index writer", ex); + } } @Value @@ -290,11 +275,4 @@ class LuceneIndexTest { String value; } - @Value - @IndexedType - private static class OtherStorable { - @Indexed - String value; - } - } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java new file mode 100644 index 0000000000..a20bd85a6d --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java @@ -0,0 +1,98 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.user.User; + +import javax.inject.Inject; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class LuceneInjectingIndexTaskTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private LuceneIndexFactory indexFactory; + + private static String capturedValue; + + private final SearchableTypeResolver resolver = new SearchableTypeResolver(User.class); + + @BeforeEach + void cleanUp() { + capturedValue = "notAsExpected"; + } + + @Test + void shouldInjectAndUpdate() { + Injector injector = createInjector(); + + LuceneSearchableType searchableType = resolver.resolve(User.class); + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + + LuceneInjectingIndexTask task = new LuceneInjectingIndexTask(params, InjectingTask.class); + injector.injectMembers(task); + + task.run(); + + assertThat(capturedValue).isEqualTo("runAsExpected"); + } + + private Injector createInjector() { + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(SearchableTypeResolver.class).toInstance(resolver); + bind(LuceneIndexFactory.class).toInstance(indexFactory); + bind(String.class).toInstance("runAsExpected"); + } + }); + } + + public static class InjectingTask implements IndexTask { + + private final String value; + + @Inject + public InjectingTask(String value) { + this.value = value; + } + + @Override + public void update(Index index) { + capturedValue = value; + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java index 77844de063..5f9cebaf87 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java @@ -42,7 +42,6 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.shiro.authz.AuthorizationException; import org.github.sdorra.jse.ShiroExtension; import org.github.sdorra.jse.SubjectAware; import org.junit.jupiter.api.BeforeEach; @@ -70,7 +69,7 @@ class LuceneQueryBuilderTest { private Directory directory; @Mock - private IndexOpener opener; + private IndexManager opener; @BeforeEach void setUpDirectory() { @@ -181,17 +180,6 @@ class LuceneQueryBuilderTest { assertThat(result.getTotalHits()).isOne(); } - @Test - void shouldIgnoreHitsOfOtherType() throws IOException { - try (IndexWriter writer = writer()) { - writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); - writer.addDocument(personDoc("Dent")); - } - - QueryResult result = query(InetOrgPerson.class, "Dent"); - assertThat(result.getTotalHits()).isOne(); - } - @Test void shouldThrowQueryParseExceptionOnInvalidQuery() throws IOException { try (IndexWriter writer = writer()) { @@ -251,17 +239,6 @@ class LuceneQueryBuilderTest { assertThat(result.getHits()).hasSize(3); } - @Test - void shouldReturnOnlyHitsOfTypeForExpertQuery() throws IOException { - try (IndexWriter writer = writer()) { - writer.addDocument(inetOrgPersonDoc("Ford", "Prefect", "Ford Prefect", "4211")); - writer.addDocument(personDoc("Prefect")); - } - - QueryResult result = query(InetOrgPerson.class, "lastName:prefect"); - assertThat(result.getTotalHits()).isEqualTo(1L); - } - @Test void shouldReturnOnlyPermittedHits() throws IOException { try (IndexWriter writer = writer()) { @@ -302,10 +279,11 @@ class LuceneQueryBuilderTest { QueryResult result; try (DirectoryReader reader = DirectoryReader.open(directory)) { - when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class); + LuceneSearchableType searchableType = resolver.resolve(Simple.class); + when(opener.openForRead(searchableType, "default")).thenReturn(reader); LuceneQueryBuilder builder = new LuceneQueryBuilder<>( - opener, "default", resolver.resolve(Simple.class), new StandardAnalyzer() + opener, "default", searchableType, new StandardAnalyzer() ); result = builder.repository("cde").execute("content:awesome"); } @@ -560,9 +538,9 @@ class LuceneQueryBuilderTest { private long count(Class type, String queryString) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { - lenient().when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(type); LuceneSearchableType searchableType = resolver.resolve(type); + lenient().when(opener.openForRead(searchableType, "default")).thenReturn(reader); LuceneQueryBuilder builder = new LuceneQueryBuilder( opener, "default", searchableType, new StandardAnalyzer() ); @@ -572,10 +550,11 @@ class LuceneQueryBuilderTest { private QueryResult query(Class type, String queryString, Integer start, Integer limit) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { - lenient().when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(type); LuceneSearchableType searchableType = resolver.resolve(type); - LuceneQueryBuilder builder = new LuceneQueryBuilder( + + lenient().when(opener.openForRead(searchableType, "default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder<>( opener, "default", searchableType, new StandardAnalyzer() ); if (start != null) { @@ -597,14 +576,14 @@ class LuceneQueryBuilderTest { private Document simpleDoc(String content) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); return document; } private Document permissionDoc(String content, String permission) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES)); return document; } @@ -612,7 +591,7 @@ class LuceneQueryBuilderTest { private Document repositoryDoc(String content, String repository) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES)); return document; } @@ -624,14 +603,14 @@ class LuceneQueryBuilderTest { document.add(new TextField("displayName", displayName, Field.Store.YES)); document.add(new TextField("carLicense", carLicense, Field.Store.YES)); document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES)); return document; } private Document personDoc(String lastName) { Document document = new Document(); document.add(new TextField("lastName", lastName, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES)); return document; } @@ -644,14 +623,14 @@ class LuceneQueryBuilderTest { document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES)); document.add(new LongPoint("instantValue", instantValue.toEpochMilli())); document.add(new StoredField("instantValue", instantValue.toEpochMilli())); - document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES)); return document; } private Document denyDoc(String value) { Document document = new Document(); document.add(new TextField("value", value, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "deny", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "deny", Field.Store.YES)); return document; } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java index e30b67c1f6..f4e1193f3f 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java @@ -27,12 +27,20 @@ package sonia.scm.search; import org.apache.shiro.authz.AuthorizationException; import org.github.sdorra.jse.ShiroExtension; import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.repository.Repository; +import sonia.scm.user.User; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.Task; import java.util.Arrays; import java.util.Collection; @@ -46,176 +54,329 @@ import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SubjectAware("trillian") @ExtendWith({MockitoExtension.class, ShiroExtension.class}) class LuceneSearchEngineTest { + @Mock + private IndexManager indexManager; + @Mock private SearchableTypeResolver resolver; - @Mock - private IndexQueue indexQueue; - @Mock private LuceneQueryBuilderFactory queryBuilderFactory; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private CentralWorkQueue centralWorkQueue; + @InjectMocks private LuceneSearchEngine searchEngine; @Mock private LuceneSearchableType searchableType; - @Test - void shouldDelegateGetSearchableTypes() { - List mockedTypes = Collections.singletonList(searchableType("repository")); - when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + @Nested + class GetSearchableTypesTests { - Collection searchableTypes = searchEngine.getSearchableTypes(); + @Test + void shouldDelegateGetSearchableTypes() { + List mockedTypes = Collections.singletonList(searchableType("repository")); + when(resolver.getSearchableTypes()).thenReturn(mockedTypes); - assertThat(searchableTypes).containsAll(mockedTypes); + Collection searchableTypes = searchEngine.getSearchableTypes(); + + assertThat(searchableTypes).containsAll(mockedTypes); + } + + @Test + @SubjectAware(value = "dent", permissions = "user:list") + void shouldExcludeTypesWithoutPermission() { + LuceneSearchableType repository = searchableType("repository"); + LuceneSearchableType user = searchableType("user", "user:list"); + LuceneSearchableType group = searchableType("group", "group:list"); + List mockedTypes = Arrays.asList(repository, user, group); + when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + + Collection searchableTypes = searchEngine.getSearchableTypes(); + + assertThat(searchableTypes).containsOnly(repository, user); + } + + private LuceneSearchableType searchableType(String name) { + return searchableType(name, null); + } + + private LuceneSearchableType searchableType(String name, String permission) { + LuceneSearchableType searchableType = mock(LuceneSearchableType.class); + lenient().when(searchableType.getName()).thenReturn(name); + when(searchableType.getPermission()).thenReturn(Optional.ofNullable(permission)); + return searchableType; + } } - @Test - @SubjectAware(value = "dent", permissions = "user:list") - void shouldExcludeTypesWithoutPermission() { - LuceneSearchableType repository = searchableType("repository"); - LuceneSearchableType user = searchableType("user", "user:list"); - LuceneSearchableType group = searchableType("group", "group:list"); - List mockedTypes = Arrays.asList(repository, user, group); - when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + @Nested + class SearchTests { - Collection searchableTypes = searchEngine.getSearchableTypes(); + @Test + @SuppressWarnings("unchecked") + void shouldDelegateSearchWithDefaults() { + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); - assertThat(searchableTypes).containsOnly(repository, user); + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); + + QueryBuilder queryBuilder = searchEngine.forType(Repository.class).search(); + + assertThat(queryBuilder).isSameAs(mockedBuilder); + } + + @Test + @SuppressWarnings("unchecked") + void shouldDelegateSearch() { + IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + IndexParams params = new IndexParams("idx", searchableType, options); + when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); + + QueryBuilder queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search(); + + assertThat(queryBuilder).isSameAs(mockedBuilder); + } + + @Test + void shouldFailWithoutRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + SearchEngine.ForType forType = searchEngine.forType(Repository.class); + assertThrows(AuthorizationException.class, forType::search); + } + + @Test + @SuppressWarnings("unchecked") + @SubjectAware(permissions = "repository:read") + void shouldNotFailWithRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); + + SearchEngine.ForType forType = searchEngine.forType(Repository.class); + assertThat(forType.search()).isNotNull(); + } + + @Test + void shouldFailWithTypeNameWithoutRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolveByName("repository")).thenReturn(searchableType); + + SearchEngine.ForType forType = searchEngine.forType("repository"); + assertThrows(AuthorizationException.class, forType::search); + } + + @Test + @SuppressWarnings("unchecked") + @SubjectAware(permissions = "repository:read") + void shouldNotFailWithTypeNameAndRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolveByName("repository")).thenReturn(searchableType); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); + + SearchEngine.ForType forType = searchEngine.forType("repository"); + assertThat(forType.search()).isNotNull(); + } } - private LuceneSearchableType searchableType(String name) { - return searchableType(name, null); + @Nested + class IndexTests { + + @Captor + private ArgumentCaptor taskCaptor; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private CentralWorkQueue.Enqueue enqueue; + + @BeforeEach + void setUp() { + when(centralWorkQueue.append()).thenReturn(enqueue); + } + + @Test + void shouldSubmitSimpleTask() { + mockType(); + + searchEngine.forType(Repository.class).update(index -> {}); + + verifyTaskSubmitted(LuceneSimpleIndexTask.class); + } + + @Test + void shouldSubmitInjectingTask() { + mockType(); + + searchEngine.forType(Repository.class).update(DummyIndexTask.class); + + verifyTaskSubmitted(LuceneInjectingIndexTask.class); + } + + @Test + void shouldLockTypeAndDefaultIndex() { + mockType(); + + searchEngine.forType(Repository.class).update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index"); + } + + @Test + void shouldLockTypeAndIndex() { + mockType(); + + searchEngine.forType(Repository.class).withIndex("sample").update(DummyIndexTask.class); + + verify(enqueue).locks("repository-sample-index"); + } + + @Test + void shouldLockSpecificResource() { + mockType(); + + searchEngine.forType(Repository.class).forResource("one").update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index", "one"); + } + + @Test + void shouldLockMultipleSpecificResources() { + mockType(); + + searchEngine.forType(Repository.class) + .forResource("one") + .forResource("two") + .update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index", "one"); + verify(enqueue).locks("repository-default-index", "two"); + } + + @Test + void shouldBatchSimpleTask() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(index -> {}); + + verifyTaskSubmitted(LuceneSimpleIndexTask.class); + } + + @Test + void shouldBatchAndLock() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(index -> {}); + + verify(enqueue).locks("repository-default-index"); + } + + @Test + void shouldBatchAndLockSpecificResource() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().forResource("one").batch(index -> {}); + + verify(enqueue).locks("repository-default-index", "one"); + } + + @Test + void shouldBatchAndLockMultipleSpecificResources() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().forResource("one").forResource("two").batch(index -> {}); + + verify(enqueue).locks("repository-default-index", "one"); + verify(enqueue).locks("repository-default-index", "two"); + } + + @Test + void shouldBatchInjectingTask() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(DummyIndexTask.class); + + verifyTaskSubmitted(LuceneInjectingIndexTask.class); + } + + @Test + void shouldBatchMultipleTasks() { + mockDetails( + new LuceneIndexDetails(Repository.class, "default"), + new LuceneIndexDetails(User.class, "default") + ); + + searchEngine.forIndices().batch(index -> {}); + + verify(enqueue.runAsAdmin(), times(2)).enqueue(any(Task.class)); + } + + @Test + void shouldFilterWithPredicate() { + mockDetails( + new LuceneIndexDetails(Repository.class, "default"), + new LuceneIndexDetails(User.class, "default") + ); + + searchEngine.forIndices() + .matching(details -> details.getType() == Repository.class) + .batch(index -> {}); + + verify(enqueue.runAsAdmin()).enqueue(any(Task.class)); + } + + private void mockDetails(LuceneIndexDetails... details) { + for (LuceneIndexDetails detail : details) { + mockType(detail.getType()); + } + when(indexManager.all()).thenAnswer(ic -> Arrays.asList(details)); + } + + private void verifyTaskSubmitted(Class typeOfTask) { + verify(enqueue.runAsAdmin()).enqueue(taskCaptor.capture()); + + Task task = taskCaptor.getValue(); + assertThat(task).isInstanceOf(typeOfTask); + } + + private void mockType() { + mockType(Repository.class); + } + + private void mockType(Class type){ + LuceneSearchableType searchableType = mock(LuceneSearchableType.class); + lenient().when(searchableType.getType()).thenAnswer(ic -> type); + lenient().when(searchableType.getName()).thenReturn(type.getSimpleName().toLowerCase(Locale.ENGLISH)); + lenient().when(resolver.resolve(type)).thenReturn(searchableType); + } + } - private LuceneSearchableType searchableType(String name, String permission) { - LuceneSearchableType searchableType = mock(LuceneSearchableType.class); - lenient().when(searchableType.getName()).thenReturn(name); - when(searchableType.getPermission()).thenReturn(Optional.ofNullable(permission)); - return searchableType; - } - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateWithDefaultIndex() { - Index index = mock(Index.class); + public static class DummyIndexTask implements IndexTask { - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); + @Override + public void update(Index index) { - Index idx = searchEngine.forType(Repository.class).getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateIndexWithDefaults() { - Index index = mock(Index.class); - - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("idx", searchableType, IndexOptions.defaults()); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); - - Index idx = searchEngine.forType(Repository.class).withIndex("idx").getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateIndex() { - Index index = mock(Index.class); - IndexOptions options = IndexOptions.naturalLanguage(Locale.ENGLISH); - - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("default", searchableType, options); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); - - Index idx = searchEngine.forType(Repository.class).withOptions(options).getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateSearchWithDefaults() { - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); - when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); - - QueryBuilder queryBuilder = searchEngine.forType(Repository.class).search(); - - assertThat(queryBuilder).isSameAs(mockedBuilder); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateSearch() { - IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - IndexParams params = new IndexParams("idx", searchableType, options); - when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); - - QueryBuilder queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search(); - - assertThat(queryBuilder).isSameAs(mockedBuilder); - } - - @Test - void shouldFailWithoutRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - SearchEngine.ForType forType = searchEngine.forType(Repository.class); - assertThrows(AuthorizationException.class, forType::search); - } - - @Test - @SuppressWarnings("unchecked") - @SubjectAware(permissions = "repository:read") - void shouldNotFailWithRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); - - SearchEngine.ForType forType = searchEngine.forType(Repository.class); - assertThat(forType.search()).isNotNull(); - } - - @Test - void shouldFailWithTypeNameWithoutRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolveByName("repository")).thenReturn(searchableType); - - SearchEngine.ForType forType = searchEngine.forType("repository"); - assertThrows(AuthorizationException.class, forType::search); - } - - @Test - @SuppressWarnings("unchecked") - @SubjectAware(permissions = "repository:read") - void shouldNotFailWithTypeNameAndRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolveByName("repository")).thenReturn(searchableType); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); - - SearchEngine.ForType forType = searchEngine.forType("repository"); - assertThat(forType.search()).isNotNull(); + } } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java new file mode 100644 index 0000000000..96617a2842 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java @@ -0,0 +1,83 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LuceneSimpleIndexTaskTest { + + @Mock + private LuceneIndexFactory indexFactory; + + @Mock + private LuceneIndex index; + + private final SearchableTypeResolver resolver = new SearchableTypeResolver(Repository.class); + + @Test + void shouldUpdate() { + Injector injector = createInjector(); + + LuceneSearchableType searchableType = resolver.resolve(Repository.class); + + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + + AtomicReference> ref = new AtomicReference<>(); + LuceneSimpleIndexTask task = new LuceneSimpleIndexTask(params, ref::set); + injector.injectMembers(task); + + when(indexFactory.create(params)).then(ic -> index); + + task.run(); + + assertThat(index).isSameAs(ref.get()); + verify(index).close(); + } + + private Injector createInjector() { + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(SearchableTypeResolver.class).toInstance(resolver); + bind(LuceneIndexFactory.class).toInstance(indexFactory); + } + }); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java b/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java new file mode 100644 index 0000000000..327c058f7a --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java @@ -0,0 +1,249 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SharableIndexWriterTest { + + @Mock + private IndexWriter underlyingWriter; + + @Test + @SuppressWarnings("unchecked") + void shouldCreateIndexOnOpen() { + Supplier supplier = mock(Supplier.class); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + verifyNoInteractions(supplier); + + writer.open(); + verify(supplier).get(); + } + + @Test + @SuppressWarnings("unchecked") + void shouldOpenWriterOnlyOnce() { + Supplier supplier = mock(Supplier.class); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + writer.open(); + writer.open(); + writer.open(); + + verify(supplier).get(); + } + + @Test + void shouldIncreaseUsageCounter() { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + assertThat(writer.getUsageCounter()).isEqualTo(3); + } + + @Test + void shouldDecreaseUsageCounter() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + assertThat(writer.getUsageCounter()).isEqualTo(3); + + writer.close(); + writer.close(); + + assertThat(writer.getUsageCounter()).isOne(); + } + + @Test + void shouldNotCloseWriterIfUsageCounterIsGreaterZero() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + writer.close(); + writer.close(); + + verify(underlyingWriter, never()).close(); + } + + @Test + void shouldCloseIfUsageCounterIsZero() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + + writer.close(); + writer.close(); + + verify(underlyingWriter).close(); + } + + @Test + @SuppressWarnings("unchecked") + void shouldReOpen() throws IOException { + Supplier supplier = mock(Supplier.class); + when(supplier.get()).thenReturn(underlyingWriter); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + writer.open(); + + writer.close(); + writer.open(); + + verify(supplier, times(2)).get(); + verify(underlyingWriter).close(); + } + + @Test + void shouldDelegateUpdates() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + Term term = new Term("field", "value"); + Document document = new Document(); + writer.updateDocument(term, document); + + verify(underlyingWriter).updateDocument(term, document); + } + + @Test + void shouldDelegateDeleteAll() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + writer.deleteAll(); + + verify(underlyingWriter).deleteAll(); + } + + @Test + void shouldDelegateDeletes() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + Term term = new Term("field", "value"); + writer.deleteDocuments(term); + + verify(underlyingWriter).deleteDocuments(term); + } + + @Nested + class ConcurrencyTests { + + private ExecutorService executorService; + + private final AtomicInteger openCounter = new AtomicInteger(); + private final AtomicInteger commitCounter = new AtomicInteger(); + private final AtomicInteger closeCounter = new AtomicInteger(); + + private final AtomicInteger invocations = new AtomicInteger(); + + private SharableIndexWriter writer; + + @BeforeEach + void setUp() throws IOException { + executorService = Executors.newFixedThreadPool(4); + writer = new SharableIndexWriter(() -> { + openCounter.incrementAndGet(); + return underlyingWriter; + }); + + doAnswer(ic -> commitCounter.incrementAndGet()).when(underlyingWriter).commit(); + doAnswer(ic -> closeCounter.incrementAndGet()).when(underlyingWriter).close(); + } + + @Test + @SuppressWarnings("java:S2925") // sleep is ok to simulate some work + void shouldKeepIndexOpen() { + AtomicBoolean fail = new AtomicBoolean(false); + for (int i = 0; i < 50; i++) { + executorService.submit(() -> { + writer.open(); + try { + Thread.sleep(25); + writer.deleteAll(); + writer.close(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail.set(true); + } catch (IOException e) { + fail.set(true); + } finally { + invocations.incrementAndGet(); + } + }); + } + + executorService.shutdown(); + + await().atMost(2, TimeUnit.SECONDS).until(() -> invocations.get() == 50); + + assertThat(fail.get()).isFalse(); + + // It should be one, but it is possible that tasks finish before new added to the queue. + // This behaviour depends heavily on the cpu's of the machine which executes this test. + assertThat(openCounter.get()).isPositive().isLessThan(10); + // should be 49, but see comment above + assertThat(commitCounter.get()).isGreaterThan(40); + // should be 1, but see comment above + assertThat(closeCounter.get()).isPositive().isLessThan(10); + } + + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java b/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java new file mode 100644 index 0000000000..24b07c2485 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java @@ -0,0 +1,91 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.security; + + +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.UnavailableSecurityManagerException; +import org.apache.shiro.mgt.DefaultSecurityManager; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import sonia.scm.security.Impersonator.Session; + +import javax.annotation.Nonnull; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ImpersonatorTest { + + private SecurityManager securityManager = new DefaultSecurityManager(); + + private Impersonator impersonator; + + @BeforeEach + void setUp() { + impersonator = new Impersonator(securityManager); + } + + @Test + void shouldBindAndRestoreNonWebThread() { + try (Session session = impersonator.impersonate(principal("dent"))) { + assertPrincipal("dent"); + } + assertThrows(UnavailableSecurityManagerException.class, SecurityUtils::getSubject); + } + + @Nonnull + private SimplePrincipalCollection principal(String principal) { + return new SimplePrincipalCollection(principal, "test"); + } + + private void assertPrincipal(String principal) { + assertThat(SecurityUtils.getSubject().getPrincipal()).isEqualTo(principal); + } + + @Nested + @ExtendWith(ShiroExtension.class) + class WithSecurityManager { + + @Test + @SubjectAware("trillian") + void shouldBindAndRestoreWebThread() { + assertPrincipal("trillian"); + + try (Session session = impersonator.impersonate(principal("slarti"))) { + assertPrincipal("slarti"); + } + + assertPrincipal("trillian"); + } + + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java b/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java new file mode 100644 index 0000000000..0c34d1147f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java @@ -0,0 +1,91 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.update.index; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RemoveCombinedIndexTest { + + private Path home; + + @Mock + private SCMContextProvider contextProvider; + + @InjectMocks + private RemoveCombinedIndex updateStep; + + @BeforeEach + void setUp(@TempDir Path home) { + this.home = home; + when(contextProvider.resolve(any())).then( + ic -> home.resolve(ic.getArgument(0, Path.class)) + ); + } + + @Test + void shouldRemoveIndexDirectory() throws IOException { + Path indexDirectory = home.resolve("index"); + Path specificIndexDirectory = indexDirectory.resolve("repository").resolve("default"); + Files.createDirectories(specificIndexDirectory); + Path helloTxt = specificIndexDirectory.resolve("hello.txt"); + Files.write(helloTxt, "hello".getBytes(StandardCharsets.UTF_8)); + + updateStep.doUpdate(); + + assertThat(helloTxt).doesNotExist(); + assertThat(indexDirectory).doesNotExist(); + } + + @Test + void shouldRemoveIndexLogDirectory() throws IOException { + Path logDirectory = home.resolve("var").resolve("data").resolve("index-log"); + Files.createDirectories(logDirectory); + Path helloXml = logDirectory.resolve("hello.xml"); + Files.write(helloXml, "world".getBytes(StandardCharsets.UTF_8)); + + updateStep.doUpdate(); + + assertThat(helloXml).doesNotExist(); + assertThat(logDirectory).doesNotExist(); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java index 534931ead6..7321bdef53 100644 --- a/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java @@ -24,20 +24,23 @@ package sonia.scm.user; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +57,16 @@ class UserIndexerTest { @InjectMocks private UserIndexer indexer; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + @Test void shouldReturnType() { assertThat(indexer.getType()).isEqualTo(User.class); @@ -64,58 +77,54 @@ class UserIndexerTest { assertThat(indexer.getVersion()).isEqualTo(UserIndexer.VERSION); } - @Nested - class UpdaterTests { - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(UserIndexer.ReIndexAll.class); + } - private final User user = UserTestData.createTrillian(); + @Test + void shouldCreateUser() { + User trillian = UserTestData.createTrillian(); - @BeforeEach - void open() { - when(searchEngine.forType(User.class).getOrCreate()).thenReturn(index); - } + indexer.createStoreTask(trillian).update(index); - @Test - void shouldStore() { - indexer.open().store(user); + verify(index).store(Id.of(trillian), UserPermissions.read(trillian).asShiroString(), trillian); + } - verify(index).store(Id.of(user), "user:read:trillian", user); - } + @Test + void shouldDeleteUser() { + User trillian = UserTestData.createTrillian(); - @Test - void shouldDeleteById() { - indexer.open().delete(user); + indexer.createDeleteTask(trillian).update(index); - verify(index.delete().byType()).byId(Id.of(user)); - } + verify(index.delete()).byId(Id.of(trillian)); + } - @Test - void shouldReIndexAll() { - when(userManager.getAll()).thenReturn(singletonList(user)); + @Test + void shouldReIndexAll() { + User trillian = UserTestData.createTrillian(); + User slarti = UserTestData.createSlarti(); + when(userManager.getAll()).thenReturn(Arrays.asList(trillian, slarti)); - indexer.open().reIndexAll(); + UserIndexer.ReIndexAll reIndexAll = new UserIndexer.ReIndexAll(indexLogStore, userManager); + reIndexAll.update(index); - verify(index.delete().byType()).all(); - verify(index).store(Id.of(user), "user:read:trillian", user); - } + verify(index.delete()).all(); + verify(index).store(Id.of(trillian), UserPermissions.read(trillian).asShiroString(), trillian); + verify(index).store(Id.of(slarti), UserPermissions.read(slarti).asShiroString(), slarti); + } - @Test - void shouldHandleEvent() { - UserEvent event = new UserEvent(HandlerEventType.DELETE, user); + @Test + void shouldHandleEvents() { + User trillian = UserTestData.createTrillian(); + UserEvent event = new UserEvent(HandlerEventType.DELETE, trillian); - indexer.handleEvent(event); + indexer.handleEvent(event); - verify(index.delete().byType()).byId(Id.of(user)); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + verify(searchEngine.forType(User.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byId(Id.of(trillian)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java b/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java index b6a946e61e..470201c1b4 100644 --- a/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java +++ b/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java @@ -37,6 +37,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.security.Authentications; +import sonia.scm.security.Impersonator; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; @@ -54,7 +55,7 @@ class DefaultAdministrationContextTest { Injector injector = Guice.createInjector(); SecurityManager securityManager = new DefaultSecurityManager(); - context = new DefaultAdministrationContext(injector, securityManager); + context = new DefaultAdministrationContext(injector, new Impersonator(securityManager)); } @Test diff --git a/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java b/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java new file mode 100644 index 0000000000..558f649472 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java @@ -0,0 +1,375 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Inject; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.security.Authentications; + +import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.when; + +@SubjectAware("trillian") +@ExtendWith({MockitoExtension.class, ShiroExtension.class}) +class DefaultCentralWorkQueueTest { + + private final PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + private static final int ITERATIONS = 50; + private static final int TIMEOUT = 1; // seconds + + @Mock + private Persistence persistence; + + @Nested + class WithDefaultInjector { + + private MeterRegistry meterRegistry; + private DefaultCentralWorkQueue queue; + + @BeforeEach + void setUp() { + meterRegistry = new SimpleMeterRegistry(); + queue = new DefaultCentralWorkQueue(Guice.createInjector(new SecurityModule()), persistence, meterRegistry); + } + + private final AtomicInteger runs = new AtomicInteger(); + private int counter = 0; + private int copy = -1; + + @Test + void shouldRunInSequenceWithBlock() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + } + + private void waitForTasks() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + assertThat(runs.get()).isEqualTo(ITERATIONS); + } + + @Test + void shouldRunInParallel() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().enqueue(new Increase()); + } + waitForTasks(); + + // we test if the resulting counter is less than the iteration, + // because it is extremely likely that we miss a counter update + // when we run in parallel + assertThat(counter) + .isPositive() + .isLessThan(ITERATIONS); + } + + @Test + void shouldNotBlocked() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isNotNegative().isLessThan(ITERATIONS); + } + + @Test + void shouldNotBlockedByDifferentResource() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("copy").enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isNotNegative().isLessThan(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentResource() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("counter", "one").enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isEqualTo(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentAndExactResource() { + for (int i = 0; i < ITERATIONS; i++) { + if (i % 2 == 0) { + queue.append().locks("counter", "c").enqueue(new Increase()); + } else { + queue.append().locks("counter").enqueue(new Increase()); + } + } + waitForTasks(); + assertThat(counter).isEqualTo(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentResourceWithModelObject() { + Repository one = repository("one"); + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("counter", one).enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isEqualTo(ITERATIONS); + } + + @Test + void shouldFinalizeOnError() { + queue.append().enqueue(() -> { + throw new IllegalStateException("failed"); + }); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + } + + @Test + void shouldSetThreadName() { + AtomicReference threadName = new AtomicReference<>(); + queue.append().enqueue(() -> threadName.set(Thread.currentThread().getName())); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> threadName.get() != null); + + assertThat(threadName.get()).startsWith("CentralWorkQueue"); + } + + @Test + void shouldCaptureExecutorMetrics() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().enqueue(new Increase()); + } + waitForTasks(); + + double count = meterRegistry.get("executor.completed").functionCounter().count(); + assertThat(count).isEqualTo(ITERATIONS); + } + + @Test + void shouldCaptureExecutionDuration() { + queue.append().enqueue(new Increase()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + + Timer timer = meterRegistry.get(UnitOfWork.METRIC_EXECUTION).timer(); + assertThat(timer.count()).isEqualTo(1); + } + + @Test + void shouldCaptureWaitDuration() { + queue.append().enqueue(new Increase()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + + Timer timer = meterRegistry.get(UnitOfWork.METRIC_WAIT).timer(); + assertThat(timer.count()).isEqualTo(1); + } + + @Test + void shouldIncreaseBlockCount() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + waitForTasks(); + + int blockCount = 0; + for (Meter meter : meterRegistry.getMeters()) { + Meter.Id id = meter.getId(); + if ("cwq.task.wait.duration".equals(id.getName())) { + String blocked = id.getTag("blocked"); + if (blocked != null) { + blockCount += Integer.parseInt(blocked); + } + } + } + + assertThat(blockCount).isPositive(); + } + + @Nonnull + private Repository repository(String id) { + Repository one = new Repository(); + one.setId(id); + return one; + } + + @AfterEach + void tearDown() { + queue.close(); + } + + private class Increase implements Task { + + @Override + @SuppressWarnings("java:S2925") + public void run() { + int currentCounter = counter; + runs.incrementAndGet(); + try { + Thread.sleep(5); + counter = currentCounter + 1; + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + @Test + void shouldInjectDependencies() { + Context ctx = new Context(); + DefaultCentralWorkQueue queue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule(), binder -> binder.bind(Context.class).toInstance(ctx)), + persistence, + new SimpleMeterRegistry(), + () -> 2 + ); + + queue.append().enqueue(InjectingTask.class); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> ctx.value != null); + assertThat(ctx.value).isEqualTo("Hello"); + } + + @Test + void shouldLoadFromPersistence() { + Context context = new Context(); + SimpleUnitOfWork one = new SimpleUnitOfWork( + 21L, principal, Collections.singleton(new Resource("a")), new InjectingTask(context, "one") + ); + SimpleUnitOfWork two = new SimpleUnitOfWork( + 42L, principal, Collections.singleton(new Resource("a")), new InjectingTask(context, "two") + ); + two.restore(42L); + when(persistence.loadAll()).thenReturn(Arrays.asList(one, two)); + + new DefaultCentralWorkQueue(Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry()); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> context.value != null); + assertThat(context.value).isEqualTo("two"); + assertThat(one.getOrder()).isEqualTo(1L); + assertThat(one.getRestoreCount()).isEqualTo(1); + assertThat(two.getOrder()).isEqualTo(2L); + assertThat(two.getRestoreCount()).isEqualTo(2); + } + + @Test + void shouldRunAsUser() { + DefaultCentralWorkQueue workQueue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry() + ); + + AtomicReference ref = new AtomicReference<>(); + workQueue.append().enqueue(() -> ref.set(SecurityUtils.getSubject().getPrincipal())); + await().atMost(1, TimeUnit.SECONDS).until(() -> "trillian".equals(ref.get())); + } + + @Test + void shouldRunAsAdminUser() { + DefaultCentralWorkQueue workQueue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry() + ); + + AtomicReference ref = new AtomicReference<>(); + workQueue.append().runAsAdmin().enqueue(() -> ref.set(SecurityUtils.getSubject().getPrincipal())); + await().atMost(1, TimeUnit.SECONDS).until(() -> Authentications.PRINCIPAL_SYSTEM.equals(ref.get())); + } + + public static class Context { + + private String value; + + public void setValue(String value) { + this.value = value; + } + } + + public static class InjectingTask implements Task { + + private final Context context; + private final String value; + + @Inject + public InjectingTask(Context context) { + this(context, "Hello"); + } + + public InjectingTask(Context context, String value) { + this.context = context; + this.value = value; + } + + @Override + public void run() { + context.setValue(value); + } + } + + public static class SecurityModule extends AbstractModule { + + @Override + protected void configure() { + bind(SecurityManager.class).toInstance(SecurityUtils.getSecurityManager()); + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java b/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java new file mode 100644 index 0000000000..a09ed8c771 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java @@ -0,0 +1,178 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.plugin.PluginLoader; +import sonia.scm.store.Blob; +import sonia.scm.store.InMemoryBlobStore; +import sonia.scm.store.InMemoryBlobStoreFactory; + +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PersistenceTest { + + private final PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + @Nested + class Default { + + @Mock + private PluginLoader pluginLoader; + + private Persistence persistence; + + @BeforeEach + void setUp() { + when(pluginLoader.getUberClassLoader()).thenReturn(PersistenceTest.class.getClassLoader()); + persistence = new Persistence(pluginLoader, new InMemoryBlobStoreFactory()); + } + + @Test + void shouldStoreSimpleChunkOfWork() { + UnitOfWork work = new SimpleUnitOfWork( + 1L, principal, Collections.singleton(new Resource("a")), new MyTask() + ); + persistence.store(work); + + UnitOfWork loaded = persistence.loadAll().iterator().next(); + assertThat(loaded).isEqualTo(work); + } + + @Test + void shouldStoreInjectingChunkOfWork() { + UnitOfWork work = new InjectingUnitOfWork( + 1L, principal, Collections.singleton(new Resource("a")), MyTask.class + ); + persistence.store(work); + + UnitOfWork loaded = persistence.loadAll().iterator().next(); + assertThat(loaded).isEqualTo(work); + } + + @Test + void shouldLoadInOrder() { + store(5, 3, 1, 4, 2); + + long[] orderIds = persistence.loadAll() + .stream() + .mapToLong(UnitOfWork::getOrder) + .toArray(); + + assertThat(orderIds).containsExactly(1, 2, 3, 4, 5); + } + + @Test + void shouldRemoveAfterLoad() { + store(1, 2); + + assertThat(persistence.loadAll()).hasSize(2); + assertThat(persistence.loadAll()).isEmpty(); + } + + @Test + void shouldFailIfNotSerializable() { + store(1); + + SimpleUnitOfWork unitOfWork = new SimpleUnitOfWork( + 2L, principal, Collections.emptySet(), new NotSerializable() + ); + + assertThrows(NonPersistableTaskException.class, () -> persistence.store(unitOfWork)); + } + + @Test + void shouldRemoveStored() { + store(1); + SimpleUnitOfWork chunkOfWork = new SimpleUnitOfWork( + 2L, principal, Collections.emptySet(), new MyTask() + ); + persistence.store(chunkOfWork); + persistence.remove(chunkOfWork); + + assertThat(persistence.loadAll()).hasSize(1); + } + + private void store(long... orderIds) { + for (long order : orderIds) { + persistence.store(new SimpleUnitOfWork( + order, principal, Collections.emptySet(), new MyTask() + )); + } + } + + } + + @Test + void shouldNotFailForNonChunkOfWorkItems() throws IOException { + InMemoryBlobStore blobStore = new InMemoryBlobStore(); + + Persistence persistence = new Persistence(PersistenceTest.class.getClassLoader(), blobStore); + persistence.store(new SimpleUnitOfWork( + 1L, principal, Collections.emptySet(), new MyTask()) + ); + + Blob blob = blobStore.create(); + try (ObjectOutputStream stream = new ObjectOutputStream(blob.getOutputStream())) { + stream.writeObject(new MyTask()); + blob.commit(); + } + + assertThat(persistence.loadAll()).hasSize(1); + } + + @EqualsAndHashCode + public static class MyTask implements Task { + + @Override + public void run() { + + } + } + + // non static inner classes are not serializable + private class NotSerializable implements Task { + @Override + public void run() { + + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java b/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java new file mode 100644 index 0000000000..6f4cee20c6 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java @@ -0,0 +1,72 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ResourceTest { + + @Test + void shouldReturnResourceName() { + assertThat(res("a")).hasToString("a"); + } + + @Test + void shouldReturnResourceNameAndId() { + assertThat(res("a", "b")).hasToString("a:b"); + } + + @Nested + class IsBlockedByTests { + + @Test + void shouldReturnTrue() { + assertThat(res("a").isBlockedBy(res("a"))).isTrue(); + assertThat(res("a", "b").isBlockedBy(res("a", "b"))).isTrue(); + assertThat(res("a").isBlockedBy(res("a", "b"))).isTrue(); + assertThat(res("a", "b").isBlockedBy(res("a"))).isTrue(); + } + + @Test + void shouldReturnFalse() { + assertThat(res("a").isBlockedBy(res("b"))).isFalse(); + assertThat(res("a", "b").isBlockedBy(res("a", "c"))).isFalse(); + assertThat(res("a", "b").isBlockedBy(res("c", "b"))).isFalse(); + } + + } + + private Resource res(String name) { + return new Resource(name); + } + + private Resource res(String name, String id) { + return new Resource(name, id); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java b/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java new file mode 100644 index 0000000000..5a936768eb --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import lombok.Value; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.junit.jupiter.api.Test; + +import javax.inject.Inject; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +class SimpleUnitOfWorkTest { + + private PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + @Test + void shouldInjectMember() { + Context context = new Context("awesome"); + Injector injector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(Context.class).toInstance(context); + } + }); + + SimpleTask simpleTask = new SimpleTask(); + SimpleUnitOfWork unitOfWork = new SimpleUnitOfWork(1L, principal, Collections.emptySet(), simpleTask); + unitOfWork.task(injector); + + simpleTask.run(); + + assertThat(simpleTask.value).isEqualTo("awesome"); + } + + @Value + public class Context { + String value; + } + + public class SimpleTask implements Task { + + private Context context; + + private String value = "no value set"; + + @Inject + public void setContext(Context context) { + this.context = context; + } + + @Override + public void run() { + if (context != null) { + value = context.getValue(); + } + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java b/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java new file mode 100644 index 0000000000..6d9d6b0d7f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java @@ -0,0 +1,80 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class ThreadCountProviderTest { + + @Test + void shouldUseTwoWorkersForOneCPU() { + ThreadCountProvider provider = new ThreadCountProvider(() -> 1); + + assertThat(provider.getAsInt()).isEqualTo(2); + } + + @ParameterizedTest(name = "shouldUseFourWorkersFor{argumentsWithNames}CPU") + @ValueSource(ints = {2, 4, 8, 16}) + void shouldUseFourWorkersForMoreThanOneCPU(int cpus) { + ThreadCountProvider provider = new ThreadCountProvider(() -> cpus); + + assertThat(provider.getAsInt()).isEqualTo(4); + } + + @Nested + class SystemPropertyTests { + + @BeforeEach + void setUp() { + System.clearProperty(ThreadCountProvider.PROPERTY); + } + + @Test + void shouldUseCountFromSystemProperty() { + ThreadCountProvider provider = new ThreadCountProvider(); + System.setProperty(ThreadCountProvider.PROPERTY, "6"); + assertThat(provider.getAsInt()).isEqualTo(6); + } + + @ParameterizedTest + @ValueSource(strings = {"-1", "0", "100", "a", ""}) + void shouldUseDefaultForInvalidValue(String value) { + ThreadCountProvider provider = new ThreadCountProvider(() -> 1); + System.setProperty(ThreadCountProvider.PROPERTY, value); + assertThat(provider.getAsInt()).isEqualTo(2); + } + + } + +}