diff --git a/gradle/changelog/one_index_per_type.yaml b/gradle/changelog/one_index_per_type.yaml new file mode 100644 index 0000000000..69a05dbc4e --- /dev/null +++ b/gradle/changelog/one_index_per_type.yaml @@ -0,0 +1,8 @@ +- type: Added + description: Central Work Queue for coordinating long-running tasks ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Changed + description: One index per type instead of one index for all types ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Added + description: Api to modify mutliple indices at once ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) +- type: Changed + description: Use central work queue for all indexing tasks ([#1781](https://github.com/scm-manager/scm-manager/pull/1781)) diff --git a/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java b/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java index 4832e3d448..136d29138e 100644 --- a/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java +++ b/scm-core/src/main/java/sonia/scm/search/HandlerEventIndexSyncer.java @@ -24,6 +24,7 @@ package sonia.scm.search; +import com.google.common.annotations.Beta; import sonia.scm.HandlerEventType; import sonia.scm.event.HandlerEvent; @@ -33,11 +34,14 @@ import sonia.scm.event.HandlerEvent; * @param type of indexed item * @since 2.22.0 */ +@Beta public final class HandlerEventIndexSyncer { + private final SearchEngine searchEngine; private final Indexer indexer; - public HandlerEventIndexSyncer(Indexer indexer) { + public HandlerEventIndexSyncer(SearchEngine searchEngine, Indexer indexer) { + this.searchEngine = searchEngine; this.indexer = indexer; } @@ -49,17 +53,16 @@ public final class HandlerEventIndexSyncer { public void handleEvent(HandlerEvent event) { HandlerEventType type = event.getEventType(); if (type.isPost()) { - updateIndex(type, event.getItem()); + SerializableIndexTask task = createTask(type, event.getItem()); + searchEngine.forType(indexer.getType()).update(task); } } - private void updateIndex(HandlerEventType type, T item) { - try (Indexer.Updater updater = indexer.open()) { - if (type == HandlerEventType.DELETE) { - updater.delete(item); - } else { - updater.store(item); - } + private SerializableIndexTask createTask(HandlerEventType type, T item) { + if (type == HandlerEventType.DELETE) { + return indexer.createDeleteTask(item); + } else { + return indexer.createStoreTask(item); } } diff --git a/scm-core/src/main/java/sonia/scm/search/Index.java b/scm-core/src/main/java/sonia/scm/search/Index.java index 5e23016054..3a00fc50b0 100644 --- a/scm-core/src/main/java/sonia/scm/search/Index.java +++ b/scm-core/src/main/java/sonia/scm/search/Index.java @@ -25,6 +25,7 @@ package sonia.scm.search; import com.google.common.annotations.Beta; +import sonia.scm.repository.Repository; /** * Can be used to index objects for full text searches. @@ -32,7 +33,15 @@ import com.google.common.annotations.Beta; * @since 2.21.0 */ @Beta -public interface Index extends AutoCloseable { +public interface Index { + + /** + * Returns details such as name and type of index. + * + * @return details of index + * @since 2.23.0 + */ + IndexDetails getDetails(); /** * Store the given object in the index. @@ -53,12 +62,6 @@ public interface Index extends AutoCloseable { */ Deleter delete(); - /** - * Close index and commit changes. - */ - @Override - void close(); - /** * Deleter provides an api to delete object from index. * @@ -66,27 +69,6 @@ public interface Index extends AutoCloseable { */ interface Deleter { - /** - * Returns an api which allows deletion of objects from the type of this index. - * @return type restricted delete api - */ - ByTypeDeleter byType(); - - /** - * Returns an api which allows deletion of objects of every type. - * @return unrestricted delete api for all types. - */ - AllTypesDeleter allTypes(); - } - - /** - * Delete api for the type of the index. This means, that only entries for this - * type will be deleted. - * - * @since 2.23.0 - */ - interface ByTypeDeleter { - /** * Delete the object with the given id and type from index. * @param id id of object @@ -104,27 +86,15 @@ public interface Index extends AutoCloseable { * @param repositoryId id of repository */ void byRepository(String repositoryId); - } - - /** - * Delete api for the overall index regarding all types. - * - * @since 2.23.0 - */ - interface AllTypesDeleter { /** - * Delete all objects which are related to the given repository from index regardless of their type. - * @param repositoryId repository id + * Delete all objects which are related the given type and repository from index. + * + * @param repository repository */ - void byRepository(String repositoryId); - - /** - * Delete all objects with the given type from index. - * This method is mostly useful if the index type has changed and the old type (in form of a class) - * is no longer available. - * @param typeName type name of objects - */ - void byTypeName(String typeName); + default void byRepository(Repository repository) { + byRepository(repository.getId()); + } } + } diff --git a/scm-core/src/main/java/sonia/scm/search/IndexDetails.java b/scm-core/src/main/java/sonia/scm/search/IndexDetails.java new file mode 100644 index 0000000000..23db92a53c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexDetails.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * Details of an index. + * @since 2.23.0 + */ +@Beta +public interface IndexDetails { + + /** + * Returns type of objects which are indexed. + * @return type of objects + */ + Class getType(); + + /** + * Returns the name of the index (e.g. `default`) + * @return name + */ + String getName(); +} diff --git a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java index 4b99655206..21c9cb931a 100644 --- a/scm-core/src/main/java/sonia/scm/search/IndexOptions.java +++ b/scm-core/src/main/java/sonia/scm/search/IndexOptions.java @@ -27,6 +27,7 @@ package sonia.scm.search; import com.google.common.annotations.Beta; import lombok.EqualsAndHashCode; +import java.io.Serializable; import java.util.Locale; /** @@ -36,7 +37,7 @@ import java.util.Locale; */ @Beta @EqualsAndHashCode -public class IndexOptions { +public class IndexOptions implements Serializable { private final Type type; private final Locale locale; diff --git a/scm-core/src/main/java/sonia/scm/search/IndexTask.java b/scm-core/src/main/java/sonia/scm/search/IndexTask.java new file mode 100644 index 0000000000..11b5d9bfab --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/IndexTask.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +/** + * A task which updates an index. + * @param type of indexed objects + * @since 2.23.0 + */ +@Beta +@FunctionalInterface +public interface IndexTask { + + /** + * Execute operations on the index. + * @param index index to update + */ + void update(Index index); + + /** + * This method is called after work is committed to the index. + */ + default void afterUpdate() { +// Do nothing + } + +} diff --git a/scm-core/src/main/java/sonia/scm/search/Indexer.java b/scm-core/src/main/java/sonia/scm/search/Indexer.java index 13c8935927..7e6967d78a 100644 --- a/scm-core/src/main/java/sonia/scm/search/Indexer.java +++ b/scm-core/src/main/java/sonia/scm/search/Indexer.java @@ -24,6 +24,7 @@ package sonia.scm.search; +import com.google.common.annotations.Beta; import sonia.scm.plugin.ExtensionPoint; /** @@ -35,6 +36,7 @@ import sonia.scm.plugin.ExtensionPoint; * @since 2.22.0 * @see HandlerEventIndexSyncer */ +@Beta @ExtensionPoint public interface Indexer { @@ -54,42 +56,48 @@ public interface Indexer { int getVersion(); /** - * Opens the index and return an updater for the given type. - * - * @return updater with open index + * Returns task which re index all items. + * @return task to re index all + * @since 2.23.0 */ - Updater open(); + Class> getReIndexAllTask(); /** - * Updater for index. - * - * @param type to index + * Creates a task which stores the given item in the index. + * @param item item to store + * @return task which stores the item + * @since 2.23.0 */ - interface Updater extends AutoCloseable { + SerializableIndexTask createStoreTask(T item); - /** - * Stores the given item in the index. - * - * @param item item to index - */ - void store(T item); + /** + * Creates a task which deletes the given item from index. + * @param item item to delete + * @return task which deletes the item + * @since 2.23.0 + */ + SerializableIndexTask createDeleteTask(T item); - /** - * Delete the given item from the index - * - * @param item item to delete - */ - void delete(T item); + /** + * Abstract class which builds the foundation for tasks which re-index all items. + * + * @since 2.23.0 + */ + abstract class ReIndexAllTask implements IndexTask { - /** - * Re index all existing items. - */ - void reIndexAll(); + private final IndexLogStore logStore; + private final Class type; + private final int version; - /** - * Close the given index. - */ - void close(); + protected ReIndexAllTask(IndexLogStore logStore, Class type, int version) { + this.logStore = logStore; + this.type = type; + this.version = version; + } + + @Override + public void afterUpdate() { + logStore.defaultIndex().log(type, version); + } } - } diff --git a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java index c95c79ef37..1d81422064 100644 --- a/scm-core/src/main/java/sonia/scm/search/SearchEngine.java +++ b/scm-core/src/main/java/sonia/scm/search/SearchEngine.java @@ -25,8 +25,10 @@ package sonia.scm.search; import com.google.common.annotations.Beta; +import sonia.scm.ModelObject; import java.util.Collection; +import java.util.function.Predicate; /** * The {@link SearchEngine} is the main entry point for indexing and searching. @@ -61,6 +63,80 @@ public interface SearchEngine { */ ForType forType(String name); + /** + * Returns an api which allows the modification of multiple indices at once. + * @return api to modify multiple indices + * @since 2.23.0 + */ + ForIndices forIndices(); + + /** + * Api for modifying multiple indices at once. + * @since 2.23.0 + */ + interface ForIndices { + + /** + * This method can be used to filter the indices. + * If no predicate is used the tasks are enqueued for every existing index. + * + * @param predicate predicate to filter indices + * @return {@code this} + */ + ForIndices matching(Predicate predicate); + + /** + * Apply a lock for a specific resource. By default, a lock for the whole index is used. + * If one or more specific resources are locked, than the lock is applied only for those resources + * and tasks which targets other resources of the same index can run in parallel. + * + * @param resource specific resource to lock + * @return {@code this} + */ + ForIndices forResource(String resource); + + /** + * This method is a shortcut for {@link #forResource(String)} with the id of the given resource. + * + * @param resource resource in form of model object + * @return {@code this} + */ + default ForIndices forResource(ModelObject resource) { + return forResource(resource.getId()); + } + + /** + * Specify options for the index. + * If not used the default options will be used. + * @param options index options + * @return {@code this} + * @see IndexOptions#defaults() + */ + ForIndices withOptions(IndexOptions options); + + /** + * Submits the task and execute it for every index + * which are matching the predicate ({@link #matching(Predicate)}. + * The task is executed asynchronous and will be finished some time in the future. + * Note: the task must be serializable because it is submitted to the + * {@link sonia.scm.work.CentralWorkQueue}. + * For more information on task serialization have a look at the + * {@link sonia.scm.work.CentralWorkQueue} documentation. + * + * @param task serializable task for updating multiple indices + */ + void batch(SerializableIndexTask task); + + /** + * Submits the task and executes it for every index + * which are matching the predicate ({@link #matching(Predicate)}. + * The task is executed asynchronously and will finish at some unknown point in the future. + * + * @param task task for updating multiple indices + */ + void batch(Class> task); + } + /** * Search and index api. * @@ -87,10 +163,44 @@ public interface SearchEngine { ForType withIndex(String name); /** - * Returns an index object which provides method to update the search index. - * @return index object + * Apply a lock for a specific resource. By default, a lock for the whole index is used. + * If one or more specific resources are locked, then the lock is applied only for those resources + * and tasks which target other resources of the same index can run in parallel. + * + * @param resource specific resource to lock + * @return {@code this} */ - Index getOrCreate(); + ForType forResource(String resource); + + /** + * This method is a shortcut for {@link #forResource(String)} with the id of the given resource. + * + * @param resource resource in form of model object + * @return {@code this} + */ + default ForType forResource(ModelObject resource) { + return forResource(resource.getId()); + } + + /** + * Submits a task to update the index. + * The task is executed asynchronously and will finish at some unknown point in the future. + * Note: the task must be serializable because it is submitted to the + * {@link sonia.scm.work.CentralWorkQueue}, + * for more information about the task serialization have a look at the + * {@link sonia.scm.work.CentralWorkQueue} documentation. + * + * @param task serializable task for updating the index + */ + void update(SerializableIndexTask task); + + /** + * Submits a task to update the index. + * The task is executed asynchronous and will be finished some time in the future. + * + * @param task task for updating multiple indices + */ + void update(Class> task); /** * Returns a query builder object which can be used to search the index. diff --git a/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java b/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java new file mode 100644 index 0000000000..754d056a0c --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/SerializableIndexTask.java @@ -0,0 +1,40 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.Beta; + +import java.io.Serializable; + +/** + * A serializable version of {@link IndexTask}. + * + * @param type of indexed objects + * @since 2.23.0 + */ +@Beta +@FunctionalInterface +public interface SerializableIndexTask extends IndexTask, Serializable { +} diff --git a/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java b/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java new file mode 100644 index 0000000000..c526fd642e --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/CentralWorkQueue.java @@ -0,0 +1,174 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; +import sonia.scm.ModelObject; + +import javax.annotation.Nullable; + +/** + * The {@link CentralWorkQueue} provides an api to submit and coordinate long-running or resource intensive tasks. + * + * The tasks are executed in parallel, but if some tasks access the same resource this can become a problem. + * To avoid this problem a task can be enqueued with a lock e.g.: + * + * 
{@code
+ *   queue.append().locks("my-resources").enqueue(MyTask.class)
+ * }
+ * + * No tasks with the same lock will run in parallel. The example above locks a whole group of resources. + * It is possible to assign the lock to a more specific resource by adding the id parameter to the lock e.g.: + * + * 
{@code
+ *   queue.append().locks("my-resources", "42").enqueue(MyTask.class)
+ * }
+ * + * This will ensure, that no task with a lock for the my-resource 42 or for my-resources will run at the + * same time as the task which is enqueued in the example above. + * But this will also allow a task for my-resources with an id other than 42 can run in parallel. + * + * All tasks are executed with the permissions of the user which enqueues the task. + * If the task should run as admin, the {@link Enqueue#runAsAdmin()} method can be used. + * + * Tasks which could not be finished, + * before a restart of shutdown of the server, will be restored and executed on startup. + * In order to achieve the persistence of tasks, + * the enqueued task must be provided as a class or it must be serializable. + * This could become unhandy if the task has parameters and dependencies which must be injected. + * The injected objects should not be serialized with the task. + * In order to avoid this, dependencies should be declared as {@code transient} + * and injected via setters instead of the constructor parameters e.g.: + * + * 

+ *   public class MyTask implements Task {
+ *
+ *     private final Repository repository;
+ *     private transient RepositoryServiceFactory repositoryServiceFactory;
+ *
+ *     public MyTask(Repository repository) {
+ *       this.repository = repository;
+ *     }
+ *
+ *     {@code @}Inject
+ *     public void setRepositoryServiceFactory(RepositoryServiceFactory repositoryServiceFactory) {
+ *       this.repositoryServiceFactory = repositoryServiceFactory;
+ *     }
+ *
+ *     {@code @}Override
+ *     public void run() {
+ *       // do something with the repository and the repositoryServiceFactory
+ *     }
+ *
+ *   }
+ *
+ * + * The {@link CentralWorkQueue} will inject the requested members before the {@code run} method of the task is executed. + * + * @since 2.23.0 + */ +@Beta +public interface CentralWorkQueue { + + /** + * Append a new task to the central work queue. + * The method will return a builder interface to configure how the task will be enqueued. + * + * @return builder api for enqueue a task + */ + Enqueue append(); + + /** + * Returns the count of pending or running tasks. + * + * @return count of pending or running tasks + */ + int getSize(); + + /** + * Builder interface for the enqueueing of a new task. + */ + interface Enqueue { + + /** + * Configure a lock for the given resource type. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @return {@code this} + * @see CentralWorkQueue + */ + Enqueue locks(String resourceType); + + /** + * Configure a lock for the resource with the given type and id. + * Note if the id is {@code null} the whole resource type is locked. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @param id id of resource to lock + * @return {@code this} + * @see CentralWorkQueue + */ + Enqueue locks(String resourceType, @Nullable String id); + + /** + * Configure a lock for the resource with the given type and the id from the given {@link ModelObject}. + * For more information on locks see the class documentation ({@link CentralWorkQueue}). + * + * @param resourceType resource type to lock + * @param object which holds the id of the resource to lock + * @return {@code this} + * @see CentralWorkQueue + */ + default Enqueue locks(String resourceType, ModelObject object) { + return locks(resourceType, object.getId()); + } + + /** + * Run the enqueued task with administrator permission. + * + * @return {@code this} + */ + Enqueue runAsAdmin(); + + /** + * Enqueue the given task to {@link CentralWorkQueue}. + * Warning: Ensure that the task is serializable. + * If the task is not serializable an {@link NonPersistableTaskException} will be thrown. + * For more information about the persistence of tasks see class documentation. + * + * @param task serializable task to enqueue + * @see CentralWorkQueue + */ + void enqueue(Task task); + + /** + * Enqueue the given task to {@link CentralWorkQueue}. + * @param task task to enqueue + */ + void enqueue(Class task); + } +} diff --git a/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java b/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java new file mode 100644 index 0000000000..6014171acc --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/NonPersistableTaskException.java @@ -0,0 +1,39 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; + +/** + * Exception thrown when a task is enqueued to the {@link CentralWorkQueue} which cannot persisted. + * + * @since 2.23.0 + */ +@Beta +public final class NonPersistableTaskException extends RuntimeException { + public NonPersistableTaskException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/scm-core/src/main/java/sonia/scm/work/Task.java b/scm-core/src/main/java/sonia/scm/work/Task.java new file mode 100644 index 0000000000..962ae97dae --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/work/Task.java @@ -0,0 +1,38 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.Beta; + +import java.io.Serializable; + +/** + * Serializable task which can be enqueued to the {@link CentralWorkQueue}. + * + * @since 2.23.0 + */ +@Beta +public interface Task extends Runnable, Serializable { +} diff --git a/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java b/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java index aa441e64a4..759e9b98bc 100644 --- a/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java +++ b/scm-dao-xml/src/main/java/sonia/scm/store/StoreConstants.java @@ -21,12 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.store; /** * Store constants for xml implementations. - * + * * @author Sebastian Sdorra */ public class StoreConstants @@ -36,6 +36,24 @@ public class StoreConstants public static final String CONFIG_DIRECTORY_NAME = "config"; + /** + * Name of the parent of data or blob directories. + * @since 2.23.0 + */ + public static final String VARIABLE_DATA_DIRECTORY_NAME = "var"; + + /** + * Name of data directories. + * @since 2.23.0 + */ + public static final String DATA_DIRECTORY_NAME = "data"; + + /** + * Name of blob directories. + * @since 2.23.0 + */ + public static final String BLOG_DIRECTORY_NAME = "data"; + public static final String REPOSITORY_METADATA = "metadata"; public static final String FILE_EXTENSION = ".xml"; diff --git a/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java b/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java index e95742b6ac..01d00d76a8 100644 --- a/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java +++ b/scm-test/src/main/java/sonia/scm/store/InMemoryBlobStore.java @@ -24,6 +24,8 @@ package sonia.scm.store; +import com.google.common.collect.ImmutableList; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; @@ -59,7 +61,7 @@ public class InMemoryBlobStore implements BlobStore { @Override public List getAll() { - return blobs; + return ImmutableList.copyOf(blobs); } @Override diff --git a/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java b/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java index a0c3253f72..8170da9c4a 100644 --- a/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java +++ b/scm-test/src/main/java/sonia/scm/web/RestDispatcher.java @@ -28,7 +28,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.shiro.authz.AuthorizationException; import org.apache.shiro.authz.UnauthorizedException; import org.jboss.resteasy.mock.MockDispatcherFactory; -import org.jboss.resteasy.mock.MockHttpRequest; import org.jboss.resteasy.spi.Dispatcher; import org.jboss.resteasy.spi.HttpRequest; import org.jboss.resteasy.spi.HttpResponse; diff --git a/scm-ui/ui-webapp/src/containers/OmniSearch.tsx b/scm-ui/ui-webapp/src/containers/OmniSearch.tsx index 0972ce97d3..0dedf8f18d 100644 --- a/scm-ui/ui-webapp/src/containers/OmniSearch.tsx +++ b/scm-ui/ui-webapp/src/containers/OmniSearch.tsx @@ -99,7 +99,7 @@ const AvatarSection: FC = ({ hit }) => { const name = useStringHitFieldValue(hit, "name"); const type = useStringHitFieldValue(hit, "type"); - const repository = hit._embedded.repository; + const repository = hit._embedded?.repository; if (!namespace || !name || !type || !repository) { return null; } diff --git a/scm-ui/ui-webapp/src/search/RepositoryHit.tsx b/scm-ui/ui-webapp/src/search/RepositoryHit.tsx index 8d30a36ed6..a4eb491f6d 100644 --- a/scm-ui/ui-webapp/src/search/RepositoryHit.tsx +++ b/scm-ui/ui-webapp/src/search/RepositoryHit.tsx @@ -44,7 +44,7 @@ const RepositoryHit: FC = ({ hit }) => { // the embedded repository is only a subset of the repository (RepositoryCoordinates), // so we should use the fields to get more information - const repository = hit._embedded.repository; + const repository = hit._embedded?.repository; if (!namespace || !name || !type || !repository) { return null; } diff --git a/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java b/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java index 38dcd78157..103b1d5638 100644 --- a/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/group/GroupIndexer.java @@ -30,8 +30,10 @@ import sonia.scm.plugin.Extension; import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class GroupIndexer implements Indexer { @VisibleForTesting static final int VERSION = 1; - private final GroupManager groupManager; private final SearchEngine searchEngine; @Inject - public GroupIndexer(GroupManager groupManager, SearchEngine searchEngine) { - this.groupManager = groupManager; + public GroupIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,47 +62,47 @@ public class GroupIndexer implements Indexer { return VERSION; } - @Subscribe(async = false) - public void handleEvent(GroupEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; } @Override - public Updater open() { - return new GroupIndexUpdater(groupManager, searchEngine.forType(Group.class).getOrCreate()); + public SerializableIndexTask createStoreTask(Group group) { + return index -> store(index, group); } - public static class GroupIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(Group group) { + return index -> index.delete().byId(Id.of(group)); + } + + @Subscribe(async = false) + public void handleEvent(GroupEvent event) { + new HandlerEventIndexSyncer<>(searchEngine, this).handleEvent(event); + } + + public static void store(Index index, Group group) { + index.store(Id.of(group), GroupPermissions.read(group).asShiroString(), group); + } + + public static class ReIndexAll extends ReIndexAllTask { private final GroupManager groupManager; - private final Index index; - private GroupIndexUpdater(GroupManager groupManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, GroupManager groupManager) { + super(logStore, Group.class, VERSION); this.groupManager = groupManager; - this.index = index; } @Override - public void store(Group group) { - index.store(Id.of(group), GroupPermissions.read(group).asShiroString(), group); - } - - @Override - public void delete(Group group) { - index.delete().byType().byId(Id.of(group)); - } - - @Override - public void reIndexAll() { - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (Group group : groupManager.getAll()) { - store(group); + store(index, group); } } - - @Override - public void close() { - index.close(); - } } + } diff --git a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java index b9b82e8289..fa52c95594 100644 --- a/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java +++ b/scm-webapp/src/main/java/sonia/scm/lifecycle/modules/ScmServletModule.java @@ -134,6 +134,8 @@ import sonia.scm.web.cgi.DefaultCGIExecutorFactory; import sonia.scm.web.filter.LoggingFilter; import sonia.scm.web.security.AdministrationContext; import sonia.scm.web.security.DefaultAdministrationContext; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.DefaultCentralWorkQueue; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; @@ -290,6 +292,8 @@ class ScmServletModule extends ServletModule { bind(SearchEngine.class, LuceneSearchEngine.class); bind(IndexLogStore.class, DefaultIndexLogStore.class); + bind(CentralWorkQueue.class, DefaultCentralWorkQueue.class); + bind(ContentTypeResolver.class).to(DefaultContentTypeResolver.class); } diff --git a/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java b/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java index 1eadd222f3..9016d72b1d 100644 --- a/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/repository/RepositoryIndexer.java @@ -26,12 +26,14 @@ package sonia.scm.repository; import com.github.legman.Subscribe; import com.google.common.annotations.VisibleForTesting; +import sonia.scm.HandlerEventType; import sonia.scm.plugin.Extension; -import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class RepositoryIndexer implements Indexer { @VisibleForTesting static final int VERSION = 3; - private final RepositoryManager repositoryManager; private final SearchEngine searchEngine; @Inject - public RepositoryIndexer(RepositoryManager repositoryManager, SearchEngine searchEngine) { - this.repositoryManager = repositoryManager; + public RepositoryIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,49 +62,58 @@ public class RepositoryIndexer implements Indexer { return Repository.class; } + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; + } + @Subscribe(async = false) public void handleEvent(RepositoryEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + HandlerEventType type = event.getEventType(); + if (type.isPost()) { + Repository repository = event.getItem(); + if (type == HandlerEventType.DELETE) { + searchEngine.forIndices() + .forResource(repository) + .batch(createDeleteTask(repository)); + } else { + searchEngine.forType(Repository.class) + .update(createStoreTask(repository)); + } + } } @Override - public Updater open() { - return new RepositoryIndexUpdater(repositoryManager, searchEngine.forType(getType()).getOrCreate()); + public SerializableIndexTask createStoreTask(Repository repository) { + return index -> store(index, repository); } - public static class RepositoryIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(Repository repository) { + return index -> index.delete().byRepository(repository); + } + + private static void store(Index index, Repository repository) { + index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); + } + + public static class ReIndexAll extends ReIndexAllTask { private final RepositoryManager repositoryManager; - private final Index index; - public RepositoryIndexUpdater(RepositoryManager repositoryManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, RepositoryManager repositoryManager) { + super(logStore, Repository.class, VERSION); this.repositoryManager = repositoryManager; - this.index = index; } @Override - public void store(Repository repository) { - index.store(Id.of(repository), RepositoryPermissions.read(repository).asShiroString(), repository); - } - - @Override - public void delete(Repository repository) { - index.delete().allTypes().byRepository(repository.getId()); - } - - @Override - public void reIndexAll() { - // v1 used the whole classname as type - index.delete().allTypes().byTypeName(Repository.class.getName()); - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (Repository repository : repositoryManager.getAll()) { - store(repository); + store(index, repository); } } - - @Override - public void close() { - index.close(); - } } + } diff --git a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java index 9c11baa34c..7681eda584 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java +++ b/scm-webapp/src/main/java/sonia/scm/search/FieldNames.java @@ -27,9 +27,7 @@ package sonia.scm.search; final class FieldNames { private FieldNames(){} - static final String UID = "_uid"; static final String ID = "_id"; - static final String TYPE = "_type"; static final String REPOSITORY = "_repository"; static final String PERMISSION = "_permission"; } diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java b/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java index 752fe9619e..923da1ff44 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexBootstrapListener.java @@ -27,7 +27,6 @@ package sonia.scm.search; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import sonia.scm.plugin.Extension; -import sonia.scm.web.security.AdministrationContext; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,13 +42,13 @@ public class IndexBootstrapListener implements ServletContextListener { private static final Logger LOG = LoggerFactory.getLogger(IndexBootstrapListener.class); - private final AdministrationContext administrationContext; + private final SearchEngine searchEngine; private final IndexLogStore indexLogStore; private final Set indexers; @Inject - public IndexBootstrapListener(AdministrationContext administrationContext, IndexLogStore indexLogStore, Set indexers) { - this.administrationContext = administrationContext; + public IndexBootstrapListener(SearchEngine searchEngine, IndexLogStore indexLogStore, Set indexers) { + this.searchEngine = searchEngine; this.indexLogStore = indexLogStore; this.indexers = indexers; } @@ -65,8 +64,11 @@ public class IndexBootstrapListener implements ServletContextListener { Optional indexLog = indexLogStore.defaultIndex().get(indexer.getType()); if (indexLog.isPresent()) { int version = indexLog.get().getVersion(); - if (version < indexer.getVersion()) { - LOG.debug("index version {} is older then {}, start reindexing of all {}", version, indexer.getVersion(), indexer.getType()); + if (version != indexer.getVersion()) { + LOG.debug( + "index version {} is older then {}, start reindexing of all {}", + version, indexer.getVersion(), indexer.getType() + ); indexAll(indexer); } } else { @@ -75,14 +77,9 @@ public class IndexBootstrapListener implements ServletContextListener { } } + @SuppressWarnings("unchecked") private void indexAll(Indexer indexer) { - administrationContext.runAsAdmin(() -> { - try (Indexer.Updater updater = indexer.open()) { - updater.reIndexAll(); - } - }); - - indexLogStore.defaultIndex().log(indexer.getType(), indexer.getVersion()); + searchEngine.forType(indexer.getType()).update(indexer.getReIndexAllTask()); } @Override diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java b/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java new file mode 100644 index 0000000000..9a6b3ca68a --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexManager.java @@ -0,0 +1,139 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.Data; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.FSDirectory; +import sonia.scm.SCMContextProvider; +import sonia.scm.plugin.PluginLoader; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.xml.bind.JAXB; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@Singleton +public class IndexManager { + + private final Path directory; + private final AnalyzerFactory analyzerFactory; + private final IndexXml indexXml; + + @Inject + public IndexManager(SCMContextProvider context, PluginLoader pluginLoader, AnalyzerFactory analyzerFactory) { + directory = context.resolve(Paths.get("index")); + this.analyzerFactory = analyzerFactory; + this.indexXml = readIndexXml(pluginLoader.getUberClassLoader()); + } + + private IndexXml readIndexXml(ClassLoader uberClassLoader) { + Path path = directory.resolve("index.xml"); + if (Files.exists(path)) { + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(uberClassLoader); + return JAXB.unmarshal(path.toFile(), IndexXml.class); + } finally { + Thread.currentThread().setContextClassLoader(contextClassLoader); + } + } + return new IndexXml(); + } + + public Collection all() { + return Collections.unmodifiableSet(indexXml.indices); + } + + public IndexReader openForRead(LuceneSearchableType type, String indexName) throws IOException { + Path path = resolveIndexDirectory(type, indexName); + return DirectoryReader.open(FSDirectory.open(path)); + } + + public IndexWriter openForWrite(IndexParams indexParams) { + IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())); + config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + + Path path = resolveIndexDirectory(indexParams); + if (!Files.exists(path)) { + store(new LuceneIndexDetails(indexParams.getType(), indexParams.getIndex())); + } + + try { + return new IndexWriter(FSDirectory.open(path), config); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index at " + path, ex); + } + } + + private Path resolveIndexDirectory(IndexParams indexParams) { + return directory.resolve(indexParams.getSearchableType().getName()).resolve(indexParams.getIndex()); + } + + private Path resolveIndexDirectory(LuceneSearchableType searchableType, String indexName) { + return directory.resolve(searchableType.getName()).resolve(indexName); + } + + private synchronized void store(LuceneIndexDetails details) { + if (!indexXml.getIndices().add(details)) { + return; + } + + if (!Files.exists(directory)) { + try { + Files.createDirectory(directory); + } catch (IOException e) { + throw new SearchEngineException("failed to create index directory", e); + } + } + + Path path = directory.resolve("index.xml"); + JAXB.marshal(indexXml, path.toFile()); + } + + @Data + @XmlRootElement(name = "indices") + @XmlAccessorType(XmlAccessType.FIELD) + public static class IndexXml { + + @XmlElement(name = "index") + private Set indices = new HashSet<>(); + + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java b/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java index 2fca1540e3..14f456ab3e 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java +++ b/scm-webapp/src/main/java/sonia/scm/search/IndexParams.java @@ -27,10 +27,24 @@ package sonia.scm.search; import lombok.Value; @Value -public class IndexParams { +public class IndexParams implements IndexDetails { String index; LuceneSearchableType searchableType; IndexOptions options; + @Override + public Class getType() { + return searchableType.getType(); + } + + @Override + public String getName() { + return index; + } + + @Override + public String toString() { + return searchableType.getName() + "/" + index; + } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java index 1612dc3fd6..4d201b38af 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndex.java @@ -24,54 +24,71 @@ package sonia.scm.search; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.TermQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; import java.io.IOException; +import java.util.function.Supplier; import static sonia.scm.search.FieldNames.ID; import static sonia.scm.search.FieldNames.PERMISSION; import static sonia.scm.search.FieldNames.REPOSITORY; -import static sonia.scm.search.FieldNames.TYPE; -import static sonia.scm.search.FieldNames.UID; -class LuceneIndex implements Index { +class LuceneIndex implements Index, AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(LuceneIndex.class); + + private final IndexDetails details; private final LuceneSearchableType searchableType; - private final IndexWriter writer; + private final SharableIndexWriter writer; - LuceneIndex(LuceneSearchableType searchableType, IndexWriter writer) { - this.searchableType = searchableType; - this.writer = writer; + LuceneIndex(IndexParams params, Supplier writerFactory) { + this.details = params; + this.searchableType = params.getSearchableType(); + this.writer = new SharableIndexWriter(writerFactory); + this.open(); + } + + void open() { + writer.open(); + } + + @VisibleForTesting + SharableIndexWriter getWriter() { + return writer; + } + + @Override + public IndexDetails getDetails() { + return details; } @Override public void store(Id id, String permission, Object object) { - String uid = createUid(id, searchableType); Document document = searchableType.getTypeConverter().convert(object); try { - field(document, UID, uid); - field(document, ID, id.getValue()); + field(document, ID, id.asString()); id.getRepository().ifPresent(repository -> field(document, REPOSITORY, repository)); - field(document, TYPE, searchableType.getName()); if (!Strings.isNullOrEmpty(permission)) { field(document, PERMISSION, permission); } - writer.updateDocument(new Term(UID, uid), document); + writer.updateDocument(idTerm(id), document); } catch (IOException e) { throw new SearchEngineException("failed to add document to index", e); } } - private String createUid(Id id, LuceneSearchableType type) { - return id.asString() + "/" + type.getName(); + @Nonnull + private Term idTerm(Id id) { + return new Term(ID, id.asString()); } private void field(Document document, String type, String name) { @@ -94,24 +111,11 @@ class LuceneIndex implements Index { private class LuceneDeleter implements Deleter { - @Override - public ByTypeDeleter byType() { - return new LuceneByTypeDeleter(); - } - - @Override - public AllTypesDeleter allTypes() { - return new LuceneAllTypesDelete(); - } - } - - @SuppressWarnings("java:S1192") - private class LuceneByTypeDeleter implements ByTypeDeleter { - @Override public void byId(Id id) { try { - writer.deleteDocuments(new Term(UID, createUid(id, searchableType))); + long count = writer.deleteDocuments(idTerm(id)); + LOG.debug("delete {} document(s) by id {} from index {}", count, id, details); } catch (IOException e) { throw new SearchEngineException("failed to delete document from index", e); } @@ -120,7 +124,8 @@ class LuceneIndex implements Index { @Override public void all() { try { - writer.deleteDocuments(new Term(TYPE, searchableType.getName())); + long count = writer.deleteAll(); + LOG.debug("deleted all {} documents from index {}", count, details); } catch (IOException ex) { throw new SearchEngineException("failed to delete documents by type " + searchableType.getName() + " from index", ex); } @@ -129,35 +134,16 @@ class LuceneIndex implements Index { @Override public void byRepository(String repositoryId) { try { - BooleanQuery query = new BooleanQuery.Builder() - .add(new TermQuery(new Term(TYPE, searchableType.getName())), BooleanClause.Occur.MUST) - .add(new TermQuery(new Term(REPOSITORY, repositoryId)), BooleanClause.Occur.MUST) - .build(); - writer.deleteDocuments(query); + long count = writer.deleteDocuments(repositoryTerm(repositoryId)); + LOG.debug("deleted {} documents by repository {} from index {}", count, repositoryId, details); } catch (IOException ex) { throw new SearchEngineException("failed to delete documents by repository " + repositoryId + " from index", ex); } } - } - private class LuceneAllTypesDelete implements AllTypesDeleter { - - @Override - public void byRepository(String repositoryId) { - try { - writer.deleteDocuments(new Term(REPOSITORY, repositoryId)); - } catch (IOException ex) { - throw new SearchEngineException("failed to delete all documents by repository " + repositoryId + " from index", ex); - } - } - - @Override - public void byTypeName(String typeName) { - try { - writer.deleteDocuments(new Term(TYPE, typeName)); - } catch (IOException ex) { - throw new SearchEngineException("failed to delete documents by type " + typeName + " from index", ex); - } + @Nonnull + private Term repositoryTerm(String repositoryId) { + return new Term(REPOSITORY, repositoryId); } } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java new file mode 100644 index 0000000000..e4010bc115 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexDetails.java @@ -0,0 +1,43 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@Data +@XmlRootElement +@NoArgsConstructor +@AllArgsConstructor +@XmlAccessorType(XmlAccessType.FIELD) +public class LuceneIndexDetails implements IndexDetails { + private Class type; + private String name; +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java index 103fe63c55..aca51bb430 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexFactory.java @@ -24,23 +24,50 @@ package sonia.scm.search; -import javax.inject.Inject; -import java.io.IOException; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Singleton +@SuppressWarnings("unchecked") public class LuceneIndexFactory { - private final IndexOpener indexOpener; + private final IndexManager indexManager; + @SuppressWarnings("rawtypes") + private final Map indexes = new ConcurrentHashMap<>(); @Inject - public LuceneIndexFactory(IndexOpener indexOpener) { - this.indexOpener = indexOpener; + public LuceneIndexFactory(IndexManager indexManager) { + this.indexManager = indexManager; } public LuceneIndex create(IndexParams indexParams) { - try { - return new LuceneIndex<>(indexParams.getSearchableType(), indexOpener.openForWrite(indexParams)); - } catch (IOException ex) { - throw new SearchEngineException("failed to open index " + indexParams.getIndex(), ex); - } + return indexes.compute(keyOf(indexParams), (key, index) -> { + if (index != null) { + index.open(); + return index; + } + return new LuceneIndex<>( + indexParams, + () -> indexManager.openForWrite(indexParams) + ); + }); + } + + private IndexKey keyOf(IndexParams indexParams) { + return new IndexKey( + indexParams.getSearchableType().getName(), indexParams.getIndex() + ); + } + + @EqualsAndHashCode + @AllArgsConstructor + private static class IndexKey { + String type; + String name; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java new file mode 100644 index 0000000000..589ec39945 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneIndexTask.java @@ -0,0 +1,76 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.Injector; + +import javax.inject.Inject; +import java.io.Serializable; + +public abstract class LuceneIndexTask implements Runnable, Serializable { + + private final Class type; + private final String indexName; + private final IndexOptions options; + + private transient LuceneIndexFactory indexFactory; + private transient SearchableTypeResolver searchableTypeResolver; + private transient Injector injector; + + protected LuceneIndexTask(IndexParams params) { + this.type = params.getSearchableType().getType(); + this.indexName = params.getIndex(); + this.options = params.getOptions(); + } + + @Inject + public void setIndexFactory(LuceneIndexFactory indexFactory) { + this.indexFactory = indexFactory; + } + + @Inject + public void setSearchableTypeResolver(SearchableTypeResolver searchableTypeResolver) { + this.searchableTypeResolver = searchableTypeResolver; + } + + @Inject + public void setInjector(Injector injector) { + this.injector = injector; + } + + public abstract IndexTask task(Injector injector); + + @SuppressWarnings({"unchecked", "rawtypes"}) + public void run() { + LuceneSearchableType searchableType = searchableTypeResolver.resolve(type); + IndexTask task = task(injector); + try (LuceneIndex index = indexFactory.create(new IndexParams(indexName, searchableType, options))) { + task.update(index); + } + task.afterUpdate(); + } + + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java similarity index 58% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java rename to scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java index 2a7c152f97..a62feb5b60 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTaskWrapper.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneInjectingIndexTask.java @@ -24,31 +24,21 @@ package sonia.scm.search; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.google.inject.Injector; +import sonia.scm.work.Task; -public final class IndexQueueTaskWrapper implements Runnable { +@SuppressWarnings("rawtypes") +public class LuceneInjectingIndexTask extends LuceneIndexTask implements Task { - private static final Logger LOG = LoggerFactory.getLogger(IndexQueueTaskWrapper.class); + private final Class taskClass; - private final LuceneIndexFactory indexFactory; - private final IndexParams indexParams; - private final Iterable> tasks; - - IndexQueueTaskWrapper(LuceneIndexFactory indexFactory, IndexParams indexParams, Iterable> tasks) { - this.indexFactory = indexFactory; - this.indexParams = indexParams; - this.tasks = tasks; + LuceneInjectingIndexTask(IndexParams params, Class taskClass) { + super(params); + this.taskClass = taskClass; } @Override - public void run() { - try (Index index = indexFactory.create(indexParams)) { - for (IndexQueueTask task : tasks) { - task.updateIndex(index); - } - } catch (Exception e) { - LOG.warn("failure during execution of index task for index {}", indexParams.getIndex(), e); - } + public IndexTask task(Injector injector) { + return injector.getInstance(taskClass); } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java index 35b030bb35..7ac161bc46 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilder.java @@ -54,12 +54,12 @@ public class LuceneQueryBuilder extends QueryBuilder { private static final Logger LOG = LoggerFactory.getLogger(LuceneQueryBuilder.class); - private final IndexOpener opener; + private final IndexManager opener; private final LuceneSearchableType searchableType; private final String indexName; private final Analyzer analyzer; - LuceneQueryBuilder(IndexOpener opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) { + LuceneQueryBuilder(IndexManager opener, String indexName, LuceneSearchableType searchableType, Analyzer analyzer) { this.opener = opener; this.indexName = indexName; this.searchableType = searchableType; @@ -88,11 +88,11 @@ public class LuceneQueryBuilder extends QueryBuilder { String queryString = Strings.nullToEmpty(queryParams.getQueryString()); Query parsedQuery = createQuery(searchableType, queryParams, queryString); - Query query = Queries.filter(parsedQuery, searchableType, queryParams); + Query query = Queries.filter(parsedQuery, queryParams); if (LOG.isDebugEnabled()) { LOG.debug("execute lucene query: {}", query); } - try (IndexReader reader = opener.openForRead(indexName)) { + try (IndexReader reader = opener.openForRead(searchableType, indexName)) { IndexSearcher searcher = new IndexSearcher(reader); searcher.search(query, new PermissionAwareCollector(reader, collector)); diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java index c3e20036bd..b543634a1d 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneQueryBuilderFactory.java @@ -28,18 +28,18 @@ import javax.inject.Inject; public class LuceneQueryBuilderFactory { - private final IndexOpener indexOpener; + private final IndexManager indexManager; private final AnalyzerFactory analyzerFactory; @Inject - public LuceneQueryBuilderFactory(IndexOpener indexOpener, AnalyzerFactory analyzerFactory) { - this.indexOpener = indexOpener; + public LuceneQueryBuilderFactory(IndexManager indexManager, AnalyzerFactory analyzerFactory) { + this.indexManager = indexManager; this.analyzerFactory = analyzerFactory; } public LuceneQueryBuilder create(IndexParams indexParams) { return new LuceneQueryBuilder<>( - indexOpener, + indexManager, indexParams.getIndex(), indexParams.getSearchableType(), analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions()) diff --git a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java index 7a7c5c9b23..9c71f76402 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSearchEngine.java @@ -24,24 +24,34 @@ package sonia.scm.search; +import com.google.common.base.Joiner; import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.Subject; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.CentralWorkQueue.Enqueue; +import sonia.scm.work.Task; import javax.inject.Inject; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; public class LuceneSearchEngine implements SearchEngine { + private final IndexManager indexManager; private final SearchableTypeResolver resolver; - private final IndexQueue indexQueue; private final LuceneQueryBuilderFactory queryBuilderFactory; + private final CentralWorkQueue centralWorkQueue; @Inject - public LuceneSearchEngine(SearchableTypeResolver resolver, IndexQueue indexQueue, LuceneQueryBuilderFactory queryBuilderFactory) { + public LuceneSearchEngine(IndexManager indexManager, SearchableTypeResolver resolver, LuceneQueryBuilderFactory queryBuilderFactory, CentralWorkQueue centralWorkQueue) { + this.indexManager = indexManager; this.resolver = resolver; - this.indexQueue = indexQueue; this.queryBuilderFactory = queryBuilderFactory; + this.centralWorkQueue = centralWorkQueue; } @Override @@ -67,11 +77,79 @@ public class LuceneSearchEngine implements SearchEngine { return new LuceneForType<>(searchableType); } + private void enqueue(LuceneSearchableType searchableType, String index, List resources, Task task) { + Enqueue enqueuer = centralWorkQueue.append(); + + String resourceName = Joiner.on('-').join(searchableType.getName(), index, "index"); + if (resources.isEmpty()) { + enqueuer.locks(resourceName); + } else { + for (String resource : resources) { + enqueuer.locks(resourceName, resource); + } + } + + enqueuer.runAsAdmin().enqueue(task); + } + + @Override + public ForIndices forIndices() { + return new LuceneForIndices(); + } + + class LuceneForIndices implements ForIndices { + + private final List resources = new ArrayList<>(); + private Predicate predicate = details -> true; + private IndexOptions options = IndexOptions.defaults(); + + @Override + public ForIndices matching(Predicate predicate) { + this.predicate = predicate; + return this; + } + + @Override + public ForIndices withOptions(IndexOptions options) { + this.options = options; + return this; + } + + @Override + public ForIndices forResource(String resource) { + this.resources.add(resource); + return this; + } + + @Override + public void batch(SerializableIndexTask task) { + exec(params -> batch(params, new LuceneSimpleIndexTask(params, task))); + } + + @Override + public void batch(Class> task) { + exec(params -> batch(params, new LuceneInjectingIndexTask(params, task))); + } + + private void exec(Consumer consumer) { + indexManager.all() + .stream() + .filter(predicate) + .map(details -> new IndexParams(details.getName(), resolver.resolve(details.getType()), options)) + .forEach(consumer); + } + + private void batch(IndexParams params, Task task) { + LuceneSearchEngine.this.enqueue(params.getSearchableType(), params.getIndex(), resources, task); + } + } + class LuceneForType implements ForType { private final LuceneSearchableType searchableType; private IndexOptions options = IndexOptions.defaults(); private String index = "default"; + private final List resources = new ArrayList<>(); private LuceneForType(LuceneSearchableType searchableType) { this.searchableType = searchableType; @@ -94,8 +172,23 @@ public class LuceneSearchEngine implements SearchEngine { } @Override - public Index getOrCreate() { - return indexQueue.getQueuedIndex(params()); + public ForType forResource(String resource) { + resources.add(resource); + return this; + } + + @Override + public void update(Class> task) { + enqueue(new LuceneInjectingIndexTask(params(), task)); + } + + @Override + public void update(SerializableIndexTask task) { + enqueue(new LuceneSimpleIndexTask(params(), task)); + } + + private void enqueue(Task task) { + LuceneSearchEngine.this.enqueue(searchableType, index, resources, task); } @Override diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java b/scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java similarity index 51% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java rename to scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java index a8d7de863c..c12be9d9e9 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueue.java +++ b/scm-webapp/src/main/java/sonia/scm/search/LuceneSimpleIndexTask.java @@ -24,53 +24,21 @@ package sonia.scm.search; -import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Injector; +import sonia.scm.work.Task; -import javax.inject.Inject; -import javax.inject.Singleton; -import java.io.Closeable; -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicLong; +public final class LuceneSimpleIndexTask extends LuceneIndexTask implements Task { -@Singleton -public class IndexQueue implements Closeable { + private final SerializableIndexTask task; - private final ExecutorService executor = Executors.newSingleThreadExecutor(); - - private final AtomicLong size = new AtomicLong(0); - - private final LuceneIndexFactory indexFactory; - - @Inject - public IndexQueue(LuceneIndexFactory indexFactory) { - this.indexFactory = indexFactory; - } - - public Index getQueuedIndex(IndexParams indexParams) { - return new QueuedIndex<>(this, indexParams); - } - - public LuceneIndexFactory getIndexFactory() { - return indexFactory; - } - - void enqueue(IndexQueueTaskWrapper task) { - size.incrementAndGet(); - executor.execute(() -> { - task.run(); - size.decrementAndGet(); - }); - } - - @VisibleForTesting - long getSize() { - return size.get(); + LuceneSimpleIndexTask(IndexParams params, SerializableIndexTask task) { + super(params); + this.task = task; } @Override - public void close() throws IOException { - executor.shutdown(); + public IndexTask task(Injector injector) { + injector.injectMembers(task); + return task; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/Queries.java b/scm-webapp/src/main/java/sonia/scm/search/Queries.java index 6ec016fb65..af7a1d8953 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/Queries.java +++ b/scm-webapp/src/main/java/sonia/scm/search/Queries.java @@ -29,6 +29,8 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import java.util.Optional; + import static org.apache.lucene.search.BooleanClause.Occur.MUST; final class Queries { @@ -36,19 +38,18 @@ final class Queries { private Queries() { } - private static Query typeQuery(LuceneSearchableType type) { - return new TermQuery(new Term(FieldNames.TYPE, type.getName())); - } - private static Query repositoryQuery(String repositoryId) { return new TermQuery(new Term(FieldNames.REPOSITORY, repositoryId)); } - static Query filter(Query query, LuceneSearchableType searchableType, QueryBuilder.QueryParams params) { - BooleanQuery.Builder builder = new BooleanQuery.Builder() - .add(query, MUST) - .add(typeQuery(searchableType), MUST); - params.getRepositoryId().ifPresent(repo -> builder.add(repositoryQuery(repo), MUST)); - return builder.build(); + static Query filter(Query query, QueryBuilder.QueryParams params) { + Optional repositoryId = params.getRepositoryId(); + if (repositoryId.isPresent()) { + return new BooleanQuery.Builder() + .add(query, MUST) + .add(repositoryQuery(repositoryId.get()), MUST) + .build(); + } + return query; } } diff --git a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java b/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java deleted file mode 100644 index 70ccaef7f2..0000000000 --- a/scm-webapp/src/main/java/sonia/scm/search/QueuedIndex.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import java.util.ArrayList; -import java.util.List; - -class QueuedIndex implements Index { - - private final IndexQueue queue; - private final IndexParams indexParams; - private final List> tasks = new ArrayList<>(); - - QueuedIndex(IndexQueue queue, IndexParams indexParams) { - this.queue = queue; - this.indexParams = indexParams; - } - - @Override - public void store(Id id, String permission, T object) { - tasks.add(index -> index.store(id, permission, object)); - } - - @Override - public Deleter delete() { - return new QueueDeleter(); - } - - @Override - public void close() { - IndexQueueTaskWrapper wrappedTask = new IndexQueueTaskWrapper<>( - queue.getIndexFactory(), indexParams, tasks - ); - queue.enqueue(wrappedTask); - } - - private class QueueDeleter implements Deleter { - - @Override - public ByTypeDeleter byType() { - return new QueueByTypeDeleter(); - } - - @Override - public AllTypesDeleter allTypes() { - return new QueueAllTypesDeleter(); - } - } - - private class QueueByTypeDeleter implements ByTypeDeleter { - - @Override - public void byId(Id id) { - tasks.add(index -> index.delete().byType().byId(id)); - } - - @Override - public void all() { - tasks.add(index -> index.delete().byType().all()); - } - - @Override - public void byRepository(String repositoryId) { - tasks.add(index -> index.delete().byType().byRepository(repositoryId)); - } - } - - private class QueueAllTypesDeleter implements AllTypesDeleter { - - @Override - public void byRepository(String repositoryId) { - tasks.add(index -> index.delete().allTypes().byRepository(repositoryId)); - } - - @Override - public void byTypeName(String typeName) { - tasks.add(index -> index.delete().allTypes().byTypeName(typeName)); - } - } -} diff --git a/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java b/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java new file mode 100644 index 0000000000..769430b1d1 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/search/SharableIndexWriter.java @@ -0,0 +1,90 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.function.Supplier; + +class SharableIndexWriter { + + private static final Logger LOG = LoggerFactory.getLogger(SharableIndexWriter.class); + + private int usageCounter = 0; + + private final Supplier writerFactory; + private IndexWriter writer; + + SharableIndexWriter(Supplier writerFactory) { + this.writerFactory = writerFactory; + } + + synchronized void open() { + usageCounter++; + if (usageCounter == 1) { + LOG.trace("open writer, because usage increased from zero to one"); + writer = writerFactory.get(); + } else { + LOG.trace("new task is using the writer, counter is now at {}", usageCounter); + } + } + + @VisibleForTesting + int getUsageCounter() { + return usageCounter; + } + + void updateDocument(Term term, Document document) throws IOException { + writer.updateDocument(term, document); + } + + long deleteDocuments(Term term) throws IOException { + return writer.deleteDocuments(term); + } + + long deleteAll() throws IOException { + return writer.deleteAll(); + } + + synchronized void close() throws IOException { + usageCounter--; + if (usageCounter == 0) { + LOG.trace("no one seems to use index any longer, closing underlying writer"); + writer.close(); + writer = null; + } else if (usageCounter > 0) { + LOG.trace("index is still used by {} task(s), commit work but keep writer open", usageCounter); + writer.commit(); + } else { + LOG.warn("index is already closed"); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java b/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java new file mode 100644 index 0000000000..0b4b691cd3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/security/Impersonator.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.security; + +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.Subject; +import org.apache.shiro.subject.support.SubjectThreadState; +import org.apache.shiro.util.ThreadContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; + +/** + * Impersonator allows the usage of scm-manager api in the context of another user. + * + * @since 2.23.0 + */ +public final class Impersonator { + + private static final Logger LOG = LoggerFactory.getLogger(Impersonator.class); + + private final SecurityManager securityManager; + + @Inject + public Impersonator(SecurityManager securityManager) { + this.securityManager = securityManager; + } + + public Session impersonate(PrincipalCollection principal) { + Subject subject = createSubject(principal); + if (ThreadContext.getSecurityManager() != null) { + return new WebImpersonator(subject); + } + return new NonWebImpersonator(securityManager, subject); + } + + private Subject createSubject(PrincipalCollection principal) { + return new Subject.Builder(securityManager) + .authenticated(true) + .principals(principal) + .buildSubject(); + } + + public interface Session extends AutoCloseable { + void close(); + } + + private static class WebImpersonator implements Session { + + private final Subject subject; + private final Subject previousSubject; + + private WebImpersonator(Subject subject) { + this.subject = subject; + this.previousSubject = SecurityUtils.getSubject(); + bind(); + } + + private void bind() { + LOG.debug("user {} start impersonate session as {}", previousSubject.getPrincipal(), subject.getPrincipal()); + + + // do not use runas, because we want only bind the session to this thread. + // Runas could affect other threads. + ThreadContext.bind(this.subject); + } + + @Override + public void close() { + LOG.debug("release impersonate session from user {} to {}", previousSubject.getPrincipal(), subject.getPrincipal()); + ThreadContext.bind(previousSubject); + } + + } + + private static class NonWebImpersonator implements Session { + + private final SecurityManager securityManager; + private final SubjectThreadState state; + private final Subject subject; + + private NonWebImpersonator(SecurityManager securityManager, Subject subject) { + this.securityManager = securityManager; + this.state = new SubjectThreadState(subject); + this.subject = subject; + bind(); + } + + private void bind() { + LOG.debug("start impersonate session as user {}", subject.getPrincipal()); + SecurityUtils.setSecurityManager(securityManager); + state.bind(); + } + + @Override + public void close() { + LOG.debug("release impersonate session of {}", subject.getPrincipal()); + state.restore(); + SecurityUtils.setSecurityManager(null); + } + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java b/scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java similarity index 51% rename from scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java rename to scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java index 2e84eb4333..38ae345d08 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexOpener.java +++ b/scm-webapp/src/main/java/sonia/scm/update/index/RemoveCombinedIndex.java @@ -22,44 +22,59 @@ * SOFTWARE. */ -package sonia.scm.search; +package sonia.scm.update.index; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import sonia.scm.SCMContextProvider; +import sonia.scm.migration.UpdateStep; +import sonia.scm.plugin.Extension; +import sonia.scm.util.IOUtil; +import sonia.scm.version.Version; +import javax.annotation.Nonnull; import javax.inject.Inject; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -public class IndexOpener { +import static sonia.scm.store.StoreConstants.DATA_DIRECTORY_NAME; +import static sonia.scm.store.StoreConstants.VARIABLE_DATA_DIRECTORY_NAME; - private final Path directory; - private final AnalyzerFactory analyzerFactory; +@Extension +public class RemoveCombinedIndex implements UpdateStep { + + private final SCMContextProvider contextProvider; @Inject - public IndexOpener(SCMContextProvider context, AnalyzerFactory analyzerFactory) { - directory = context.resolve(Paths.get("index")); - this.analyzerFactory = analyzerFactory; + public RemoveCombinedIndex(SCMContextProvider contextProvider) { + this.contextProvider = contextProvider; } - public IndexReader openForRead(String name) throws IOException { - return DirectoryReader.open(directory(name)); + @Override + public void doUpdate() throws IOException { + Path index = contextProvider.resolve(Paths.get("index")); + if (Files.exists(index)) { + IOUtil.delete(index.toFile()); + } + + Path indexLog = contextProvider.resolve(indexLogPath()); + if (Files.exists(indexLog)) { + IOUtil.delete(indexLog.toFile()); + } } - public IndexWriter openForWrite(IndexParams indexParams) throws IOException { - IndexWriterConfig config = new IndexWriterConfig(analyzerFactory.create(indexParams.getSearchableType(), indexParams.getOptions())); - config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory(indexParams.getIndex()), config); + @Nonnull + private Path indexLogPath() { + return Paths.get(VARIABLE_DATA_DIRECTORY_NAME).resolve(DATA_DIRECTORY_NAME).resolve("index-log"); } - private Directory directory(String name) throws IOException { - return FSDirectory.open(directory.resolve(name)); + @Override + public Version getTargetVersion() { + return Version.parse("2.0.0"); } + @Override + public String getAffectedDataType() { + return "sonia.scm.index"; + } } diff --git a/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java b/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java index 4fd2271c63..383945e714 100644 --- a/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java +++ b/scm-webapp/src/main/java/sonia/scm/user/UserIndexer.java @@ -30,8 +30,10 @@ import sonia.scm.plugin.Extension; import sonia.scm.search.HandlerEventIndexSyncer; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.Indexer; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; import javax.inject.Inject; import javax.inject.Singleton; @@ -43,12 +45,10 @@ public class UserIndexer implements Indexer { @VisibleForTesting static final int VERSION = 1; - private final UserManager userManager; private final SearchEngine searchEngine; @Inject - public UserIndexer(UserManager userManager, SearchEngine searchEngine) { - this.userManager = userManager; + public UserIndexer(SearchEngine searchEngine) { this.searchEngine = searchEngine; } @@ -62,47 +62,46 @@ public class UserIndexer implements Indexer { return VERSION; } - @Subscribe(async = false) - public void handleEvent(UserEvent event) { - new HandlerEventIndexSyncer<>(this).handleEvent(event); + @Override + public Class> getReIndexAllTask() { + return ReIndexAll.class; } @Override - public Updater open() { - return new UserIndexUpdater(userManager, searchEngine.forType(User.class).getOrCreate()); + public SerializableIndexTask createStoreTask(User user) { + return index -> store(index, user); } - public static class UserIndexUpdater implements Updater { + @Override + public SerializableIndexTask createDeleteTask(User item) { + return index -> index.delete().byId(Id.of(item)); + } + + @Subscribe(async = false) + public void handleEvent(UserEvent event) { + new HandlerEventIndexSyncer<>(searchEngine, this).handleEvent(event); + } + + private static void store(Index index, User user) { + index.store(Id.of(user), UserPermissions.read(user).asShiroString(), user); + } + + public static class ReIndexAll extends ReIndexAllTask { private final UserManager userManager; - private final Index index; - private UserIndexUpdater(UserManager userManager, Index index) { + @Inject + public ReIndexAll(IndexLogStore logStore, UserManager userManager) { + super(logStore, User.class, VERSION); this.userManager = userManager; - this.index = index; } @Override - public void store(User user) { - index.store(Id.of(user), UserPermissions.read(user).asShiroString(), user); - } - - @Override - public void delete(User user) { - index.delete().byType().byId(Id.of(user)); - } - - @Override - public void reIndexAll() { - index.delete().byType().all(); + public void update(Index index) { + index.delete().all(); for (User user : userManager.getAll()) { - store(user); + store(index, user); } } - - @Override - public void close() { - index.close(); - } } } diff --git a/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java b/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java index 1ad6876b70..40218c89f0 100644 --- a/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java +++ b/scm-webapp/src/main/java/sonia/scm/web/security/AdministrationContextMarker.java @@ -21,10 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - + package sonia.scm.web.security; -final class AdministrationContextMarker { +import java.io.Serializable; + +final class AdministrationContextMarker implements Serializable { static final AdministrationContextMarker MARKER = new AdministrationContextMarker(); diff --git a/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java b/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java index dc24498094..e0de27a917 100644 --- a/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java +++ b/scm-webapp/src/main/java/sonia/scm/web/security/DefaultAdministrationContext.java @@ -24,223 +24,70 @@ package sonia.scm.web.security; -//~--- non-JDK imports -------------------------------------------------------- - import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Singleton; -import org.apache.shiro.SecurityUtils; import org.apache.shiro.subject.PrincipalCollection; import org.apache.shiro.subject.SimplePrincipalCollection; -import org.apache.shiro.subject.Subject; -import org.apache.shiro.subject.support.SubjectThreadState; -import org.apache.shiro.util.ThreadContext; -import org.apache.shiro.util.ThreadState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sonia.scm.SCMContext; import sonia.scm.security.Authentications; -import sonia.scm.security.Role; +import sonia.scm.security.Impersonator; import sonia.scm.user.User; import sonia.scm.util.AssertUtil; //~--- JDK imports ------------------------------------------------------------ /** - * * @author Sebastian Sdorra */ @Singleton -public class DefaultAdministrationContext implements AdministrationContext -{ +public class DefaultAdministrationContext implements AdministrationContext { - /** Field description */ private static final User SYSTEM_ACCOUNT = new User( Authentications.PRINCIPAL_SYSTEM, "SCM-Manager System Account", null ); - - - /** Field description */ static final String REALM = "AdminRealm"; - /** the logger for DefaultAdministrationContext */ - private static final Logger logger = - LoggerFactory.getLogger(DefaultAdministrationContext.class); + private static final Logger LOG = LoggerFactory.getLogger(DefaultAdministrationContext.class); - //~--- constructors --------------------------------------------------------- + private final Injector injector; + private final Impersonator impersonator; + private final PrincipalCollection adminPrincipal; - /** - * Constructs ... - * - * - * @param injector - * @param securityManager - */ @Inject - public DefaultAdministrationContext(Injector injector, - org.apache.shiro.mgt.SecurityManager securityManager) - { + public DefaultAdministrationContext(Injector injector, Impersonator impersonator) { this.injector = injector; - this.securityManager = securityManager; - - principalCollection = createAdminCollection(SYSTEM_ACCOUNT); + this.impersonator = impersonator; + this.adminPrincipal = createAdminPrincipal(); } - //~--- methods -------------------------------------------------------------- - - /** - * Method description - * - * - * @param action - */ - @Override - public void runAsAdmin(PrivilegedAction action) - { - AssertUtil.assertIsNotNull(action); - - if (ThreadContext.getSecurityManager() != null) - { - doRunAsInWebSessionContext(action); - } - else - { - doRunAsInNonWebSessionContext(action); - } - - } - - /** - * Method description - * - * - * @param actionClass - */ - @Override - public void runAsAdmin(Class actionClass) - { - PrivilegedAction action = injector.getInstance(actionClass); - - runAsAdmin(action); - } - - /** - * Method description - * - * - * @param adminUser - * - * @return - */ - private PrincipalCollection createAdminCollection(User adminUser) - { + public static PrincipalCollection createAdminPrincipal() { SimplePrincipalCollection collection = new SimplePrincipalCollection(); - collection.add(adminUser.getId(), REALM); - collection.add(adminUser, REALM); + collection.add(SYSTEM_ACCOUNT.getId(), REALM); + collection.add(SYSTEM_ACCOUNT, REALM); collection.add(AdministrationContextMarker.MARKER, REALM); return collection; } - /** - * Method description - * - * - * @return - */ - private Subject createAdminSubject() - { - //J- - return new Subject.Builder(securityManager) - .authenticated(true) - .principals(principalCollection) - .buildSubject(); - //J+ - } - - private void doRunAsInNonWebSessionContext(PrivilegedAction action) { - logger.trace("bind shiro security manager to current thread"); - - try { - SecurityUtils.setSecurityManager(securityManager); - - Subject subject = createAdminSubject(); - ThreadState state = new SubjectThreadState(subject); - - state.bind(); - try - { - logger.debug("execute action {} in administration context", action.getClass().getName()); - - action.run(); - } finally { - logger.trace("restore current thread state"); - state.restore(); - } - } finally { - SecurityUtils.setSecurityManager(null); - } - } - - /** - * Method description - * - * - * @param action - */ - private void doRunAsInWebSessionContext(PrivilegedAction action) - { - Subject subject = SecurityUtils.getSubject(); - - String principal = (String) subject.getPrincipal(); - - if (logger.isInfoEnabled()) - { - String username; - - if (subject.hasRole(Role.USER)) - { - username = principal; - } - else - { - username = SCMContext.USER_ANONYMOUS; - } - - logger.debug("user {} executes {} as admin", username, action.getClass().getName()); - } - - Subject adminSubject = createAdminSubject(); - - // do not use runas, because we want only execute this action in this - // thread as administrator. Runas could affect other threads - - ThreadContext.bind(adminSubject); - - try - { + @Override + public void runAsAdmin(PrivilegedAction action) { + AssertUtil.assertIsNotNull(action); + LOG.debug("execute action {} in administration context", action.getClass().getName()); + try (Impersonator.Session session = impersonator.impersonate(adminPrincipal)) { action.run(); } - finally - { - logger.debug("release administration context for user {}/{}", principal, - subject.getPrincipal()); - ThreadContext.bind(subject); - } } - //~--- fields --------------------------------------------------------------- + @Override + public void runAsAdmin(Class actionClass) { + PrivilegedAction action = injector.getInstance(actionClass); + runAsAdmin(action); + } - /** Field description */ - private final Injector injector; - - /** Field description */ - private final org.apache.shiro.mgt.SecurityManager securityManager; - - /** Field description */ - private PrincipalCollection principalCollection; } diff --git a/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java b/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java new file mode 100644 index 0000000000..b5802a17a8 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/DefaultCentralWorkQueue.java @@ -0,0 +1,213 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.subject.PrincipalCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.metrics.Metrics; +import sonia.scm.web.security.DefaultAdministrationContext; + +import javax.annotation.Nullable; +import javax.inject.Singleton; +import java.io.Closeable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.IntSupplier; + +@Singleton +public class DefaultCentralWorkQueue implements CentralWorkQueue, Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultCentralWorkQueue.class); + + private final List queue = new ArrayList<>(); + private final List lockedResources = new ArrayList<>(); + private final AtomicInteger size = new AtomicInteger(); + private final AtomicLong order = new AtomicLong(); + + private final Injector injector; + private final Persistence persistence; + private final ExecutorService executor; + private final MeterRegistry meterRegistry; + + @Inject + public DefaultCentralWorkQueue(Injector injector, Persistence persistence, MeterRegistry meterRegistry) { + this(injector, persistence, meterRegistry, new ThreadCountProvider()); + } + + @VisibleForTesting + DefaultCentralWorkQueue(Injector injector, Persistence persistence, MeterRegistry meterRegistry, IntSupplier threadCountProvider) { + this.injector = injector; + this.persistence = persistence; + this.executor = createExecutorService(meterRegistry, threadCountProvider.getAsInt()); + this.meterRegistry = meterRegistry; + + loadFromDisk(); + } + + private static ExecutorService createExecutorService(MeterRegistry registry, int threadCount) { + ExecutorService executorService = Executors.newFixedThreadPool( + threadCount, + new ThreadFactoryBuilder() + .setNameFormat("CentralWorkQueue-%d") + .build() + ); + Metrics.executor(registry, executorService, "CentralWorkQueue", "fixed"); + return executorService; + } + + @Override + public Enqueue append() { + return new DefaultEnqueue(); + } + + @Override + public int getSize() { + return size.get(); + } + + @Override + public void close() { + executor.shutdown(); + } + + private void loadFromDisk() { + for (UnitOfWork unitOfWork : persistence.loadAll()) { + unitOfWork.restore(order.incrementAndGet()); + append(unitOfWork); + } + run(); + } + + private synchronized void append(UnitOfWork unitOfWork) { + persistence.store(unitOfWork); + int queueSize = size.incrementAndGet(); + queue.add(unitOfWork); + LOG.debug("add task {} to queue, queue size is now {}", unitOfWork, queueSize); + } + + private synchronized void run() { + Iterator iterator = queue.iterator(); + while (iterator.hasNext()) { + UnitOfWork unitOfWork = iterator.next(); + if (isRunnable(unitOfWork)) { + run(unitOfWork); + iterator.remove(); + } else { + unitOfWork.blocked(); + } + } + } + + private void run(UnitOfWork unitOfWork) { + lockedResources.addAll(unitOfWork.getLocks()); + unitOfWork.init(injector, this::finalizeWork, meterRegistry); + LOG.trace("pass task {} to executor", unitOfWork); + executor.execute(unitOfWork); + } + + private synchronized void finalizeWork(UnitOfWork unitOfWork) { + for (Resource lock : unitOfWork.getLocks()) { + lockedResources.remove(lock); + } + persistence.remove(unitOfWork); + + int queueSize = size.decrementAndGet(); + LOG.debug("finish task, queue size is now {}", queueSize); + + run(); + } + + private boolean isRunnable(UnitOfWork unitOfWork) { + for (Resource resource : unitOfWork.getLocks()) { + for (Resource lock : lockedResources) { + if (resource.isBlockedBy(lock)) { + LOG.trace("skip {}, because resource {} is locked by {}", unitOfWork, resource, lock); + return false; + } + } + } + return true; + } + + private class DefaultEnqueue implements Enqueue { + + private final Set locks = new HashSet<>(); + private boolean runAsAdmin = false; + + @Override + public Enqueue locks(String resourceType) { + locks.add(new Resource(resourceType)); + return this; + } + + @Override + public Enqueue locks(String resource, @Nullable String id) { + locks.add(new Resource(resource, id)); + return this; + } + + @Override + public Enqueue runAsAdmin() { + this.runAsAdmin = true; + return this; + } + + @Override + public void enqueue(Task task) { + appendAndRun(new SimpleUnitOfWork(order.incrementAndGet(), principal(), locks, task)); + } + + @Override + public void enqueue(Class task) { + appendAndRun(new InjectingUnitOfWork(order.incrementAndGet(), principal(), locks, task)); + } + + private PrincipalCollection principal() { + if (runAsAdmin) { + return DefaultAdministrationContext.createAdminPrincipal(); + } + return SecurityUtils.getSubject().getPrincipals(); + } + + private synchronized void appendAndRun(UnitOfWork unitOfWork) { + append(unitOfWork); + run(); + } + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java b/scm-webapp/src/main/java/sonia/scm/work/Finalizer.java similarity index 92% rename from scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java rename to scm-webapp/src/main/java/sonia/scm/work/Finalizer.java index f1ac704782..832caa3c6e 100644 --- a/scm-webapp/src/main/java/sonia/scm/search/IndexQueueTask.java +++ b/scm-webapp/src/main/java/sonia/scm/work/Finalizer.java @@ -22,11 +22,11 @@ * SOFTWARE. */ -package sonia.scm.search; +package sonia.scm.work; @FunctionalInterface -public interface IndexQueueTask { +interface Finalizer { - void updateIndex(Index index); + void finalizeWork(UnitOfWork unitOfWork); } diff --git a/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java new file mode 100644 index 0000000000..4d5e442a68 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/InjectingUnitOfWork.java @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.Injector; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; + +import java.util.Set; + +@EqualsAndHashCode(callSuper = true) +class InjectingUnitOfWork extends UnitOfWork { + + private final Class task; + + InjectingUnitOfWork(long order, PrincipalCollection principal, Set locks, Class task) { + super(order, principal, locks); + this.task = task; + } + + @Override + protected Runnable task(Injector injector) { + return injector.getInstance(task); + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/Persistence.java b/scm-webapp/src/main/java/sonia/scm/work/Persistence.java new file mode 100644 index 0000000000..e736f4c1d0 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/Persistence.java @@ -0,0 +1,108 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import org.apache.commons.io.input.ClassLoaderObjectInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.plugin.PluginLoader; +import sonia.scm.store.Blob; +import sonia.scm.store.BlobStore; +import sonia.scm.store.BlobStoreFactory; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +class Persistence { + + private static final Logger LOG = LoggerFactory.getLogger(Persistence.class); + private static final String STORE_NAME = "central-work-queue"; + + private final ClassLoader classLoader; + private final BlobStore store; + + @Inject + public Persistence(PluginLoader pluginLoader, BlobStoreFactory storeFactory) { + this(pluginLoader.getUberClassLoader(), storeFactory.withName(STORE_NAME).build()); + } + + @VisibleForTesting + Persistence(ClassLoader classLoader, BlobStore store) { + this.classLoader = classLoader; + this.store = store; + } + + Collection loadAll() { + List chunks = new ArrayList<>(); + for (Blob blob : store.getAll()) { + load(blob).ifPresent(chunkOfWork -> { + chunkOfWork.assignStorageId(null); + chunks.add(chunkOfWork); + }); + store.remove(blob); + } + Collections.sort(chunks); + return chunks; + } + + private Optional load(Blob blob) { + try (ObjectInputStream stream = new ClassLoaderObjectInputStream(classLoader, blob.getInputStream())) { + Object o = stream.readObject(); + if (o instanceof UnitOfWork) { + return Optional.of((UnitOfWork) o); + } else { + LOG.error("loaded object is not a instance of {}: {}", UnitOfWork.class, o); + } + } catch (IOException | ClassNotFoundException ex) { + LOG.error("failed to load task from store", ex); + } + return Optional.empty(); + } + + void store(UnitOfWork unitOfWork) { + Blob blob = store.create(); + try (ObjectOutputStream outputStream = new ObjectOutputStream(blob.getOutputStream())) { + outputStream.writeObject(unitOfWork); + blob.commit(); + + unitOfWork.assignStorageId(blob.getId()); + } catch (IOException ex) { + throw new NonPersistableTaskException("Failed to persist task", ex); + } + } + + void remove(UnitOfWork unitOfWork) { + unitOfWork.getStorageId().ifPresent(store::remove); + } + +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/Resource.java b/scm-webapp/src/main/java/sonia/scm/work/Resource.java new file mode 100644 index 0000000000..dec2c7fabb --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/Resource.java @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import lombok.EqualsAndHashCode; + +import javax.annotation.Nullable; +import java.io.Serializable; + +@EqualsAndHashCode +final class Resource implements Serializable { + + private final String name; + @Nullable + private final String id; + + Resource(String name) { + this.name = name; + this.id = null; + } + + Resource(String name, @Nullable String id) { + this.name = name; + this.id = id; + } + + boolean isBlockedBy(Resource resource) { + if (name.equals(resource.name)) { + if (id != null && resource.id != null) { + return id.equals(resource.id); + } + return true; + } + return false; + } + + @Override + public String toString() { + if (id != null) { + return name + ":" + id; + } + return name; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java new file mode 100644 index 0000000000..f21a07a6fe --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/SimpleUnitOfWork.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.Injector; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; + +import java.util.Set; + +@EqualsAndHashCode(callSuper = true) +class SimpleUnitOfWork extends UnitOfWork { + + private final Task task; + + SimpleUnitOfWork(long order, PrincipalCollection principal, Set locks, Task task) { + super(order, principal, locks); + this.task = task; + } + + @Override + protected Task task(Injector injector) { + injector.injectMembers(task); + return task; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java b/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java new file mode 100644 index 0000000000..282f1843d3 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/ThreadCountProvider.java @@ -0,0 +1,79 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.IntSupplier; + +public class ThreadCountProvider implements IntSupplier { + + private static final Logger LOG = LoggerFactory.getLogger(ThreadCountProvider.class); + + @VisibleForTesting + static final String PROPERTY = "scm.central-work-queue.workers"; + + private final IntSupplier cpuCountProvider; + + public ThreadCountProvider() { + this(() -> Runtime.getRuntime().availableProcessors()); + } + + @VisibleForTesting + ThreadCountProvider(IntSupplier cpuCountProvider) { + this.cpuCountProvider = cpuCountProvider; + } + + @Override + public int getAsInt() { + Integer systemProperty = Integer.getInteger(PROPERTY); + if (systemProperty == null) { + LOG.debug("derive worker count from cpu count"); + return deriveFromCPUCount(); + } + if (isInvalid(systemProperty)) { + LOG.warn( + "system property {} contains a invalid value {}, fall back and derive worker count from cpu count", + PROPERTY, systemProperty + ); + return deriveFromCPUCount(); + } + return systemProperty; + } + + private boolean isInvalid(int value) { + return value <= 0 || value > 64; + } + + private int deriveFromCPUCount() { + int cpus = cpuCountProvider.getAsInt(); + if (cpus > 1) { + return 4; + } + return 2; + } +} diff --git a/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java b/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java new file mode 100644 index 0000000000..4ccd5b4b41 --- /dev/null +++ b/scm-webapp/src/main/java/sonia/scm/work/UnitOfWork.java @@ -0,0 +1,149 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Stopwatch; +import com.google.inject.Injector; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import sonia.scm.security.Impersonator; +import sonia.scm.security.Impersonator.Session; + +import java.io.Serializable; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +@EqualsAndHashCode +abstract class UnitOfWork implements Runnable, Serializable, Comparable { + + @VisibleForTesting + static final String METRIC_EXECUTION = "cwq.task.execution.duration"; + + @VisibleForTesting + static final String METRIC_WAIT = "cwq.task.wait.duration"; + + private static final Logger LOG = LoggerFactory.getLogger(UnitOfWork.class); + + private long order; + private int blockCount = 0; + private int restoreCount = 0; + private final Set locks; + private final PrincipalCollection principal; + + private transient Finalizer finalizer; + private transient Runnable task; + private transient MeterRegistry meterRegistry; + private transient Impersonator impersonator; + + private transient long createdAt; + private transient String storageId; + + protected UnitOfWork(long order, PrincipalCollection principal, Set locks) { + this.order = order; + this.principal = principal; + this.locks = locks; + this.createdAt = System.nanoTime(); + } + + public long getOrder() { + return order; + } + + public void restore(long newOrderId) { + this.order = newOrderId; + this.createdAt = System.nanoTime(); + this.restoreCount++; + } + + public int getRestoreCount() { + return restoreCount; + } + + public void blocked() { + blockCount++; + } + + public void assignStorageId(String storageId) { + this.storageId = storageId; + } + + public Optional getStorageId() { + return Optional.ofNullable(storageId); + } + + public Set getLocks() { + return locks; + } + + void init(Injector injector, Finalizer finalizer, MeterRegistry meterRegistry) { + this.task = task(injector); + this.finalizer = finalizer; + this.meterRegistry = meterRegistry; + this.impersonator = injector.getInstance(Impersonator.class); + } + + protected abstract Runnable task(Injector injector); + + @Override + public void run() { + Stopwatch sw = Stopwatch.createStarted(); + Timer.Sample sample = Timer.start(meterRegistry); + try (Session session = impersonator.impersonate(principal)) { + task.run(); + LOG.debug("task {} finished successful after {}", task, sw.stop()); + } catch (Exception ex) { + LOG.error("task {} failed after {}", task, sw.stop(), ex); + } finally { + sample.stop(createExecutionTimer()); + createWaitTimer().record(System.nanoTime() - createdAt, TimeUnit.NANOSECONDS); + finalizer.finalizeWork(this); + } + } + + private Timer createExecutionTimer() { + return Timer.builder(METRIC_EXECUTION) + .description("Central work queue task execution duration") + .tags("task", task.getClass().getName()) + .register(meterRegistry); + } + + private Timer createWaitTimer() { + return Timer.builder(METRIC_WAIT) + .description("Central work queue task wait duration") + .tags("task", task.getClass().getName(), "restores", String.valueOf(restoreCount), "blocked", String.valueOf(blockCount)) + .register(meterRegistry); + } + + @Override + public int compareTo(UnitOfWork o) { + return Long.compare(order, o.order); + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java index bf013ed823..099a7966d1 100644 --- a/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/group/GroupIndexerTest.java @@ -24,20 +24,23 @@ package sonia.scm.group; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +57,19 @@ class GroupIndexerTest { @InjectMocks private GroupIndexer indexer; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + + private final Group astronauts = new Group("xml", "astronauts"); + private final Group planetCreators = new Group("xml", "planet-creators"); + @Test void shouldReturnClass() { assertThat(indexer.getType()).isEqualTo(Group.class); @@ -64,58 +80,47 @@ class GroupIndexerTest { assertThat(indexer.getVersion()).isEqualTo(GroupIndexer.VERSION); } - @Nested - class UpdaterTests { - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(GroupIndexer.ReIndexAll.class); + } - private final Group group = new Group("xml", "astronauts"); + @Test + void shouldCreateGroup() { + indexer.createStoreTask(astronauts).update(index); - @BeforeEach - void open() { - when(searchEngine.forType(Group.class).getOrCreate()).thenReturn(index); - } + verify(index).store(Id.of(astronauts), GroupPermissions.read(astronauts).asShiroString(), astronauts); + } - @Test - void shouldStore() { - indexer.open().store(group); + @Test + void shouldDeleteGroup() { + indexer.createDeleteTask(astronauts).update(index); - verify(index).store(Id.of(group), "group:read:astronauts", group); - } + verify(index.delete()).byId(Id.of(astronauts)); + } - @Test - void shouldDeleteById() { - indexer.open().delete(group); + @Test + void shouldReIndexAll() { + when(groupManager.getAll()).thenReturn(Arrays.asList(astronauts, planetCreators)); - verify(index.delete().byType()).byId(Id.of(group)); - } + GroupIndexer.ReIndexAll reIndexAll = new GroupIndexer.ReIndexAll(indexLogStore, groupManager); + reIndexAll.update(index); - @Test - void shouldReIndexAll() { - when(groupManager.getAll()).thenReturn(singletonList(group)); + verify(index.delete()).all(); + verify(index).store(Id.of(astronauts), GroupPermissions.read(astronauts).asShiroString(), astronauts); + verify(index).store(Id.of(planetCreators), GroupPermissions.read(planetCreators).asShiroString(), planetCreators); + } - indexer.open().reIndexAll(); + @Test + void shouldHandleEvents() { + GroupEvent event = new GroupEvent(HandlerEventType.DELETE, astronauts); - verify(index.delete().byType()).all(); - verify(index).store(Id.of(group), "group:read:astronauts", group); - } + indexer.handleEvent(event); - @Test - void shouldHandleEvent() { - GroupEvent event = new GroupEvent(HandlerEventType.DELETE, group); - - indexer.handleEvent(event); - - verify(index.delete().byType()).byId(Id.of(group)); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + verify(searchEngine.forType(Group.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byId(Id.of(astronauts)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java index acdb64c7c8..7114296c69 100644 --- a/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/repository/RepositoryIndexerTest.java @@ -24,20 +24,25 @@ package sonia.scm.repository; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +59,15 @@ class RepositoryIndexerTest { @InjectMocks private RepositoryIndexer indexer; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + @Test void shouldReturnRepositoryClass() { assertThat(indexer.getType()).isEqualTo(Repository.class); @@ -64,61 +78,65 @@ class RepositoryIndexerTest { assertThat(indexer.getVersion()).isEqualTo(RepositoryIndexer.VERSION); } - @Nested - class UpdaterTests { - - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; - - private Repository repository; - - @BeforeEach - void open() { - when(searchEngine.forType(Repository.class).getOrCreate()).thenReturn(index); - repository = new Repository(); - repository.setId("42"); - } - - @Test - void shouldStoreRepository() { - indexer.open().store(repository); - - verify(index).store(Id.of(repository), "repository:read:42", repository); - } - - @Test - void shouldDeleteByRepository() { - indexer.open().delete(repository); - - verify(index.delete().allTypes()).byRepository("42"); - } - - @Test - void shouldReIndexAll() { - when(repositoryManager.getAll()).thenReturn(singletonList(repository)); - - indexer.open().reIndexAll(); - - verify(index.delete().allTypes()).byTypeName(Repository.class.getName()); - verify(index.delete().byType()).all(); - - verify(index).store(Id.of(repository), "repository:read:42", repository); - } - - @Test - void shouldHandleEvent() { - RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, repository); - - indexer.handleEvent(event); - - verify(index.delete().allTypes()).byRepository("42"); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(RepositoryIndexer.ReIndexAll.class); } + + @Test + void shouldCreateRepository() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + + indexer.createStoreTask(heartOfGold).update(index); + + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + } + + @Test + void shouldDeleteRepository() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + + indexer.createDeleteTask(heartOfGold).update(index); + + verify(index.delete()).byRepository(heartOfGold); + } + + @Test + void shouldReIndexAll() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle = RepositoryTestData.create42Puzzle(); + when(repositoryManager.getAll()).thenReturn(Arrays.asList(heartOfGold, puzzle)); + + RepositoryIndexer.ReIndexAll reIndexAll = new RepositoryIndexer.ReIndexAll(indexLogStore, repositoryManager); + reIndexAll.update(index); + + verify(index.delete()).all(); + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + verify(index).store(Id.of(puzzle), RepositoryPermissions.read(puzzle).asShiroString(), puzzle); + } + + @Test + void shouldHandleDeleteEvents() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, heartOfGold); + + indexer.handleEvent(event); + + verify(searchEngine.forIndices().forResource(heartOfGold)).batch(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byRepository(heartOfGold); + } + + @Test + void shouldHandleUpdateEvents() { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.CREATE, heartOfGold); + + indexer.handleEvent(event); + + verify(searchEngine.forType(Repository.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index).store(Id.of(heartOfGold), RepositoryPermissions.read(heartOfGold).asShiroString(), heartOfGold); + } + } diff --git a/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java b/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java index 360788bf22..411a9d693e 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/HandlerEventIndexSyncerTest.java @@ -24,10 +24,12 @@ package sonia.scm.search; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.Answers; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; @@ -35,6 +37,7 @@ import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryEvent; import sonia.scm.repository.RepositoryTestData; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @@ -42,18 +45,23 @@ import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class HandlerEventIndexSyncerTest { + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; + @Mock private Indexer indexer; - @Mock - private Indexer.Updater updater; + @BeforeEach + void setUpIndexer() { + lenient().when(indexer.getType()).thenReturn(Repository.class); + } @ParameterizedTest @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.MATCH_ANY, names = "BEFORE_.*") void shouldIgnoreBeforeEvents(HandlerEventType type) { RepositoryEvent event = new RepositoryEvent(type, RepositoryTestData.create42Puzzle()); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); verifyNoInteractions(indexer); } @@ -61,28 +69,29 @@ class HandlerEventIndexSyncerTest { @ParameterizedTest @EnumSource(value = HandlerEventType.class, mode = EnumSource.Mode.INCLUDE, names = {"CREATE", "MODIFY"}) void shouldStore(HandlerEventType type) { - when(indexer.open()).thenReturn(updater); + SerializableIndexTask store = index -> {}; Repository puzzle = RepositoryTestData.create42Puzzle(); + when(indexer.createStoreTask(puzzle)).thenReturn(store); + RepositoryEvent event = new RepositoryEvent(type, puzzle); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); - verify(updater).store(puzzle); - verify(updater).close(); + verify(searchEngine.forType(Repository.class)).update(store); } @Test void shouldDelete() { - when(indexer.open()).thenReturn(updater); + SerializableIndexTask delete = index -> {}; Repository puzzle = RepositoryTestData.create42Puzzle(); + when(indexer.createDeleteTask(puzzle)).thenReturn(delete); + RepositoryEvent event = new RepositoryEvent(HandlerEventType.DELETE, puzzle); + new HandlerEventIndexSyncer<>(searchEngine, indexer).handleEvent(event); - new HandlerEventIndexSyncer<>(indexer).handleEvent(event); - - verify(updater).delete(puzzle); - verify(updater).close(); + verify(searchEngine.forType(Repository.class)).update(delete); } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java index b6222c9687..a91d11a05f 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexBootstrapListenerTest.java @@ -32,8 +32,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.group.Group; import sonia.scm.repository.Repository; import sonia.scm.user.User; -import sonia.scm.web.security.AdministrationContext; -import sonia.scm.web.security.PrivilegedAction; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -42,47 +40,39 @@ import java.util.HashSet; import java.util.Optional; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; @ExtendWith(MockitoExtension.class) class IndexBootstrapListenerTest { - @Mock - private AdministrationContext administrationContext; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private SearchEngine searchEngine; @Mock(answer = Answers.RETURNS_DEEP_STUBS) private IndexLogStore indexLogStore; @Test void shouldReIndexWithoutLog() { - mockAdminContext(); Indexer indexer = indexer(Repository.class, 1); - Indexer.Updater updater = updater(indexer); mockEmptyIndexLog(Repository.class); doInitialization(indexer); - verify(updater).reIndexAll(); - verify(updater).close(); - verify(indexLogStore.defaultIndex()).log(Repository.class, 1); + verify(searchEngine.forType(Repository.class)).update(RepositoryReIndexAllTask.class); } @Test void shouldReIndexIfVersionWasUpdated() { - mockAdminContext(); Indexer indexer = indexer(User.class, 2); - Indexer.Updater updater = updater(indexer); mockIndexLog(User.class, 1); doInitialization(indexer); - verify(updater).reIndexAll(); - verify(updater).close(); - verify(indexLogStore.defaultIndex()).log(User.class, 2); + verify(searchEngine.forType(User.class)).update(UserReIndexAllTask.class); } @Test @@ -92,7 +82,7 @@ class IndexBootstrapListenerTest { mockIndexLog(Group.class, 3); doInitialization(indexer); - verify(indexer, never()).open(); + verifyNoInteractions(searchEngine); } private void mockIndexLog(Class type, int version) { @@ -107,13 +97,6 @@ class IndexBootstrapListenerTest { when(indexLogStore.defaultIndex().get(type)).thenReturn(Optional.ofNullable(indexLog)); } - private void mockAdminContext() { - doAnswer(ic -> { - PrivilegedAction action = ic.getArgument(0); - action.run(); - return null; - }).when(administrationContext).runAsAdmin(any(PrivilegedAction.class)); - } @SuppressWarnings("rawtypes") private void doInitialization(Indexer... indexers) { @@ -125,7 +108,7 @@ class IndexBootstrapListenerTest { @SuppressWarnings("rawtypes") private IndexBootstrapListener listener(Indexer... indexers) { return new IndexBootstrapListener( - administrationContext, indexLogStore, new HashSet<>(Arrays.asList(indexers)) + searchEngine, indexLogStore, new HashSet<>(Arrays.asList(indexers)) ); } @@ -133,15 +116,38 @@ class IndexBootstrapListenerTest { private Indexer indexer(Class type, int version) { Indexer indexer = mock(Indexer.class); when(indexer.getType()).thenReturn(type); - when(indexer.getVersion()).thenReturn(version); + lenient().when(indexer.getVersion()).thenReturn(version); + lenient().when(indexer.getReIndexAllTask()).thenAnswer(ic -> { + if (type == User.class) { + return UserReIndexAllTask.class; + } + return RepositoryReIndexAllTask.class; + }); return indexer; } - @SuppressWarnings("unchecked") - private Indexer.Updater updater(Indexer indexer) { - Indexer.Updater updater = mock(Indexer.Updater.class); - when(indexer.open()).thenReturn(updater); - return updater; + public static class RepositoryReIndexAllTask extends Indexer.ReIndexAllTask { + + public RepositoryReIndexAllTask(IndexLogStore logStore, Class type, int version) { + super(logStore, type, version); + } + + @Override + public void update(Index index) { + + } + } + + public static class UserReIndexAllTask extends Indexer.ReIndexAllTask { + + public UserReIndexAllTask(IndexLogStore logStore, Class type, int version) { + super(logStore, type, version); + } + + @Override + public void update(Index index) { + + } } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java new file mode 100644 index 0000000000..c3b9cd5e6d --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/IndexManagerTest.java @@ -0,0 +1,188 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.analysis.core.SimpleAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.assertj.core.api.AssertionsForClassTypes; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; +import sonia.scm.plugin.PluginLoader; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Locale; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class IndexManagerTest { + + private Path directory; + + @Mock + private AnalyzerFactory analyzerFactory; + + @Mock + private LuceneSearchableType searchableType; + + @Mock + private SCMContextProvider context; + + @Mock + private PluginLoader pluginLoader; + + private IndexManager indexManager; + + @BeforeEach + void createIndexWriterFactory(@TempDir Path tempDirectory) { + this.directory = tempDirectory; + when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory.resolve("index")); + when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); + when(pluginLoader.getUberClassLoader()).thenReturn(IndexManagerTest.class.getClassLoader()); + indexManager = new IndexManager(context, pluginLoader, analyzerFactory); + } + + @Test + void shouldCreateNewIndex() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("index").resolve("songs").resolve("new-index")).exists(); + } + + @Test + void shouldCreateNewIndexForEachType() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "new-index")) { + addDoc(writer, "Trillian"); + } + assertThat(directory.resolve("index").resolve("songs").resolve("new-index")).exists(); + assertThat(directory.resolve("index").resolve("lyrics").resolve("new-index")).exists(); + } + + @Test + void shouldReturnAllCreatedIndices() throws IOException { + try (IndexWriter writer = open(Songs.class, "special")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "awesome")) { + addDoc(writer, "Trillian"); + } + + assertThat(indexManager.all()) + .anySatisfy(details -> { + assertThat(details.getType()).isEqualTo(Songs.class); + assertThat(details.getName()).isEqualTo("special"); + }) + .anySatisfy(details -> { + assertThat(details.getType()).isEqualTo(Lyrics.class); + assertThat(details.getName()).isEqualTo("awesome"); + }); + } + + @Test + void shouldRestoreIndicesOnCreation() throws IOException { + try (IndexWriter writer = open(Songs.class, "special")) { + addDoc(writer, "Trillian"); + } + try (IndexWriter writer = open(Lyrics.class, "awesome")) { + addDoc(writer, "Trillian"); + } + + assertThat(new IndexManager(context, pluginLoader, analyzerFactory).all()) + .anySatisfy(details -> { + AssertionsForClassTypes.assertThat(details.getType()).isEqualTo(Songs.class); + AssertionsForClassTypes.assertThat(details.getName()).isEqualTo("special"); + }) + .anySatisfy(details -> { + AssertionsForClassTypes.assertThat(details.getType()).isEqualTo(Lyrics.class); + AssertionsForClassTypes.assertThat(details.getName()).isEqualTo("awesome"); + }); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private IndexWriter open(Class type, String indexName) throws IOException { + lenient().when(searchableType.getType()).thenReturn(type); + when(searchableType.getName()).thenReturn(type.getSimpleName().toLowerCase(Locale.ENGLISH)); + return indexManager.openForWrite(new IndexParams(indexName, searchableType, IndexOptions.defaults())); + } + + @Test + void shouldOpenExistingIndex() throws IOException { + try (IndexWriter writer = open(Songs.class, "reused")) { + addDoc(writer, "Dent"); + } + try (IndexWriter writer = open(Songs.class, "reused")) { + assertThat(writer.getFieldNames()).contains("hitchhiker"); + } + } + + @Test + void shouldUseAnalyzerFromFactory() throws IOException { + try (IndexWriter writer = open(Songs.class, "new-index")) { + assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); + } + } + + @Test + void shouldOpenIndexForRead() throws IOException { + try (IndexWriter writer = open(Songs.class, "idx-for-read")) { + addDoc(writer, "Dent"); + } + + try (IndexReader reader = indexManager.openForRead(searchableType, "idx-for-read")) { + assertThat(reader.numDocs()).isOne(); + } + } + + private void addDoc(IndexWriter writer, String name) throws IOException { + Document doc = new Document(); + doc.add(new TextField("hitchhiker", name, Field.Store.YES)); + writer.addDocument(doc); + } + + public static class Songs { + } + + public static class Lyrics { + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java deleted file mode 100644 index 929842104f..0000000000 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexOpenerTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import org.apache.lucene.analysis.core.SimpleAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import sonia.scm.SCMContextProvider; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class IndexOpenerTest { - - private Path directory; - - @Mock - private AnalyzerFactory analyzerFactory; - - @Mock - private LuceneSearchableType searchableType; - - private IndexOpener indexOpener; - - @BeforeEach - void createIndexWriterFactory(@TempDir Path tempDirectory) { - this.directory = tempDirectory; - SCMContextProvider context = mock(SCMContextProvider.class); - when(context.resolve(Paths.get("index"))).thenReturn(tempDirectory); - when(analyzerFactory.create(any(LuceneSearchableType.class), any(IndexOptions.class))).thenReturn(new SimpleAnalyzer()); - indexOpener = new IndexOpener(context, analyzerFactory); - } - - @Test - void shouldCreateNewIndex() throws IOException { - try (IndexWriter writer = open("new-index")) { - addDoc(writer, "Trillian"); - } - assertThat(directory.resolve("new-index")).exists(); - } - - private IndexWriter open(String index) throws IOException { - return indexOpener.openForWrite(new IndexParams(index, searchableType, IndexOptions.defaults())); - } - - @Test - void shouldOpenExistingIndex() throws IOException { - try (IndexWriter writer = open("reused")) { - addDoc(writer, "Dent"); - } - try (IndexWriter writer = open("reused")) { - assertThat(writer.getFieldNames()).contains("hitchhiker"); - } - } - - @Test - void shouldUseAnalyzerFromFactory() throws IOException { - try (IndexWriter writer = open("new-index")) { - assertThat(writer.getAnalyzer()).isInstanceOf(SimpleAnalyzer.class); - } - } - - @Test - void shouldOpenIndexForRead() throws IOException { - try (IndexWriter writer = open("idx-for-read")) { - addDoc(writer, "Dent"); - } - - try (IndexReader reader = indexOpener.openForRead("idx-for-read")) { - assertThat(reader.numDocs()).isOne(); - } - } - - private void addDoc(IndexWriter writer, String name) throws IOException { - Document doc = new Document(); - doc.add(new TextField("hitchhiker", name, Field.Store.YES)); - writer.addDocument(doc); - } - -} diff --git a/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java b/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java deleted file mode 100644 index 660b97bf21..0000000000 --- a/scm-webapp/src/test/java/sonia/scm/search/IndexQueueTest.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020-present Cloudogu GmbH and Contributors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package sonia.scm.search; - -import lombok.Value; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.in; -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class IndexQueueTest { - - private Directory directory; - - private IndexQueue queue; - - @BeforeEach - void createQueue() throws IOException { - directory = new ByteBuffersDirectory(); - IndexOpener opener = mock(IndexOpener.class); - when(opener.openForWrite(any(IndexParams.class))).thenAnswer(ic -> { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory, config); - }); - - LuceneIndexFactory indexFactory = new LuceneIndexFactory(opener); - queue = new IndexQueue(indexFactory); - } - - @AfterEach - void closeQueue() throws IOException { - queue.close(); - directory.close(); - } - - @Test - void shouldWriteToIndex() throws Exception { - try (Index index = getIndex(Account.class)) { - index.store(Id.of("tricia"), null, new Account("tricia", "Trillian", "McMillan")); - index.store(Id.of("dent"), null, new Account("dent", "Arthur", "Dent")); - } - assertDocCount(2); - } - - private Index getIndex(Class type) { - SearchableTypeResolver resolver = new SearchableTypeResolver(type); - LuceneSearchableType searchableType = resolver.resolve(type); - IndexParams indexParams = new IndexParams("default", searchableType, IndexOptions.defaults()); - return queue.getQueuedIndex(indexParams); - } - - @Test - void shouldWriteMultiThreaded() throws Exception { - ExecutorService executorService = Executors.newFixedThreadPool(4); - for (int i = 0; i < 20; i++) { - executorService.execute(new IndexNumberTask(i)); - } - executorService.execute(() -> { - try (Index index = getIndex(IndexedNumber.class)) { - index.delete().byType().byId(Id.of(String.valueOf(12))); - } - }); - executorService.shutdown(); - - assertDocCount(19); - } - - private void assertDocCount(int expectedCount) throws IOException { - // wait until all tasks are finished - await().until(() -> queue.getSize() == 0); - try (DirectoryReader reader = DirectoryReader.open(directory)) { - assertThat(reader.numDocs()).isEqualTo(expectedCount); - } - } - - @Value - @IndexedType - public static class Account { - @Indexed - String username; - @Indexed - String firstName; - @Indexed - String lastName; - } - - @Value - @IndexedType - public static class IndexedNumber { - @Indexed - int value; - } - - public class IndexNumberTask implements Runnable { - - private final int number; - - public IndexNumberTask(int number) { - this.number = number; - } - - @Override - public void run() { - try (Index index = getIndex(IndexedNumber.class)) { - index.store(Id.of(String.valueOf(number)), null, new IndexedNumber(number)); - } - } - } - -} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java new file mode 100644 index 0000000000..dbca039857 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexFactoryTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.user.User; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class LuceneIndexFactoryTest { + + @Mock + private IndexManager indexManager; + + @InjectMocks + private LuceneIndexFactory indexFactory; + + @Test + void shouldCallOpenOnCreation() { + LuceneIndex index = indexFactory.create(params("default", Repository.class)); + assertThat(index.getWriter().getUsageCounter()).isOne(); + } + + @Test + void shouldCallOpenOnReturn() { + indexFactory.create(params("default", Repository.class)); + indexFactory.create(params("default", Repository.class)); + LuceneIndex index = indexFactory.create(params("default", Repository.class)); + assertThat(index.getWriter().getUsageCounter()).isEqualTo(3); + } + + @Test + @SuppressWarnings("AssertBetweenInconvertibleTypes") + void shouldReturnDifferentIndexForDifferentTypes() { + LuceneIndex repository = indexFactory.create(params("default", Repository.class)); + LuceneIndex user = indexFactory.create(params("default", User.class)); + + assertThat(repository).isNotSameAs(user); + } + + @Test + void shouldReturnDifferentIndexForDifferentIndexNames() { + LuceneIndex def = indexFactory.create(params("default", Repository.class)); + LuceneIndex other = indexFactory.create(params("other", Repository.class)); + + assertThat(def).isNotSameAs(other); + } + + @Test + void shouldReturnSameIndex() { + LuceneIndex one = indexFactory.create(params("default", Repository.class)); + LuceneIndex two = indexFactory.create(params("default", Repository.class)); + assertThat(one).isSameAs(two); + } + + private IndexParams params(String indexName, Class type) { + return new IndexParams(indexName, SearchableTypes.create(type), IndexOptions.defaults()); + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java index 701536ca31..f39e997f72 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneIndexTest.java @@ -27,7 +27,6 @@ package sonia.scm.search; import com.google.errorprone.annotations.CanIgnoreReturnValue; import lombok.Value; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -39,12 +38,26 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.repository.RepositoryTestData; +import sonia.scm.repository.RepositoryType; import java.io.IOException; +import java.util.function.Supplier; import static org.assertj.core.api.Assertions.assertThat; -import static sonia.scm.search.FieldNames.*; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; +import static sonia.scm.search.FieldNames.ID; +import static sonia.scm.search.FieldNames.PERMISSION; +import static sonia.scm.search.FieldNames.REPOSITORY; class LuceneIndexTest { @@ -77,15 +90,6 @@ class LuceneIndexTest { assertHits("value", "content", 1); } - @Test - void shouldStoreUidOfObject() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Awesome content which should be indexed")); - } - - assertHits(UID, "one/storable", 1); - } - @Test void shouldStoreIdOfObject() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { @@ -104,15 +108,6 @@ class LuceneIndexTest { assertHits(REPOSITORY, "4211", 1); } - @Test - void shouldStoreTypeOfObject() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some other text")); - } - - assertHits(TYPE, "storable", 1); - } - @Test void shouldDeleteById() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { @@ -120,131 +115,54 @@ class LuceneIndexTest { } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(ONE); + index.delete().byId(ONE); } assertHits(ID, "one", 0); } @Test - void shouldDeleteAllByType() throws IOException { + void shouldDeleteByIdAndRepository() throws IOException { + Repository heartOfGold = RepositoryTestData.createHeartOfGold(); + Repository puzzle42 = RepositoryTestData.createHeartOfGold(); try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("content")); - index.store(Id.of("two"), null, new Storable("content")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(Id.of("three"), null, new OtherStorable("content")); + index.store(ONE.withRepository(heartOfGold), null, new Storable("content")); + index.store(ONE.withRepository(puzzle42), null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().all(); + index.delete().byId(ONE.withRepository(heartOfGold)); } assertHits("value", "content", 1); } @Test - void shouldDeleteByIdAnyType() throws IOException { + void shouldDeleteAll() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE, null, new OtherStorable("Some other text")); + index.store(ONE, null, new Storable("content")); + index.store(TWO, null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(ONE); + index.delete().all(); } - assertHits(ID, "one", 1); - ScoreDoc[] docs = assertHits(ID, "one", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get("value")).isEqualTo("Some other text"); - } - - @Test - void shouldDeleteByIdAndRepository() throws IOException { - Id withRepository = ONE.withRepository("4211"); - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("Some other text")); - index.store(withRepository, null, new Storable("New stuff")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byId(withRepository); - } - - ScoreDoc[] docs = assertHits(ID, "one", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get("value")).isEqualTo("Some other text"); + assertHits("value", "content", 0); } @Test void shouldDeleteByRepository() throws IOException { try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("Some other text")); - index.store(ONE.withRepository("4212"), null, new Storable("New stuff")); + index.store(ONE.withRepository("4211"), null, new Storable("content")); + index.store(TWO.withRepository("4212"), null, new Storable("content")); } try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byRepository("4212"); + index.delete().byRepository("4212"); } - assertHits(ID, "one", 1); - } - - @Test - void shouldDeleteByRepositoryAndType() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("some text")); - index.store(TWO.withRepository("4211"), null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE.withRepository("4211"), null, new OtherStorable("some text")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().byType().byRepository("4211"); - } - - ScoreDoc[] docs = assertHits("value", "text", 1); - Document doc = doc(docs[0].doc); - assertThat(doc.get(TYPE)).isEqualTo("otherStorable"); - } - - @Test - void shouldDeleteAllByRepository() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE.withRepository("4211"), null, new Storable("some text")); - index.store(TWO.withRepository("4211"), null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.store(ONE.withRepository("4211"), null, new OtherStorable("some text")); - } - - try (LuceneIndex index = createIndex(Storable.class)) { - index.delete().allTypes().byRepository("4211"); - } - - assertHits("value", "text", 0); - } - - @Test - void shouldDeleteAllByTypeName() throws IOException { - try (LuceneIndex index = createIndex(Storable.class)) { - index.store(ONE, null, new Storable("some text")); - index.store(TWO, null, new Storable("some text")); - } - - try (LuceneIndex index = createIndex(OtherStorable.class)) { - index.delete().allTypes().byTypeName("storable"); - } - - assertHits("value", "text", 0); + assertHits("value", "content", 1); } @Test @@ -256,12 +174,69 @@ class LuceneIndexTest { assertHits(PERMISSION, "repo:4211:read", 1); } - private Document doc(int doc) throws IOException { - try (DirectoryReader reader = DirectoryReader.open(directory)) { - return reader.document(doc); + @Test + void shouldReturnDetails() { + try (LuceneIndex index = createIndex(Storable.class)) { + IndexDetails details = index.getDetails(); + assertThat(details.getType()).isEqualTo(Storable.class); + assertThat(details.getName()).isEqualTo("default"); } } + @Nested + @ExtendWith(MockitoExtension.class) + class ExceptionTests { + + @Mock + private IndexWriter writer; + + private LuceneIndex index; + + @BeforeEach + void setUpIndex() { + index = createIndex(Storable.class, () -> writer); + } + + @Test + void shouldThrowSearchEngineExceptionOnStore() throws IOException { + when(writer.updateDocument(any(), any())).thenThrow(new IOException("failed to store")); + + Storable storable = new Storable("Some other text"); + assertThrows(SearchEngineException.class, () -> index.store(ONE, null, storable)); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteById() throws IOException { + when(writer.deleteDocuments(any(Term.class))).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, () -> deleter.byId(ONE)); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteAll() throws IOException { + when(writer.deleteAll()).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, deleter::all); + } + + @Test + void shouldThrowSearchEngineExceptionOnDeleteByRepository() throws IOException { + when(writer.deleteDocuments(any(Term.class))).thenThrow(new IOException("failed to delete")); + + Index.Deleter deleter = index.delete(); + assertThrows(SearchEngineException.class, () -> deleter.byRepository("42")); + } + + @Test + void shouldThrowSearchEngineExceptionOnClose() throws IOException { + doThrow(new IOException("failed to delete")).when(writer).close(); + assertThrows(SearchEngineException.class, () -> index.close()); + } + + } + @CanIgnoreReturnValue private ScoreDoc[] assertHits(String field, String value, int expectedHits) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { @@ -272,15 +247,25 @@ class LuceneIndexTest { } } - private LuceneIndex createIndex(Class type) throws IOException { - SearchableTypeResolver resolver = new SearchableTypeResolver(type); - return new LuceneIndex<>(resolver.resolve(type), createWriter()); + private LuceneIndex createIndex(Class type) { + return createIndex(type, this::createWriter); } - private IndexWriter createWriter() throws IOException { + private LuceneIndex createIndex(Class type, Supplier writerFactor) { + SearchableTypeResolver resolver = new SearchableTypeResolver(type); + return new LuceneIndex<>( + new IndexParams("default", resolver.resolve(type), IndexOptions.defaults()), writerFactor + ); + } + + private IndexWriter createWriter() { IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - return new IndexWriter(directory, config); + try { + return new IndexWriter(directory, config); + } catch (IOException ex) { + throw new SearchEngineException("failed to open index writer", ex); + } } @Value @@ -290,11 +275,4 @@ class LuceneIndexTest { String value; } - @Value - @IndexedType - private static class OtherStorable { - @Indexed - String value; - } - } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java new file mode 100644 index 0000000000..a20bd85a6d --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneInjectingIndexTaskTest.java @@ -0,0 +1,98 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.user.User; + +import javax.inject.Inject; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class LuceneInjectingIndexTaskTest { + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private LuceneIndexFactory indexFactory; + + private static String capturedValue; + + private final SearchableTypeResolver resolver = new SearchableTypeResolver(User.class); + + @BeforeEach + void cleanUp() { + capturedValue = "notAsExpected"; + } + + @Test + void shouldInjectAndUpdate() { + Injector injector = createInjector(); + + LuceneSearchableType searchableType = resolver.resolve(User.class); + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + + LuceneInjectingIndexTask task = new LuceneInjectingIndexTask(params, InjectingTask.class); + injector.injectMembers(task); + + task.run(); + + assertThat(capturedValue).isEqualTo("runAsExpected"); + } + + private Injector createInjector() { + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(SearchableTypeResolver.class).toInstance(resolver); + bind(LuceneIndexFactory.class).toInstance(indexFactory); + bind(String.class).toInstance("runAsExpected"); + } + }); + } + + public static class InjectingTask implements IndexTask { + + private final String value; + + @Inject + public InjectingTask(String value) { + this.value = value; + } + + @Override + public void update(Index index) { + capturedValue = value; + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java index 77844de063..5f9cebaf87 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneQueryBuilderTest.java @@ -42,7 +42,6 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; -import org.apache.shiro.authz.AuthorizationException; import org.github.sdorra.jse.ShiroExtension; import org.github.sdorra.jse.SubjectAware; import org.junit.jupiter.api.BeforeEach; @@ -70,7 +69,7 @@ class LuceneQueryBuilderTest { private Directory directory; @Mock - private IndexOpener opener; + private IndexManager opener; @BeforeEach void setUpDirectory() { @@ -181,17 +180,6 @@ class LuceneQueryBuilderTest { assertThat(result.getTotalHits()).isOne(); } - @Test - void shouldIgnoreHitsOfOtherType() throws IOException { - try (IndexWriter writer = writer()) { - writer.addDocument(inetOrgPersonDoc("Arthur", "Dent", "Arthur Dent", "4211")); - writer.addDocument(personDoc("Dent")); - } - - QueryResult result = query(InetOrgPerson.class, "Dent"); - assertThat(result.getTotalHits()).isOne(); - } - @Test void shouldThrowQueryParseExceptionOnInvalidQuery() throws IOException { try (IndexWriter writer = writer()) { @@ -251,17 +239,6 @@ class LuceneQueryBuilderTest { assertThat(result.getHits()).hasSize(3); } - @Test - void shouldReturnOnlyHitsOfTypeForExpertQuery() throws IOException { - try (IndexWriter writer = writer()) { - writer.addDocument(inetOrgPersonDoc("Ford", "Prefect", "Ford Prefect", "4211")); - writer.addDocument(personDoc("Prefect")); - } - - QueryResult result = query(InetOrgPerson.class, "lastName:prefect"); - assertThat(result.getTotalHits()).isEqualTo(1L); - } - @Test void shouldReturnOnlyPermittedHits() throws IOException { try (IndexWriter writer = writer()) { @@ -302,10 +279,11 @@ class LuceneQueryBuilderTest { QueryResult result; try (DirectoryReader reader = DirectoryReader.open(directory)) { - when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(Simple.class); + LuceneSearchableType searchableType = resolver.resolve(Simple.class); + when(opener.openForRead(searchableType, "default")).thenReturn(reader); LuceneQueryBuilder builder = new LuceneQueryBuilder<>( - opener, "default", resolver.resolve(Simple.class), new StandardAnalyzer() + opener, "default", searchableType, new StandardAnalyzer() ); result = builder.repository("cde").execute("content:awesome"); } @@ -560,9 +538,9 @@ class LuceneQueryBuilderTest { private long count(Class type, String queryString) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { - lenient().when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(type); LuceneSearchableType searchableType = resolver.resolve(type); + lenient().when(opener.openForRead(searchableType, "default")).thenReturn(reader); LuceneQueryBuilder builder = new LuceneQueryBuilder( opener, "default", searchableType, new StandardAnalyzer() ); @@ -572,10 +550,11 @@ class LuceneQueryBuilderTest { private QueryResult query(Class type, String queryString, Integer start, Integer limit) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { - lenient().when(opener.openForRead("default")).thenReturn(reader); SearchableTypeResolver resolver = new SearchableTypeResolver(type); LuceneSearchableType searchableType = resolver.resolve(type); - LuceneQueryBuilder builder = new LuceneQueryBuilder( + + lenient().when(opener.openForRead(searchableType, "default")).thenReturn(reader); + LuceneQueryBuilder builder = new LuceneQueryBuilder<>( opener, "default", searchableType, new StandardAnalyzer() ); if (start != null) { @@ -597,14 +576,14 @@ class LuceneQueryBuilderTest { private Document simpleDoc(String content) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); return document; } private Document permissionDoc(String content, String permission) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); document.add(new StringField(FieldNames.PERMISSION, permission, Field.Store.YES)); return document; } @@ -612,7 +591,7 @@ class LuceneQueryBuilderTest { private Document repositoryDoc(String content, String repository) { Document document = new Document(); document.add(new TextField("content", content, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "simple", Field.Store.YES)); document.add(new StringField(FieldNames.REPOSITORY, repository, Field.Store.YES)); return document; } @@ -624,14 +603,14 @@ class LuceneQueryBuilderTest { document.add(new TextField("displayName", displayName, Field.Store.YES)); document.add(new TextField("carLicense", carLicense, Field.Store.YES)); document.add(new StringField(FieldNames.ID, lastName, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "inetOrgPerson", Field.Store.YES)); return document; } private Document personDoc(String lastName) { Document document = new Document(); document.add(new TextField("lastName", lastName, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "person", Field.Store.YES)); return document; } @@ -644,14 +623,14 @@ class LuceneQueryBuilderTest { document.add(new StringField("boolValue", String.valueOf(boolValue), Field.Store.YES)); document.add(new LongPoint("instantValue", instantValue.toEpochMilli())); document.add(new StoredField("instantValue", instantValue.toEpochMilli())); - document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "types", Field.Store.YES)); return document; } private Document denyDoc(String value) { Document document = new Document(); document.add(new TextField("value", value, Field.Store.YES)); - document.add(new StringField(FieldNames.TYPE, "deny", Field.Store.YES)); +// document.add(new StringField(FieldNames.TYPE, "deny", Field.Store.YES)); return document; } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java index e30b67c1f6..f4e1193f3f 100644 --- a/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneSearchEngineTest.java @@ -27,12 +27,20 @@ package sonia.scm.search; import org.apache.shiro.authz.AuthorizationException; import org.github.sdorra.jse.ShiroExtension; import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.repository.Repository; +import sonia.scm.user.User; +import sonia.scm.work.CentralWorkQueue; +import sonia.scm.work.Task; import java.util.Arrays; import java.util.Collection; @@ -46,176 +54,329 @@ import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SubjectAware("trillian") @ExtendWith({MockitoExtension.class, ShiroExtension.class}) class LuceneSearchEngineTest { + @Mock + private IndexManager indexManager; + @Mock private SearchableTypeResolver resolver; - @Mock - private IndexQueue indexQueue; - @Mock private LuceneQueryBuilderFactory queryBuilderFactory; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private CentralWorkQueue centralWorkQueue; + @InjectMocks private LuceneSearchEngine searchEngine; @Mock private LuceneSearchableType searchableType; - @Test - void shouldDelegateGetSearchableTypes() { - List mockedTypes = Collections.singletonList(searchableType("repository")); - when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + @Nested + class GetSearchableTypesTests { - Collection searchableTypes = searchEngine.getSearchableTypes(); + @Test + void shouldDelegateGetSearchableTypes() { + List mockedTypes = Collections.singletonList(searchableType("repository")); + when(resolver.getSearchableTypes()).thenReturn(mockedTypes); - assertThat(searchableTypes).containsAll(mockedTypes); + Collection searchableTypes = searchEngine.getSearchableTypes(); + + assertThat(searchableTypes).containsAll(mockedTypes); + } + + @Test + @SubjectAware(value = "dent", permissions = "user:list") + void shouldExcludeTypesWithoutPermission() { + LuceneSearchableType repository = searchableType("repository"); + LuceneSearchableType user = searchableType("user", "user:list"); + LuceneSearchableType group = searchableType("group", "group:list"); + List mockedTypes = Arrays.asList(repository, user, group); + when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + + Collection searchableTypes = searchEngine.getSearchableTypes(); + + assertThat(searchableTypes).containsOnly(repository, user); + } + + private LuceneSearchableType searchableType(String name) { + return searchableType(name, null); + } + + private LuceneSearchableType searchableType(String name, String permission) { + LuceneSearchableType searchableType = mock(LuceneSearchableType.class); + lenient().when(searchableType.getName()).thenReturn(name); + when(searchableType.getPermission()).thenReturn(Optional.ofNullable(permission)); + return searchableType; + } } - @Test - @SubjectAware(value = "dent", permissions = "user:list") - void shouldExcludeTypesWithoutPermission() { - LuceneSearchableType repository = searchableType("repository"); - LuceneSearchableType user = searchableType("user", "user:list"); - LuceneSearchableType group = searchableType("group", "group:list"); - List mockedTypes = Arrays.asList(repository, user, group); - when(resolver.getSearchableTypes()).thenReturn(mockedTypes); + @Nested + class SearchTests { - Collection searchableTypes = searchEngine.getSearchableTypes(); + @Test + @SuppressWarnings("unchecked") + void shouldDelegateSearchWithDefaults() { + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); - assertThat(searchableTypes).containsOnly(repository, user); + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); + + QueryBuilder queryBuilder = searchEngine.forType(Repository.class).search(); + + assertThat(queryBuilder).isSameAs(mockedBuilder); + } + + @Test + @SuppressWarnings("unchecked") + void shouldDelegateSearch() { + IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + IndexParams params = new IndexParams("idx", searchableType, options); + when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); + + QueryBuilder queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search(); + + assertThat(queryBuilder).isSameAs(mockedBuilder); + } + + @Test + void shouldFailWithoutRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + SearchEngine.ForType forType = searchEngine.forType(Repository.class); + assertThrows(AuthorizationException.class, forType::search); + } + + @Test + @SuppressWarnings("unchecked") + @SubjectAware(permissions = "repository:read") + void shouldNotFailWithRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolve(Repository.class)).thenReturn(searchableType); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); + + SearchEngine.ForType forType = searchEngine.forType(Repository.class); + assertThat(forType.search()).isNotNull(); + } + + @Test + void shouldFailWithTypeNameWithoutRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolveByName("repository")).thenReturn(searchableType); + + SearchEngine.ForType forType = searchEngine.forType("repository"); + assertThrows(AuthorizationException.class, forType::search); + } + + @Test + @SuppressWarnings("unchecked") + @SubjectAware(permissions = "repository:read") + void shouldNotFailWithTypeNameAndRequiredPermission() { + when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); + when(resolver.resolveByName("repository")).thenReturn(searchableType); + + LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); + when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); + + SearchEngine.ForType forType = searchEngine.forType("repository"); + assertThat(forType.search()).isNotNull(); + } } - private LuceneSearchableType searchableType(String name) { - return searchableType(name, null); + @Nested + class IndexTests { + + @Captor + private ArgumentCaptor taskCaptor; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private CentralWorkQueue.Enqueue enqueue; + + @BeforeEach + void setUp() { + when(centralWorkQueue.append()).thenReturn(enqueue); + } + + @Test + void shouldSubmitSimpleTask() { + mockType(); + + searchEngine.forType(Repository.class).update(index -> {}); + + verifyTaskSubmitted(LuceneSimpleIndexTask.class); + } + + @Test + void shouldSubmitInjectingTask() { + mockType(); + + searchEngine.forType(Repository.class).update(DummyIndexTask.class); + + verifyTaskSubmitted(LuceneInjectingIndexTask.class); + } + + @Test + void shouldLockTypeAndDefaultIndex() { + mockType(); + + searchEngine.forType(Repository.class).update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index"); + } + + @Test + void shouldLockTypeAndIndex() { + mockType(); + + searchEngine.forType(Repository.class).withIndex("sample").update(DummyIndexTask.class); + + verify(enqueue).locks("repository-sample-index"); + } + + @Test + void shouldLockSpecificResource() { + mockType(); + + searchEngine.forType(Repository.class).forResource("one").update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index", "one"); + } + + @Test + void shouldLockMultipleSpecificResources() { + mockType(); + + searchEngine.forType(Repository.class) + .forResource("one") + .forResource("two") + .update(DummyIndexTask.class); + + verify(enqueue).locks("repository-default-index", "one"); + verify(enqueue).locks("repository-default-index", "two"); + } + + @Test + void shouldBatchSimpleTask() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(index -> {}); + + verifyTaskSubmitted(LuceneSimpleIndexTask.class); + } + + @Test + void shouldBatchAndLock() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(index -> {}); + + verify(enqueue).locks("repository-default-index"); + } + + @Test + void shouldBatchAndLockSpecificResource() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().forResource("one").batch(index -> {}); + + verify(enqueue).locks("repository-default-index", "one"); + } + + @Test + void shouldBatchAndLockMultipleSpecificResources() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().forResource("one").forResource("two").batch(index -> {}); + + verify(enqueue).locks("repository-default-index", "one"); + verify(enqueue).locks("repository-default-index", "two"); + } + + @Test + void shouldBatchInjectingTask() { + mockDetails(new LuceneIndexDetails(Repository.class, "default")); + + searchEngine.forIndices().batch(DummyIndexTask.class); + + verifyTaskSubmitted(LuceneInjectingIndexTask.class); + } + + @Test + void shouldBatchMultipleTasks() { + mockDetails( + new LuceneIndexDetails(Repository.class, "default"), + new LuceneIndexDetails(User.class, "default") + ); + + searchEngine.forIndices().batch(index -> {}); + + verify(enqueue.runAsAdmin(), times(2)).enqueue(any(Task.class)); + } + + @Test + void shouldFilterWithPredicate() { + mockDetails( + new LuceneIndexDetails(Repository.class, "default"), + new LuceneIndexDetails(User.class, "default") + ); + + searchEngine.forIndices() + .matching(details -> details.getType() == Repository.class) + .batch(index -> {}); + + verify(enqueue.runAsAdmin()).enqueue(any(Task.class)); + } + + private void mockDetails(LuceneIndexDetails... details) { + for (LuceneIndexDetails detail : details) { + mockType(detail.getType()); + } + when(indexManager.all()).thenAnswer(ic -> Arrays.asList(details)); + } + + private void verifyTaskSubmitted(Class typeOfTask) { + verify(enqueue.runAsAdmin()).enqueue(taskCaptor.capture()); + + Task task = taskCaptor.getValue(); + assertThat(task).isInstanceOf(typeOfTask); + } + + private void mockType() { + mockType(Repository.class); + } + + private void mockType(Class type){ + LuceneSearchableType searchableType = mock(LuceneSearchableType.class); + lenient().when(searchableType.getType()).thenAnswer(ic -> type); + lenient().when(searchableType.getName()).thenReturn(type.getSimpleName().toLowerCase(Locale.ENGLISH)); + lenient().when(resolver.resolve(type)).thenReturn(searchableType); + } + } - private LuceneSearchableType searchableType(String name, String permission) { - LuceneSearchableType searchableType = mock(LuceneSearchableType.class); - lenient().when(searchableType.getName()).thenReturn(name); - when(searchableType.getPermission()).thenReturn(Optional.ofNullable(permission)); - return searchableType; - } - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateWithDefaultIndex() { - Index index = mock(Index.class); + public static class DummyIndexTask implements IndexTask { - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); + @Override + public void update(Index index) { - Index idx = searchEngine.forType(Repository.class).getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateIndexWithDefaults() { - Index index = mock(Index.class); - - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("idx", searchableType, IndexOptions.defaults()); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); - - Index idx = searchEngine.forType(Repository.class).withIndex("idx").getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateGetOrCreateIndex() { - Index index = mock(Index.class); - IndexOptions options = IndexOptions.naturalLanguage(Locale.ENGLISH); - - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - IndexParams params = new IndexParams("default", searchableType, options); - when(indexQueue.getQueuedIndex(params)).thenReturn(index); - - Index idx = searchEngine.forType(Repository.class).withOptions(options).getOrCreate(); - assertThat(idx).isSameAs(index); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateSearchWithDefaults() { - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); - when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); - - QueryBuilder queryBuilder = searchEngine.forType(Repository.class).search(); - - assertThat(queryBuilder).isSameAs(mockedBuilder); - } - - @Test - @SuppressWarnings("unchecked") - void shouldDelegateSearch() { - IndexOptions options = IndexOptions.naturalLanguage(Locale.GERMAN); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - IndexParams params = new IndexParams("idx", searchableType, options); - when(queryBuilderFactory.create(params)).thenReturn(mockedBuilder); - - QueryBuilder queryBuilder = searchEngine.forType(Repository.class).withIndex("idx").withOptions(options).search(); - - assertThat(queryBuilder).isSameAs(mockedBuilder); - } - - @Test - void shouldFailWithoutRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - SearchEngine.ForType forType = searchEngine.forType(Repository.class); - assertThrows(AuthorizationException.class, forType::search); - } - - @Test - @SuppressWarnings("unchecked") - @SubjectAware(permissions = "repository:read") - void shouldNotFailWithRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolve(Repository.class)).thenReturn(searchableType); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); - - SearchEngine.ForType forType = searchEngine.forType(Repository.class); - assertThat(forType.search()).isNotNull(); - } - - @Test - void shouldFailWithTypeNameWithoutRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolveByName("repository")).thenReturn(searchableType); - - SearchEngine.ForType forType = searchEngine.forType("repository"); - assertThrows(AuthorizationException.class, forType::search); - } - - @Test - @SuppressWarnings("unchecked") - @SubjectAware(permissions = "repository:read") - void shouldNotFailWithTypeNameAndRequiredPermission() { - when(searchableType.getPermission()).thenReturn(Optional.of("repository:read")); - when(resolver.resolveByName("repository")).thenReturn(searchableType); - - LuceneQueryBuilder mockedBuilder = mock(LuceneQueryBuilder.class); - when(queryBuilderFactory.create(any())).thenReturn(mockedBuilder); - - SearchEngine.ForType forType = searchEngine.forType("repository"); - assertThat(forType.search()).isNotNull(); + } } } diff --git a/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java b/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java new file mode 100644 index 0000000000..96617a2842 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/LuceneSimpleIndexTaskTest.java @@ -0,0 +1,83 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class LuceneSimpleIndexTaskTest { + + @Mock + private LuceneIndexFactory indexFactory; + + @Mock + private LuceneIndex index; + + private final SearchableTypeResolver resolver = new SearchableTypeResolver(Repository.class); + + @Test + void shouldUpdate() { + Injector injector = createInjector(); + + LuceneSearchableType searchableType = resolver.resolve(Repository.class); + + IndexParams params = new IndexParams("default", searchableType, IndexOptions.defaults()); + + AtomicReference> ref = new AtomicReference<>(); + LuceneSimpleIndexTask task = new LuceneSimpleIndexTask(params, ref::set); + injector.injectMembers(task); + + when(indexFactory.create(params)).then(ic -> index); + + task.run(); + + assertThat(index).isSameAs(ref.get()); + verify(index).close(); + } + + private Injector createInjector() { + return Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(SearchableTypeResolver.class).toInstance(resolver); + bind(LuceneIndexFactory.class).toInstance(indexFactory); + } + }); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java b/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java new file mode 100644 index 0000000000..327c058f7a --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/search/SharableIndexWriterTest.java @@ -0,0 +1,249 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SharableIndexWriterTest { + + @Mock + private IndexWriter underlyingWriter; + + @Test + @SuppressWarnings("unchecked") + void shouldCreateIndexOnOpen() { + Supplier supplier = mock(Supplier.class); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + verifyNoInteractions(supplier); + + writer.open(); + verify(supplier).get(); + } + + @Test + @SuppressWarnings("unchecked") + void shouldOpenWriterOnlyOnce() { + Supplier supplier = mock(Supplier.class); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + writer.open(); + writer.open(); + writer.open(); + + verify(supplier).get(); + } + + @Test + void shouldIncreaseUsageCounter() { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + assertThat(writer.getUsageCounter()).isEqualTo(3); + } + + @Test + void shouldDecreaseUsageCounter() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + assertThat(writer.getUsageCounter()).isEqualTo(3); + + writer.close(); + writer.close(); + + assertThat(writer.getUsageCounter()).isOne(); + } + + @Test + void shouldNotCloseWriterIfUsageCounterIsGreaterZero() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + writer.open(); + + writer.close(); + writer.close(); + + verify(underlyingWriter, never()).close(); + } + + @Test + void shouldCloseIfUsageCounterIsZero() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + writer.open(); + + writer.close(); + writer.close(); + + verify(underlyingWriter).close(); + } + + @Test + @SuppressWarnings("unchecked") + void shouldReOpen() throws IOException { + Supplier supplier = mock(Supplier.class); + when(supplier.get()).thenReturn(underlyingWriter); + + SharableIndexWriter writer = new SharableIndexWriter(supplier); + writer.open(); + + writer.close(); + writer.open(); + + verify(supplier, times(2)).get(); + verify(underlyingWriter).close(); + } + + @Test + void shouldDelegateUpdates() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + Term term = new Term("field", "value"); + Document document = new Document(); + writer.updateDocument(term, document); + + verify(underlyingWriter).updateDocument(term, document); + } + + @Test + void shouldDelegateDeleteAll() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + writer.deleteAll(); + + verify(underlyingWriter).deleteAll(); + } + + @Test + void shouldDelegateDeletes() throws IOException { + SharableIndexWriter writer = new SharableIndexWriter(() -> underlyingWriter); + writer.open(); + + Term term = new Term("field", "value"); + writer.deleteDocuments(term); + + verify(underlyingWriter).deleteDocuments(term); + } + + @Nested + class ConcurrencyTests { + + private ExecutorService executorService; + + private final AtomicInteger openCounter = new AtomicInteger(); + private final AtomicInteger commitCounter = new AtomicInteger(); + private final AtomicInteger closeCounter = new AtomicInteger(); + + private final AtomicInteger invocations = new AtomicInteger(); + + private SharableIndexWriter writer; + + @BeforeEach + void setUp() throws IOException { + executorService = Executors.newFixedThreadPool(4); + writer = new SharableIndexWriter(() -> { + openCounter.incrementAndGet(); + return underlyingWriter; + }); + + doAnswer(ic -> commitCounter.incrementAndGet()).when(underlyingWriter).commit(); + doAnswer(ic -> closeCounter.incrementAndGet()).when(underlyingWriter).close(); + } + + @Test + @SuppressWarnings("java:S2925") // sleep is ok to simulate some work + void shouldKeepIndexOpen() { + AtomicBoolean fail = new AtomicBoolean(false); + for (int i = 0; i < 50; i++) { + executorService.submit(() -> { + writer.open(); + try { + Thread.sleep(25); + writer.deleteAll(); + writer.close(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail.set(true); + } catch (IOException e) { + fail.set(true); + } finally { + invocations.incrementAndGet(); + } + }); + } + + executorService.shutdown(); + + await().atMost(2, TimeUnit.SECONDS).until(() -> invocations.get() == 50); + + assertThat(fail.get()).isFalse(); + + // It should be one, but it is possible that tasks finish before new added to the queue. + // This behaviour depends heavily on the cpu's of the machine which executes this test. + assertThat(openCounter.get()).isPositive().isLessThan(10); + // should be 49, but see comment above + assertThat(commitCounter.get()).isGreaterThan(40); + // should be 1, but see comment above + assertThat(closeCounter.get()).isPositive().isLessThan(10); + } + + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java b/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java new file mode 100644 index 0000000000..24b07c2485 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/security/ImpersonatorTest.java @@ -0,0 +1,91 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.security; + + +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.UnavailableSecurityManagerException; +import org.apache.shiro.mgt.DefaultSecurityManager; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import sonia.scm.security.Impersonator.Session; + +import javax.annotation.Nonnull; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ImpersonatorTest { + + private SecurityManager securityManager = new DefaultSecurityManager(); + + private Impersonator impersonator; + + @BeforeEach + void setUp() { + impersonator = new Impersonator(securityManager); + } + + @Test + void shouldBindAndRestoreNonWebThread() { + try (Session session = impersonator.impersonate(principal("dent"))) { + assertPrincipal("dent"); + } + assertThrows(UnavailableSecurityManagerException.class, SecurityUtils::getSubject); + } + + @Nonnull + private SimplePrincipalCollection principal(String principal) { + return new SimplePrincipalCollection(principal, "test"); + } + + private void assertPrincipal(String principal) { + assertThat(SecurityUtils.getSubject().getPrincipal()).isEqualTo(principal); + } + + @Nested + @ExtendWith(ShiroExtension.class) + class WithSecurityManager { + + @Test + @SubjectAware("trillian") + void shouldBindAndRestoreWebThread() { + assertPrincipal("trillian"); + + try (Session session = impersonator.impersonate(principal("slarti"))) { + assertPrincipal("slarti"); + } + + assertPrincipal("trillian"); + } + + } +} diff --git a/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java b/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java new file mode 100644 index 0000000000..0c34d1147f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/update/index/RemoveCombinedIndexTest.java @@ -0,0 +1,91 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.update.index; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.SCMContextProvider; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class RemoveCombinedIndexTest { + + private Path home; + + @Mock + private SCMContextProvider contextProvider; + + @InjectMocks + private RemoveCombinedIndex updateStep; + + @BeforeEach + void setUp(@TempDir Path home) { + this.home = home; + when(contextProvider.resolve(any())).then( + ic -> home.resolve(ic.getArgument(0, Path.class)) + ); + } + + @Test + void shouldRemoveIndexDirectory() throws IOException { + Path indexDirectory = home.resolve("index"); + Path specificIndexDirectory = indexDirectory.resolve("repository").resolve("default"); + Files.createDirectories(specificIndexDirectory); + Path helloTxt = specificIndexDirectory.resolve("hello.txt"); + Files.write(helloTxt, "hello".getBytes(StandardCharsets.UTF_8)); + + updateStep.doUpdate(); + + assertThat(helloTxt).doesNotExist(); + assertThat(indexDirectory).doesNotExist(); + } + + @Test + void shouldRemoveIndexLogDirectory() throws IOException { + Path logDirectory = home.resolve("var").resolve("data").resolve("index-log"); + Files.createDirectories(logDirectory); + Path helloXml = logDirectory.resolve("hello.xml"); + Files.write(helloXml, "world".getBytes(StandardCharsets.UTF_8)); + + updateStep.doUpdate(); + + assertThat(helloXml).doesNotExist(); + assertThat(logDirectory).doesNotExist(); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java b/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java index 534931ead6..7321bdef53 100644 --- a/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java +++ b/scm-webapp/src/test/java/sonia/scm/user/UserIndexerTest.java @@ -24,20 +24,23 @@ package sonia.scm.user; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.HandlerEventType; import sonia.scm.search.Id; import sonia.scm.search.Index; +import sonia.scm.search.IndexLogStore; import sonia.scm.search.SearchEngine; +import sonia.scm.search.SerializableIndexTask; + +import java.util.Arrays; -import static java.util.Collections.singletonList; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -54,6 +57,16 @@ class UserIndexerTest { @InjectMocks private UserIndexer indexer; + + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private Index index; + + @Mock + private IndexLogStore indexLogStore; + + @Captor + private ArgumentCaptor> captor; + @Test void shouldReturnType() { assertThat(indexer.getType()).isEqualTo(User.class); @@ -64,58 +77,54 @@ class UserIndexerTest { assertThat(indexer.getVersion()).isEqualTo(UserIndexer.VERSION); } - @Nested - class UpdaterTests { - @Mock(answer = Answers.RETURNS_DEEP_STUBS) - private Index index; + @Test + void shouldReturnReIndexAllClass() { + assertThat(indexer.getReIndexAllTask()).isEqualTo(UserIndexer.ReIndexAll.class); + } - private final User user = UserTestData.createTrillian(); + @Test + void shouldCreateUser() { + User trillian = UserTestData.createTrillian(); - @BeforeEach - void open() { - when(searchEngine.forType(User.class).getOrCreate()).thenReturn(index); - } + indexer.createStoreTask(trillian).update(index); - @Test - void shouldStore() { - indexer.open().store(user); + verify(index).store(Id.of(trillian), UserPermissions.read(trillian).asShiroString(), trillian); + } - verify(index).store(Id.of(user), "user:read:trillian", user); - } + @Test + void shouldDeleteUser() { + User trillian = UserTestData.createTrillian(); - @Test - void shouldDeleteById() { - indexer.open().delete(user); + indexer.createDeleteTask(trillian).update(index); - verify(index.delete().byType()).byId(Id.of(user)); - } + verify(index.delete()).byId(Id.of(trillian)); + } - @Test - void shouldReIndexAll() { - when(userManager.getAll()).thenReturn(singletonList(user)); + @Test + void shouldReIndexAll() { + User trillian = UserTestData.createTrillian(); + User slarti = UserTestData.createSlarti(); + when(userManager.getAll()).thenReturn(Arrays.asList(trillian, slarti)); - indexer.open().reIndexAll(); + UserIndexer.ReIndexAll reIndexAll = new UserIndexer.ReIndexAll(indexLogStore, userManager); + reIndexAll.update(index); - verify(index.delete().byType()).all(); - verify(index).store(Id.of(user), "user:read:trillian", user); - } + verify(index.delete()).all(); + verify(index).store(Id.of(trillian), UserPermissions.read(trillian).asShiroString(), trillian); + verify(index).store(Id.of(slarti), UserPermissions.read(slarti).asShiroString(), slarti); + } - @Test - void shouldHandleEvent() { - UserEvent event = new UserEvent(HandlerEventType.DELETE, user); + @Test + void shouldHandleEvents() { + User trillian = UserTestData.createTrillian(); + UserEvent event = new UserEvent(HandlerEventType.DELETE, trillian); - indexer.handleEvent(event); + indexer.handleEvent(event); - verify(index.delete().byType()).byId(Id.of(user)); - } - - @Test - void shouldCloseIndex() { - indexer.open().close(); - - verify(index).close(); - } + verify(searchEngine.forType(User.class)).update(captor.capture()); + captor.getValue().update(index); + verify(index.delete()).byId(Id.of(trillian)); } } diff --git a/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java b/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java index b6a946e61e..470201c1b4 100644 --- a/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java +++ b/scm-webapp/src/test/java/sonia/scm/web/security/DefaultAdministrationContextTest.java @@ -37,6 +37,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import sonia.scm.security.Authentications; +import sonia.scm.security.Impersonator; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.when; @@ -54,7 +55,7 @@ class DefaultAdministrationContextTest { Injector injector = Guice.createInjector(); SecurityManager securityManager = new DefaultSecurityManager(); - context = new DefaultAdministrationContext(injector, securityManager); + context = new DefaultAdministrationContext(injector, new Impersonator(securityManager)); } @Test diff --git a/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java b/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java new file mode 100644 index 0000000000..558f649472 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/DefaultCentralWorkQueueTest.java @@ -0,0 +1,375 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Inject; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.apache.shiro.mgt.SecurityManager; +import org.apache.shiro.SecurityUtils; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.github.sdorra.jse.ShiroExtension; +import org.github.sdorra.jse.SubjectAware; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.repository.Repository; +import sonia.scm.security.Authentications; + +import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.when; + +@SubjectAware("trillian") +@ExtendWith({MockitoExtension.class, ShiroExtension.class}) +class DefaultCentralWorkQueueTest { + + private final PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + private static final int ITERATIONS = 50; + private static final int TIMEOUT = 1; // seconds + + @Mock + private Persistence persistence; + + @Nested + class WithDefaultInjector { + + private MeterRegistry meterRegistry; + private DefaultCentralWorkQueue queue; + + @BeforeEach + void setUp() { + meterRegistry = new SimpleMeterRegistry(); + queue = new DefaultCentralWorkQueue(Guice.createInjector(new SecurityModule()), persistence, meterRegistry); + } + + private final AtomicInteger runs = new AtomicInteger(); + private int counter = 0; + private int copy = -1; + + @Test + void shouldRunInSequenceWithBlock() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + } + + private void waitForTasks() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + assertThat(runs.get()).isEqualTo(ITERATIONS); + } + + @Test + void shouldRunInParallel() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().enqueue(new Increase()); + } + waitForTasks(); + + // we test if the resulting counter is less than the iteration, + // because it is extremely likely that we miss a counter update + // when we run in parallel + assertThat(counter) + .isPositive() + .isLessThan(ITERATIONS); + } + + @Test + void shouldNotBlocked() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isNotNegative().isLessThan(ITERATIONS); + } + + @Test + void shouldNotBlockedByDifferentResource() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("copy").enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isNotNegative().isLessThan(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentResource() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("counter", "one").enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isEqualTo(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentAndExactResource() { + for (int i = 0; i < ITERATIONS; i++) { + if (i % 2 == 0) { + queue.append().locks("counter", "c").enqueue(new Increase()); + } else { + queue.append().locks("counter").enqueue(new Increase()); + } + } + waitForTasks(); + assertThat(counter).isEqualTo(ITERATIONS); + } + + @Test + void shouldBeBlockedByParentResourceWithModelObject() { + Repository one = repository("one"); + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + queue.append().locks("counter", one).enqueue(() -> copy = counter); + waitForTasks(); + + assertThat(counter).isEqualTo(ITERATIONS); + assertThat(copy).isEqualTo(ITERATIONS); + } + + @Test + void shouldFinalizeOnError() { + queue.append().enqueue(() -> { + throw new IllegalStateException("failed"); + }); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + } + + @Test + void shouldSetThreadName() { + AtomicReference threadName = new AtomicReference<>(); + queue.append().enqueue(() -> threadName.set(Thread.currentThread().getName())); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> threadName.get() != null); + + assertThat(threadName.get()).startsWith("CentralWorkQueue"); + } + + @Test + void shouldCaptureExecutorMetrics() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().enqueue(new Increase()); + } + waitForTasks(); + + double count = meterRegistry.get("executor.completed").functionCounter().count(); + assertThat(count).isEqualTo(ITERATIONS); + } + + @Test + void shouldCaptureExecutionDuration() { + queue.append().enqueue(new Increase()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + + Timer timer = meterRegistry.get(UnitOfWork.METRIC_EXECUTION).timer(); + assertThat(timer.count()).isEqualTo(1); + } + + @Test + void shouldCaptureWaitDuration() { + queue.append().enqueue(new Increase()); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> queue.getSize() == 0); + + Timer timer = meterRegistry.get(UnitOfWork.METRIC_WAIT).timer(); + assertThat(timer.count()).isEqualTo(1); + } + + @Test + void shouldIncreaseBlockCount() { + for (int i = 0; i < ITERATIONS; i++) { + queue.append().locks("counter").enqueue(new Increase()); + } + waitForTasks(); + + int blockCount = 0; + for (Meter meter : meterRegistry.getMeters()) { + Meter.Id id = meter.getId(); + if ("cwq.task.wait.duration".equals(id.getName())) { + String blocked = id.getTag("blocked"); + if (blocked != null) { + blockCount += Integer.parseInt(blocked); + } + } + } + + assertThat(blockCount).isPositive(); + } + + @Nonnull + private Repository repository(String id) { + Repository one = new Repository(); + one.setId(id); + return one; + } + + @AfterEach + void tearDown() { + queue.close(); + } + + private class Increase implements Task { + + @Override + @SuppressWarnings("java:S2925") + public void run() { + int currentCounter = counter; + runs.incrementAndGet(); + try { + Thread.sleep(5); + counter = currentCounter + 1; + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + @Test + void shouldInjectDependencies() { + Context ctx = new Context(); + DefaultCentralWorkQueue queue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule(), binder -> binder.bind(Context.class).toInstance(ctx)), + persistence, + new SimpleMeterRegistry(), + () -> 2 + ); + + queue.append().enqueue(InjectingTask.class); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> ctx.value != null); + assertThat(ctx.value).isEqualTo("Hello"); + } + + @Test + void shouldLoadFromPersistence() { + Context context = new Context(); + SimpleUnitOfWork one = new SimpleUnitOfWork( + 21L, principal, Collections.singleton(new Resource("a")), new InjectingTask(context, "one") + ); + SimpleUnitOfWork two = new SimpleUnitOfWork( + 42L, principal, Collections.singleton(new Resource("a")), new InjectingTask(context, "two") + ); + two.restore(42L); + when(persistence.loadAll()).thenReturn(Arrays.asList(one, two)); + + new DefaultCentralWorkQueue(Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry()); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> context.value != null); + assertThat(context.value).isEqualTo("two"); + assertThat(one.getOrder()).isEqualTo(1L); + assertThat(one.getRestoreCount()).isEqualTo(1); + assertThat(two.getOrder()).isEqualTo(2L); + assertThat(two.getRestoreCount()).isEqualTo(2); + } + + @Test + void shouldRunAsUser() { + DefaultCentralWorkQueue workQueue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry() + ); + + AtomicReference ref = new AtomicReference<>(); + workQueue.append().enqueue(() -> ref.set(SecurityUtils.getSubject().getPrincipal())); + await().atMost(1, TimeUnit.SECONDS).until(() -> "trillian".equals(ref.get())); + } + + @Test + void shouldRunAsAdminUser() { + DefaultCentralWorkQueue workQueue = new DefaultCentralWorkQueue( + Guice.createInjector(new SecurityModule()), persistence, new SimpleMeterRegistry() + ); + + AtomicReference ref = new AtomicReference<>(); + workQueue.append().runAsAdmin().enqueue(() -> ref.set(SecurityUtils.getSubject().getPrincipal())); + await().atMost(1, TimeUnit.SECONDS).until(() -> Authentications.PRINCIPAL_SYSTEM.equals(ref.get())); + } + + public static class Context { + + private String value; + + public void setValue(String value) { + this.value = value; + } + } + + public static class InjectingTask implements Task { + + private final Context context; + private final String value; + + @Inject + public InjectingTask(Context context) { + this(context, "Hello"); + } + + public InjectingTask(Context context, String value) { + this.context = context; + this.value = value; + } + + @Override + public void run() { + context.setValue(value); + } + } + + public static class SecurityModule extends AbstractModule { + + @Override + protected void configure() { + bind(SecurityManager.class).toInstance(SecurityUtils.getSecurityManager()); + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java b/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java new file mode 100644 index 0000000000..a09ed8c771 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/PersistenceTest.java @@ -0,0 +1,178 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import lombok.EqualsAndHashCode; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import sonia.scm.plugin.PluginLoader; +import sonia.scm.store.Blob; +import sonia.scm.store.InMemoryBlobStore; +import sonia.scm.store.InMemoryBlobStoreFactory; + +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class PersistenceTest { + + private final PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + @Nested + class Default { + + @Mock + private PluginLoader pluginLoader; + + private Persistence persistence; + + @BeforeEach + void setUp() { + when(pluginLoader.getUberClassLoader()).thenReturn(PersistenceTest.class.getClassLoader()); + persistence = new Persistence(pluginLoader, new InMemoryBlobStoreFactory()); + } + + @Test + void shouldStoreSimpleChunkOfWork() { + UnitOfWork work = new SimpleUnitOfWork( + 1L, principal, Collections.singleton(new Resource("a")), new MyTask() + ); + persistence.store(work); + + UnitOfWork loaded = persistence.loadAll().iterator().next(); + assertThat(loaded).isEqualTo(work); + } + + @Test + void shouldStoreInjectingChunkOfWork() { + UnitOfWork work = new InjectingUnitOfWork( + 1L, principal, Collections.singleton(new Resource("a")), MyTask.class + ); + persistence.store(work); + + UnitOfWork loaded = persistence.loadAll().iterator().next(); + assertThat(loaded).isEqualTo(work); + } + + @Test + void shouldLoadInOrder() { + store(5, 3, 1, 4, 2); + + long[] orderIds = persistence.loadAll() + .stream() + .mapToLong(UnitOfWork::getOrder) + .toArray(); + + assertThat(orderIds).containsExactly(1, 2, 3, 4, 5); + } + + @Test + void shouldRemoveAfterLoad() { + store(1, 2); + + assertThat(persistence.loadAll()).hasSize(2); + assertThat(persistence.loadAll()).isEmpty(); + } + + @Test + void shouldFailIfNotSerializable() { + store(1); + + SimpleUnitOfWork unitOfWork = new SimpleUnitOfWork( + 2L, principal, Collections.emptySet(), new NotSerializable() + ); + + assertThrows(NonPersistableTaskException.class, () -> persistence.store(unitOfWork)); + } + + @Test + void shouldRemoveStored() { + store(1); + SimpleUnitOfWork chunkOfWork = new SimpleUnitOfWork( + 2L, principal, Collections.emptySet(), new MyTask() + ); + persistence.store(chunkOfWork); + persistence.remove(chunkOfWork); + + assertThat(persistence.loadAll()).hasSize(1); + } + + private void store(long... orderIds) { + for (long order : orderIds) { + persistence.store(new SimpleUnitOfWork( + order, principal, Collections.emptySet(), new MyTask() + )); + } + } + + } + + @Test + void shouldNotFailForNonChunkOfWorkItems() throws IOException { + InMemoryBlobStore blobStore = new InMemoryBlobStore(); + + Persistence persistence = new Persistence(PersistenceTest.class.getClassLoader(), blobStore); + persistence.store(new SimpleUnitOfWork( + 1L, principal, Collections.emptySet(), new MyTask()) + ); + + Blob blob = blobStore.create(); + try (ObjectOutputStream stream = new ObjectOutputStream(blob.getOutputStream())) { + stream.writeObject(new MyTask()); + blob.commit(); + } + + assertThat(persistence.loadAll()).hasSize(1); + } + + @EqualsAndHashCode + public static class MyTask implements Task { + + @Override + public void run() { + + } + } + + // non static inner classes are not serializable + private class NotSerializable implements Task { + @Override + public void run() { + + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java b/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java new file mode 100644 index 0000000000..6f4cee20c6 --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/ResourceTest.java @@ -0,0 +1,72 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class ResourceTest { + + @Test + void shouldReturnResourceName() { + assertThat(res("a")).hasToString("a"); + } + + @Test + void shouldReturnResourceNameAndId() { + assertThat(res("a", "b")).hasToString("a:b"); + } + + @Nested + class IsBlockedByTests { + + @Test + void shouldReturnTrue() { + assertThat(res("a").isBlockedBy(res("a"))).isTrue(); + assertThat(res("a", "b").isBlockedBy(res("a", "b"))).isTrue(); + assertThat(res("a").isBlockedBy(res("a", "b"))).isTrue(); + assertThat(res("a", "b").isBlockedBy(res("a"))).isTrue(); + } + + @Test + void shouldReturnFalse() { + assertThat(res("a").isBlockedBy(res("b"))).isFalse(); + assertThat(res("a", "b").isBlockedBy(res("a", "c"))).isFalse(); + assertThat(res("a", "b").isBlockedBy(res("c", "b"))).isFalse(); + } + + } + + private Resource res(String name) { + return new Resource(name); + } + + private Resource res(String name, String id) { + return new Resource(name, id); + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java b/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java new file mode 100644 index 0000000000..5a936768eb --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/SimpleUnitOfWorkTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import lombok.Value; +import org.apache.shiro.subject.PrincipalCollection; +import org.apache.shiro.subject.SimplePrincipalCollection; +import org.junit.jupiter.api.Test; + +import javax.inject.Inject; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +class SimpleUnitOfWorkTest { + + private PrincipalCollection principal = new SimplePrincipalCollection("trillian", "test"); + + @Test + void shouldInjectMember() { + Context context = new Context("awesome"); + Injector injector = Guice.createInjector(new AbstractModule() { + @Override + protected void configure() { + bind(Context.class).toInstance(context); + } + }); + + SimpleTask simpleTask = new SimpleTask(); + SimpleUnitOfWork unitOfWork = new SimpleUnitOfWork(1L, principal, Collections.emptySet(), simpleTask); + unitOfWork.task(injector); + + simpleTask.run(); + + assertThat(simpleTask.value).isEqualTo("awesome"); + } + + @Value + public class Context { + String value; + } + + public class SimpleTask implements Task { + + private Context context; + + private String value = "no value set"; + + @Inject + public void setContext(Context context) { + this.context = context; + } + + @Override + public void run() { + if (context != null) { + value = context.getValue(); + } + } + } + +} diff --git a/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java b/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java new file mode 100644 index 0000000000..6d9d6b0d7f --- /dev/null +++ b/scm-webapp/src/test/java/sonia/scm/work/ThreadCountProviderTest.java @@ -0,0 +1,80 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.work; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(MockitoExtension.class) +class ThreadCountProviderTest { + + @Test + void shouldUseTwoWorkersForOneCPU() { + ThreadCountProvider provider = new ThreadCountProvider(() -> 1); + + assertThat(provider.getAsInt()).isEqualTo(2); + } + + @ParameterizedTest(name = "shouldUseFourWorkersFor{argumentsWithNames}CPU") + @ValueSource(ints = {2, 4, 8, 16}) + void shouldUseFourWorkersForMoreThanOneCPU(int cpus) { + ThreadCountProvider provider = new ThreadCountProvider(() -> cpus); + + assertThat(provider.getAsInt()).isEqualTo(4); + } + + @Nested + class SystemPropertyTests { + + @BeforeEach + void setUp() { + System.clearProperty(ThreadCountProvider.PROPERTY); + } + + @Test + void shouldUseCountFromSystemProperty() { + ThreadCountProvider provider = new ThreadCountProvider(); + System.setProperty(ThreadCountProvider.PROPERTY, "6"); + assertThat(provider.getAsInt()).isEqualTo(6); + } + + @ParameterizedTest + @ValueSource(strings = {"-1", "0", "100", "a", ""}) + void shouldUseDefaultForInvalidValue(String value) { + ThreadCountProvider provider = new ThreadCountProvider(() -> 1); + System.setProperty(ThreadCountProvider.PROPERTY, value); + assertThat(provider.getAsInt()).isEqualTo(2); + } + + } + +}