diff --git a/gradle/changelog/distinct_and_project.yaml b/gradle/changelog/distinct_and_project.yaml new file mode 100644 index 0000000000..84c3894242 --- /dev/null +++ b/gradle/changelog/distinct_and_project.yaml @@ -0,0 +1,2 @@ +- type: added + description: Possibility to use distinct and projection in queryable stores diff --git a/scm-core/src/main/java/sonia/scm/store/QueryableStore.java b/scm-core/src/main/java/sonia/scm/store/QueryableStore.java index 8367d81488..5f60ef6409 100644 --- a/scm-core/src/main/java/sonia/scm/store/QueryableStore.java +++ b/scm-core/src/main/java/sonia/scm/store/QueryableStore.java @@ -22,8 +22,10 @@ import lombok.NoArgsConstructor; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Optional; @@ -123,7 +125,11 @@ public interface QueryableStore extends AutoCloseable { * @param offset The offset to start the result list. * @param limit The maximum number of results to return. */ - List findAll(long offset, long limit); + default List findAll(long offset, long limit) { + List result = new ArrayList<>(); + forEach(result::add, offset, limit); + return Collections.unmodifiableList(result); + } /** * Calls the given consumer for all objects that match the query. @@ -143,6 +149,28 @@ public interface QueryableStore extends AutoCloseable { */ void forEach(Consumer consumer, long offset, long limit); + /** + * Projects the found objects to a specific values. This is useful if you want to retrieve only a subset of the + * fields of the found objects. + *
+ * The projection will return an array of objects, where each object corresponds to a field that was specified in + * the {@code fields} parameter. The order of the fields in the array will be the same as the order of the fields in + * the parameter. + * + * @param fields The fields to project. + * @return The query object to continue building the query. + */ + Query project(QueryField... fields); + + /** + * Returns the found objects as a distinct set. This is useful if you want to ensure that no duplicate values are + * returned, for example to determine the unique values of all parent ids. Most likely this is usefull only with + * #project(QueryField[]) to limit the selected "columns". + * + * @return The query object to continue building the query. + */ + Query distinct(); + /** * Returns the found objects in combination with the parent ids they belong to. This is useful if you are using a * queryable store that is not scoped to specific parent objects, and you therefore want to know to which parent diff --git a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLCountField.java b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLCountField.java new file mode 100644 index 0000000000..02f3c63b3f --- /dev/null +++ b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLCountField.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 - present Cloudogu GmbH + * + * This program is free software: you can redistribute it and/or modify it under + * the terms of the GNU Affero General Public License as published by the Free + * Software Foundation, version 3. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +package sonia.scm.store.sqlite; + +import lombok.Getter; + +import java.util.Collection; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; + +/** + * Representation of a SQL COUNT field. + * + * @since 3.9.0 + */ +@Getter +class SQLCountField implements SQLNode { + + private final Collection fields; + private final boolean distinct; + + public SQLCountField() { + this(emptyList(), false); + } + + SQLCountField(Collection fields, boolean distinct) { + this.fields = fields; + this.distinct = distinct; + } + + @Override + public String toSQL() { + StringBuilder sqlBuilder = new StringBuilder("COUNT("); + if (distinct) { + sqlBuilder.append("DISTINCT "); + } + if (fields.isEmpty()) { + sqlBuilder.append("*"); + } else { + sqlBuilder.append(fields.stream().map(SQLNode::toSQL).collect(Collectors.joining(", "))); + } + return sqlBuilder.append(")").toString(); + } +} diff --git a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLSelectStatement.java b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLSelectStatement.java index a0c95733aa..c26feb16e4 100644 --- a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLSelectStatement.java +++ b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLSelectStatement.java @@ -21,17 +21,35 @@ import java.util.stream.Collectors; class SQLSelectStatement extends ConditionalSQLStatement { - private final List columns; + private final List columns; private final SQLTable fromTable; private final String orderBy; private final long limit; private final long offset; + private final boolean distinct; - SQLSelectStatement(List columns, SQLTable fromTable, List whereCondition) { + SQLSelectStatement(List columns, + SQLTable fromTable, + List whereCondition) { this(columns, fromTable, whereCondition, null, 0, 0); } - SQLSelectStatement(List columns, SQLTable fromTable, List whereCondition, String orderBy, long limit, long offset) { + SQLSelectStatement(List columns, + SQLTable fromTable, + List whereCondition, + String orderBy, + long limit, + long offset) { + this(columns, fromTable, whereCondition, orderBy, limit, offset, false); + } + + SQLSelectStatement(List columns, + SQLTable fromTable, + List whereCondition, + String orderBy, + long limit, + long offset, + boolean distinct) { super(whereCondition); if (limit < 0 || offset < 0) { throw new IllegalArgumentException("limit and offset must be non-negative"); @@ -41,6 +59,7 @@ class SQLSelectStatement extends ConditionalSQLStatement { this.orderBy = orderBy; this.limit = limit; this.offset = offset; + this.distinct = distinct; } @Override @@ -48,9 +67,12 @@ class SQLSelectStatement extends ConditionalSQLStatement { StringBuilder query = new StringBuilder(); query.append("SELECT "); + if (distinct) { + query.append("DISTINCT "); + } if (columns != null && !columns.isEmpty()) { String columnList = columns.stream() - .map(SQLField::toSQL) + .map(SQLNode::toSQL) .collect(Collectors.joining(", ")); query.append(columnList); } diff --git a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableMutableStore.java b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableMutableStore.java index daeb57be12..dcc291a3bd 100644 --- a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableMutableStore.java +++ b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableMutableStore.java @@ -105,7 +105,7 @@ class SQLiteQueryableMutableStore extends SQLiteQueryableStore implements @Override public Map getAll() { - List columns = List.of( + List columns = List.of( SQLField.PAYLOAD, new SQLField("ID") ); @@ -221,7 +221,7 @@ class SQLiteQueryableMutableStore extends SQLiteQueryableStore implements @Override public void retain(long n) { - List columns = new ArrayList<>(); + List columns = new ArrayList<>(); addParentIdSQLFields(columns); List conditions = new ArrayList<>(); diff --git a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableStore.java b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableStore.java index a2fc8f890f..85a7fee142 100644 --- a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableStore.java +++ b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteQueryableStore.java @@ -133,7 +133,7 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance private Collection readAllAs(RowBuilder rowBuilder) { List parentConditions = new ArrayList<>(); evaluateParentConditions(parentConditions); - List fields = new ArrayList<>(); + List fields = new ArrayList<>(); addParentIdSQLFields(fields); int parentIdsLength = fields.size() - 1; // addParentIdSQLFields has already added the ID field fields.add(new SQLField("PAYLOAD")); @@ -201,7 +201,7 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance @Override public MaintenanceIterator iterateAll() { - List columns = new LinkedList<>(); + List columns = new LinkedList<>(); columns.add(new SQLField("payload")); addParentIdSQLFields(columns); @@ -292,7 +292,7 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance } } - void addParentIdSQLFields(List fields) { + void addParentIdSQLFields(List fields) { for (int i = 0; i < queryableTypeDescriptor.getTypes().length; i++) { fields.add(new SQLField(computeColumnIdentifier(queryableTypeDescriptor.getTypes()[i]))); } @@ -343,18 +343,21 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance private final Class resultType; private final Class entityType; private final Condition condition; + private final QueryField[] projection; private List> orderBy; + private boolean distinct = false; SQLiteQuery(Class resultType, Condition[] conditions) { - this(resultType, resultType, conjunct(conditions), Collections.emptyList()); + this(resultType, resultType, conjunct(conditions), Collections.emptyList(), null); } @SuppressWarnings({"rawtypes", "unchecked"}) - private SQLiteQuery(Class resultType, Class entityType, Condition condition, List> orderBy) { + private SQLiteQuery(Class resultType, Class entityType, Condition condition, List> orderBy, QueryField[] projection) { this.resultType = resultType; this.entityType = entityType; this.condition = condition; this.orderBy = orderBy; + this.projection = projection; } private static Condition conjunct(Condition[] conditions) { @@ -384,13 +387,6 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance } } - @Override - public List findAll(long offset, long limit) { - List result = new ArrayList<>(); - forEach(result::add, offset, limit); - return Collections.unmodifiableList(result); - } - @Override public void forEach(Consumer consumer, long offset, long limit) { String orderByString = getOrderByString(); @@ -402,7 +398,8 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance computeCondition(), orderByString, limit, - offset + offset, + distinct ); executeRead( @@ -418,6 +415,17 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance ); } + @Override + public Query distinct() { + this.distinct = true; + return this; + } + + @Override + public Query project(QueryField... fields) { + return new SQLiteQuery<>(Object[].class, resultType, condition, orderBy, fields); + } + String getOrderByString() { StringBuilder orderByBuilder = new StringBuilder(); if (orderBy != null && !orderBy.isEmpty()) { @@ -429,14 +437,14 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance @Override @SuppressWarnings("unchecked") public Query, ?> withIds() { - return new SQLiteQuery<>((Class>) (Class) Result.class, resultType, condition, orderBy); + return new SQLiteQuery<>((Class>) (Class) Result.class, resultType, condition, orderBy, null); } @Override public long count() { SQLSelectStatement sqlStatementQuery = new SQLSelectStatement( - List.of(new SQLField("COUNT(*)")), + List.of(new SQLCountField(computeFields(), distinct)), computeFromTable(), computeCondition() ); @@ -505,8 +513,15 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance return newOrderBy; } - private List computeFields() { - List fields = new ArrayList<>(); + private List computeFields() { + if (projection != null && projection.length > 0) { + return computeProjectedFields(); + } + return computeDefaultFields(); + } + + private List computeDefaultFields() { + List fields = new ArrayList<>(); fields.add(SQLField.PAYLOAD); if (resultType.isAssignableFrom(Result.class)) { addParentIdSQLFields(fields); @@ -514,6 +529,14 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance return fields; } + private List computeProjectedFields() { + return Arrays.stream(projection) + .map(SQLFieldHelper::computeSQLField) + .map(SQLField::new) + .map(field -> (SQLNode) field) + .toList(); + } + List computeCondition() { List conditions = new ArrayList<>(); @@ -556,6 +579,22 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance @SuppressWarnings("unchecked") private T_RESULT extractResult(ResultSet resultSet) throws JsonProcessingException, SQLException { + if (projection != null && projection.length > 0) { + Object[] values = new Object[projection.length]; + for (int i = 0; i < projection.length; i++) { + QueryField field = projection[i]; + if (field instanceof QueryableStore.CollectionSizeQueryField) { + values[i] = resultSet.getInt(i + 1); + } else if (field instanceof QueryableStore.MapSizeQueryField) { + values[i] = resultSet.getInt(i + 1); + } else if (field.isIdField()) { + values[i] = resultSet.getString(i + 1); + } else { + values[i] = resultSet.getObject(i + 1); + } + } + return (T_RESULT) values; + } T entity = objectMapper.readValue(resultSet.getString(1), entityType); if (resultType.isAssignableFrom(Result.class)) { Map parentIdMapping = new HashMap<>(queryableTypeDescriptor.getTypes().length); @@ -602,11 +641,11 @@ class SQLiteQueryableStore implements QueryableStore, QueryableMaintenance private class TemporaryTableMaintenanceIterator implements MaintenanceIterator { private final PreparedStatement iterateStatement; - private final List columns; + private final List columns; private final ResultSet resultSet; private Boolean hasNext; - public TemporaryTableMaintenanceIterator(List columns) { + public TemporaryTableMaintenanceIterator(List columns) { this.columns = columns; this.hasNext = null; SQLSelectStatement iterateQuery = diff --git a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteRetainStatement.java b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteRetainStatement.java index e64fdc6d17..63d08902fa 100644 --- a/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteRetainStatement.java +++ b/scm-persistence/src/main/java/sonia/scm/store/sqlite/SQLiteRetainStatement.java @@ -26,12 +26,12 @@ import static java.lang.String.format; class SQLiteRetainStatement implements SQLNodeWithValue { private final SQLTable table; - private final List columns; + private final List columns; private final SQLSelectStatement selectStatement; private final List parentConditions; - SQLiteRetainStatement(SQLTable table, List columns, SQLSelectStatement selectStatement, List parentConditions) { + SQLiteRetainStatement(SQLTable table, List columns, SQLSelectStatement selectStatement, List parentConditions) { this.table = table; this.columns = columns; this.selectStatement = selectStatement; @@ -57,7 +57,7 @@ class SQLiteRetainStatement implements SQLNodeWithValue { } return format("DELETE FROM %s WHERE (%s) NOT IN (%s) %s", table.toSQL(), - columns.stream().map(SQLField::toSQL).collect(Collectors.joining(",")), + columns.stream().map(SQLNode::toSQL).collect(Collectors.joining(",")), selectStatement.toSQL(), parentConditionStatement); } diff --git a/scm-persistence/src/test/java/sonia/scm/store/sqlite/SQLiteQueryableStoreTest.java b/scm-persistence/src/test/java/sonia/scm/store/sqlite/SQLiteQueryableStoreTest.java index 90d336566e..8ede882009 100644 --- a/scm-persistence/src/test/java/sonia/scm/store/sqlite/SQLiteQueryableStoreTest.java +++ b/scm-persistence/src/test/java/sonia/scm/store/sqlite/SQLiteQueryableStoreTest.java @@ -546,7 +546,7 @@ class SQLiteQueryableStoreTest { assertThat(all) .extracting("name") - .containsExactly("arthur","trillian","trish"); + .containsExactly("arthur", "trillian", "trish"); } @Test @@ -564,7 +564,7 @@ class SQLiteQueryableStoreTest { System.out.println(all); assertThat(all) .extracting("displayName") - .containsExactly("Tricia","Trillian McMillan","Arthur Dent"); + .containsExactly("Tricia", "Trillian McMillan", "Arthur Dent"); } @Test @@ -909,6 +909,69 @@ class SQLiteQueryableStoreTest { } } + @Nested + class Distinct { + + @BeforeEach + void fillData() { + new StoreTestBuilder(connectionString, "sonia.Group") + .withIds("42") + .put("1", new User("trillian", "Trillian McMillan", "")); + new StoreTestBuilder(connectionString, "sonia.Group") + .withIds("42") + .put("2", new User("zaphod", "Zaphod Beeblebrox", "")); + new StoreTestBuilder(connectionString, "sonia.Group") + .withIds("42") + .put("3", new User("marvin", "Marvin", "")); + new StoreTestBuilder(connectionString, "sonia.Group") + .withIds("23") + .put("1", new User("dent", "Arthur Dent", "")); + new StoreTestBuilder(connectionString, "sonia.Group") + .withIds("23") + .put("2", new User("trillian", "Trillian McMillan", "")); + } + + @Test + void shouldReturnDistinctValuesFromObject() { + SQLiteQueryableMutableStore store = new StoreTestBuilder(connectionString, "sonia.Group") + .withIds(); + + List result = store.query().project(USER_NAME).distinct().findAll(); + + assertThat(result) + .containsExactlyInAnyOrder( + new String[]{"trillian"}, + new String[]{"zaphod"}, + new String[]{"marvin"}, + new String[]{"dent"} + ); + } + + @Test + void shouldReturnDistinctParentIds() { + SQLiteQueryableMutableStore store = new StoreTestBuilder(connectionString, "sonia.Group") + .withIds(); + + List result = store.query().project(GROUP).distinct().findAll(); + + assertThat(result) + .containsExactlyInAnyOrder( + new String[]{"42"}, + new String[]{"23"} + ); + } + + @Test + void shouldReturnDistinctCount() { + SQLiteQueryableMutableStore store = new StoreTestBuilder(connectionString, "sonia.Group") + .withIds(); + + long count = store.query().project(GROUP).distinct().count(); + + assertThat(count).isEqualTo(2); + } + } + @Nested class ForMaintenance { @Test