From 56ace2811bed0b0bfe51d6e478f0b610b13c9e40 Mon Sep 17 00:00:00 2001 From: Konstantin Schaper Date: Wed, 17 Aug 2022 13:22:34 +0200 Subject: [PATCH] Implement reindex mechanism for search (#2104) Adds a new button to repository settings to allow users to manually delete and re-create search indices. The actual re-indexing is happening in plugins that subscribe to the newly created event. Co-authored-by: Eduard Heimbuch --- gradle/changelog/reindex.yaml | 2 + .../scm/search/ReindexRepositoryEvent.java | 40 ++++++++++++ scm-ui/ui-api/src/repositories.ts | 21 ++++++ scm-ui/ui-webapp/public/locales/de/repos.json | 6 ++ scm-ui/ui-webapp/public/locales/en/repos.json | 6 ++ .../src/repos/components/Reindex.tsx | 64 +++++++++++++++++++ .../src/repos/containers/EditRepo.tsx | 2 + .../api/v2/resources/RepositoryResource.java | 23 +++++++ .../RepositoryToRepositoryDtoMapper.java | 3 + .../scm/api/v2/resources/ResourceLinks.java | 4 ++ .../resources/RepositoryRootResourceTest.java | 60 +++++++++++++++++ .../RepositoryToRepositoryDtoMapperTest.java | 17 +++++ 12 files changed, 248 insertions(+) create mode 100644 gradle/changelog/reindex.yaml create mode 100644 scm-core/src/main/java/sonia/scm/search/ReindexRepositoryEvent.java create mode 100644 scm-ui/ui-webapp/src/repos/components/Reindex.tsx diff --git a/gradle/changelog/reindex.yaml b/gradle/changelog/reindex.yaml new file mode 100644 index 0000000000..4405e8aeeb --- /dev/null +++ b/gradle/changelog/reindex.yaml @@ -0,0 +1,2 @@ +- type: added + description: Reindex mechanism for search ([#2104](https://github.com/scm-manager/scm-manager/pull/2104)) diff --git a/scm-core/src/main/java/sonia/scm/search/ReindexRepositoryEvent.java b/scm-core/src/main/java/sonia/scm/search/ReindexRepositoryEvent.java new file mode 100644 index 0000000000..c9440b0233 --- /dev/null +++ b/scm-core/src/main/java/sonia/scm/search/ReindexRepositoryEvent.java @@ -0,0 +1,40 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package sonia.scm.search; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import sonia.scm.event.Event; +import sonia.scm.repository.Repository; + +/** + * @since 2.39.0 + */ +@Event +@AllArgsConstructor +@Getter +public class ReindexRepositoryEvent { + private Repository repository; +} diff --git a/scm-ui/ui-api/src/repositories.ts b/scm-ui/ui-api/src/repositories.ts index 1e53f23c33..29418c572d 100644 --- a/scm-ui/ui-api/src/repositories.ts +++ b/scm-ui/ui-api/src/repositories.ts @@ -373,3 +373,24 @@ export const useRenameRepository = (repository: Repository) => { isRenamed: !!data }; }; + +export const useReindexRepository = () => { + const queryClient = useQueryClient(); + const { mutate, isLoading, error, data } = useMutation( + (repository) => { + const link = requiredLink(repository, "reindex"); + return apiClient.post(link); + }, + { + onSuccess: async (_, repository) => { + await queryClient.invalidateQueries(repoQueryKey(repository)); + }, + } + ); + return { + reindex: (repository: Repository) => mutate(repository), + isLoading, + error, + isRunning: !!data, + }; +}; diff --git a/scm-ui/ui-webapp/public/locales/de/repos.json b/scm-ui/ui-webapp/public/locales/de/repos.json index a70c915dd8..a71ece8ff3 100644 --- a/scm-ui/ui-webapp/public/locales/de/repos.json +++ b/scm-ui/ui-webapp/public/locales/de/repos.json @@ -492,6 +492,12 @@ "descriptionNotRunning": "Starten der Integritätsprüfung dieses Repositories. Dieser Vorgang kann einige Zeit in Anspruch nehmen.", "descriptionRunning": "Die Integritätsprüfung für dieses Repository läuft bereits und kann nicht parallel erneut gestartet werden." }, + "reindex": { + "button": "Reindizieren", + "subtitle": "Suchindizes neu erstellen", + "description": "Löscht alle existierenden Suchindizes für dieses Repository and erstellt sie komplett neu. Dieser Vorgang kann einige Zeit in Anspruch nehmen.", + "started": "Die Reindizierung wurde erfolgreich gestartet. Dies ist eine asynchrone Operation und kann einige Zeit in Anspruch nehmen." + }, "diff": { "jumpToSource": "Zur Quelldatei springen", "jumpToTarget": "Zur vorherigen Version der Datei springen", diff --git a/scm-ui/ui-webapp/public/locales/en/repos.json b/scm-ui/ui-webapp/public/locales/en/repos.json index cc2a428945..778854e0cf 100644 --- a/scm-ui/ui-webapp/public/locales/en/repos.json +++ b/scm-ui/ui-webapp/public/locales/en/repos.json @@ -481,6 +481,12 @@ "descriptionNotRunning": "Run the health checks for this repository. This may take a while.", "descriptionRunning": "Health checks for this repository are currently running and cannot be started again in parallel." }, + "reindex": { + "button": "Reindex", + "subtitle": "Recreate Search Indices", + "description": "Deletes all existing search indices for this repository and recreates them from scratch. This may take a while.", + "started": "Reindexing has been started successfully. This is an asynchronous operation and may take a while." + }, "archive": { "tooltip": "Read only. The archive cannot be changed." }, diff --git a/scm-ui/ui-webapp/src/repos/components/Reindex.tsx b/scm-ui/ui-webapp/src/repos/components/Reindex.tsx new file mode 100644 index 0000000000..f91fe52ba1 --- /dev/null +++ b/scm-ui/ui-webapp/src/repos/components/Reindex.tsx @@ -0,0 +1,64 @@ +/* + * MIT License + * + * Copyright (c) 2020-present Cloudogu GmbH and Contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Repository } from "@scm-manager/ui-types"; +import React, { FC } from "react"; +import { useReindexRepository } from "@scm-manager/ui-api"; +import { Button, ErrorNotification, Level, Notification, Subtitle } from "@scm-manager/ui-components"; +import { useTranslation } from "react-i18next"; + +type Props = { + repository: Repository; +}; + +const Reindex: FC = ({ repository }) => { + const [t] = useTranslation("repos"); + const { reindex, error, isLoading, isRunning } = useReindexRepository(); + + return ( + <> +
+ + {isRunning ? {t("reindex.started")} : null} + {t("reindex.subtitle")} +

{t("reindex.description")}

+ reindex(repository)} + disabled={isLoading} + loading={isLoading} + > + {t("reindex.button")} + + } + /> + + ); +}; + +export default Reindex; diff --git a/scm-ui/ui-webapp/src/repos/containers/EditRepo.tsx b/scm-ui/ui-webapp/src/repos/containers/EditRepo.tsx index 68b4ffbb46..64c9cefab5 100644 --- a/scm-ui/ui-webapp/src/repos/containers/EditRepo.tsx +++ b/scm-ui/ui-webapp/src/repos/containers/EditRepo.tsx @@ -34,6 +34,7 @@ import { useUpdateRepository } from "@scm-manager/ui-api"; import HealthCheckWarning from "./HealthCheckWarning"; import RunHealthCheck from "./RunHealthCheck"; import UpdateNotification from "../../components/UpdateNotification"; +import Reindex from "../components/Reindex"; type Props = { repository: Repository; @@ -71,6 +72,7 @@ const EditRepo: FC = ({ repository }) => { {(repository._links.runHealthCheck || repository.healthCheckRunning) && ( )} + {repository._links.reindex ? : null} ); diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryResource.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryResource.java index 3a85370c31..1196a64d99 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryResource.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryResource.java @@ -30,10 +30,13 @@ import io.swagger.v3.oas.annotations.media.ExampleObject; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; +import sonia.scm.event.ScmEventBus; import sonia.scm.repository.HealthCheckService; import sonia.scm.repository.NamespaceAndName; import sonia.scm.repository.Repository; import sonia.scm.repository.RepositoryManager; +import sonia.scm.repository.RepositoryPermissions; +import sonia.scm.search.ReindexRepositoryEvent; import sonia.scm.web.VndMediaType; import javax.inject.Inject; @@ -294,6 +297,26 @@ public class RepositoryResource { healthCheckService.fullCheck(repository); } + @POST + @Path("reindex") + @Operation(summary = "Manually reindex repository", description = "Asynchronously update search indices for repository", tags = "Repository") + @ApiResponse(responseCode = "204", description = "event submitted") + @ApiResponse(responseCode = "401", description = "not authenticated / invalid credentials") + @ApiResponse(responseCode = "403", description = "not authorized, the current user does not have owner permissions for this repository") + @ApiResponse( + responseCode = "404", + description = "not found, no repository with the specified namespace and name available", + content = @Content( + mediaType = VndMediaType.ERROR_TYPE, + schema = @Schema(implementation = ErrorDto.class) + )) + @ApiResponse(responseCode = "500", description = "internal server error") + public void reindex(@PathParam("namespace") String namespace, @PathParam("name") String name) { + Repository repository = loadBy(namespace, name).get(); + RepositoryPermissions.custom("*", repository).check(); + ScmEventBus.getInstance().post(new ReindexRepositoryEvent(repository)); + } + private Repository processUpdate(RepositoryDto repositoryDto, Repository existing) { Repository changedRepository = dtoToRepositoryMapper.map(repositoryDto, existing.getId()); changedRepository.setPermissions(existing.getPermissions()); diff --git a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java index 046bbaec31..4a2f925a7d 100644 --- a/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java +++ b/scm-webapp/src/main/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapper.java @@ -111,6 +111,9 @@ public abstract class RepositoryToRepositoryDtoMapper extends BaseMapper cmds) { RepositoryHandler repositoryHandler = mock(RepositoryHandler.class); RepositoryType repositoryType = mock(RepositoryType.class); diff --git a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java index dbd7fb3a70..83e065f944 100644 --- a/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java +++ b/scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java @@ -408,6 +408,23 @@ public class RepositoryToRepositoryDtoMapperTest { .isEqualTo("http://example.com/base/v2/search/searchableTypes/testspace/test"); } + @Test + public void shouldCreateReindexLink() { + Repository testRepository = createTestRepository(); + RepositoryDto dto = mapper.map(testRepository); + assertThat(dto.getLinks().getLinkBy("reindex")) + .get() + .hasFieldOrPropertyWithValue("href", "http://example.com/base/v2/repositories/testspace/test/reindex"); + } + + @SubjectAware(username = "unpriv") + @Test + public void shouldNotCreateReindexLinkWithoutPermission() { + Repository testRepository = createTestRepository(); + RepositoryDto dto = mapper.map(testRepository); + assertThat(dto.getLinks().getLinkBy("reindex")).isEmpty(); + } + private ScmProtocol mockProtocol(String type, String protocol) { return new MockScmProtocol(type, protocol); }