Enable Health Checks (#1621)

In the release of version 2.0.0 of SCM-Manager, the health checks had been neglected. This makes them visible again in the frontend and adds the ability to trigger them. In addition there are two types of health checks: The "normal" ones, now called "light checks", that are run on startup, and more intense checks run only on request.

As a change to version 1.x, health checks will no longer be persisted for repositories.

Co-authored-by: Eduard Heimbuch <eduard.heimbuch@cloudogu.com>

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryRootResourceTest.java

@@ -55,6 +55,7 @@ import sonia.scm.importexport.FullScmRepositoryImporter;
 import sonia.scm.importexport.RepositoryImportExportEncryption;
 import sonia.scm.importexport.RepositoryImportLoggerFactory;
 import sonia.scm.repository.CustomNamespaceStrategy;
+import sonia.scm.repository.HealthCheckService;
 import sonia.scm.repository.NamespaceAndName;
 import sonia.scm.repository.NamespaceStrategy;
 import sonia.scm.repository.Repository;
@@ -157,6 +158,8 @@ public class RepositoryRootResourceTest extends RepositoryTestBase {
   private RepositoryImportLoggerFactory importLoggerFactory;
   @Mock
   private ExportService exportService;
+  @Mock
+  private HealthCheckService healthCheckService;
 
   @Captor
   private ArgumentCaptor<Predicate<Repository>> filterCaptor;

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryTestBase.java

@@ -24,6 +24,7 @@
 
 package sonia.scm.api.v2.resources;
 
+import sonia.scm.repository.HealthCheckService;
 import sonia.scm.repository.RepositoryManager;
 
 import static com.google.inject.util.Providers.of;
@@ -48,6 +49,7 @@ abstract class RepositoryTestBase {
   RepositoryImportResource repositoryImportResource;
   RepositoryExportResource repositoryExportResource;
   RepositoryPathsResource repositoryPathsResource;
+  HealthCheckService healthCheckService;
 
   RepositoryRootResource getRepositoryRootResource() {
     RepositoryBasedResourceProvider repositoryBasedResourceProvider = new RepositoryBasedResourceProvider(
@@ -70,7 +72,9 @@ abstract class RepositoryTestBase {
         repositoryToDtoMapper,
         dtoToRepositoryMapper,
         manager,
-        repositoryBasedResourceProvider)),
-            of(repositoryCollectionResource), of(repositoryImportResource));
+        repositoryBasedResourceProvider,
+        healthCheckService)),
+      of(repositoryCollectionResource),
+      of(repositoryImportResource));
   }
 }

scm-webapp/src/test/java/sonia/scm/api/v2/resources/RepositoryToRepositoryDtoMapperTest.java

@@ -34,9 +34,11 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
+import sonia.scm.SCMContextProvider;
 import sonia.scm.config.ScmConfiguration;
 import sonia.scm.repository.CustomNamespaceStrategy;
 import sonia.scm.repository.HealthCheckFailure;
+import sonia.scm.repository.HealthCheckService;
 import sonia.scm.repository.NamespaceStrategy;
 import sonia.scm.repository.Repository;
 import sonia.scm.repository.api.Command;
@@ -57,6 +59,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.when;
 import static org.mockito.MockitoAnnotations.initMocks;
+import static sonia.scm.repository.HealthCheckFailure.templated;
 
 @SubjectAware(
   username = "trillian",
@@ -83,6 +86,10 @@ public class RepositoryToRepositoryDtoMapperTest {
   private ScmConfiguration configuration;
   @Mock
   private Set<NamespaceStrategy> strategies;
+  @Mock
+  private HealthCheckService healthCheckService;
+  @Mock
+  private SCMContextProvider scmContextProvider;
 
   @InjectMocks
   private RepositoryToRepositoryDtoMapperImpl mapper;
@@ -311,6 +318,62 @@ public class RepositoryToRepositoryDtoMapperTest {
                   });
   }
 
+  @Test
+  public void shouldCreateRunHealthCheckLink() {
+    RepositoryDto dto = mapper.map(createTestRepository());
+    assertEquals(
+      "http://example.com/base/v2/repositories/testspace/test/runHealthCheck",
+      dto.getLinks().getLinkBy("runHealthCheck").get().getHref());
+    assertFalse(dto.isHealthCheckRunning());
+  }
+
+  @Test
+  public void shouldNotCreateHealthCheckLinkIfCheckIsRunning() {
+    Repository testRepository = createTestRepository();
+    when(healthCheckService.checkRunning(testRepository)).thenReturn(true);
+    RepositoryDto dto = mapper.map(testRepository);
+    assertFalse(dto.getLinks().getLinkBy("runHealthCheck").isPresent());
+    assertTrue(dto.isHealthCheckRunning());
+  }
+
+  @Test
+  public void shouldCreateCorrectLinksForHealthChecks() {
+    when(scmContextProvider.getDocumentationVersion()).thenReturn("2.17.x");
+
+    Repository testRepository = createTestRepository();
+    HealthCheckFailure failure = new HealthCheckFailure("1", "vogons", templated("http://hog/{0}/vogons"), "met vogons");
+    testRepository.setHealthCheckFailures(singletonList(failure));
+
+    RepositoryDto dto = mapper.map(testRepository);
+
+    assertThat(dto.getHealthCheckFailures())
+    .extracting("url")
+    .containsExactly("http://hog/2.17.x/vogons");
+
+    assertThat(dto.getHealthCheckFailures().get(0).getLinks().getLinkBy("documentation"))
+      .get()
+      .extracting("href")
+      .isEqualTo("http://hog/2.17.x/vogons");
+  }
+
+  @Test
+  public void shouldCreateNoLinksForHealthChecksWithoutUrl() {
+    when(scmContextProvider.getDocumentationVersion()).thenReturn("2.17.x");
+
+    Repository testRepository = createTestRepository();
+    HealthCheckFailure failure = new HealthCheckFailure("1", "vogons", "met vogons");
+    testRepository.setHealthCheckFailures(singletonList(failure));
+
+    RepositoryDto dto = mapper.map(testRepository);
+
+    assertThat(dto.getHealthCheckFailures())
+    .extracting("url")
+    .containsExactly(new Object[] {null});
+
+    assertThat(dto.getHealthCheckFailures().get(0).getLinks().getLinkBy("documentation"))
+      .isNotPresent();
+  }
+
   private ScmProtocol mockProtocol(String type, String protocol) {
     return new MockScmProtocol(type, protocol);
   }

scm-webapp/src/test/java/sonia/scm/repository/DefaultRepositoryManagerPerfTest.java

@@ -102,6 +102,8 @@ public class DefaultRepositoryManagerPerfTest {
 
   @Mock
   private AuthorizationCollector authzCollector;
+  @Mock
+  private RepositoryPostProcessor repositoryPostProcessor;
 
   /**
    * Setup object under test.
@@ -116,8 +118,8 @@ public class DefaultRepositoryManagerPerfTest {
       keyGenerator,
       repositoryDAO,
       handlerSet,
-      Providers.of(namespaceStrategy)
-    );
+      Providers.of(namespaceStrategy),
+      repositoryPostProcessor);
 
     setUpTestRepositories();
 

scm-webapp/src/test/java/sonia/scm/repository/DefaultRepositoryManagerTest.java

@@ -106,6 +106,7 @@ import sonia.scm.TempSCMContextProvider;
 public class DefaultRepositoryManagerTest extends ManagerTestBase<Repository> {
 
   private RepositoryDAO repositoryDAO;
+  private RepositoryPostProcessor postProcessor = mock(RepositoryPostProcessor.class);
 
   static {
     ThreadContext.unbindSubject();
@@ -181,6 +182,7 @@ public class DefaultRepositoryManagerTest extends ManagerTestBase<Repository> {
     heartOfGold = manager.get(id);
     assertNotNull(heartOfGold);
     assertEquals(description, heartOfGold.getDescription());
+    verify(postProcessor).postProcess(argThat(repository -> repository.getId().equals(id)));
   }
 
   @Test
@@ -227,6 +229,8 @@ public class DefaultRepositoryManagerTest extends ManagerTestBase<Repository> {
     assertNotSame(heartOfGold, heartReference);
     heartReference.setDescription("prototype ship");
     assertNotEquals(heartOfGold.getDescription(), heartReference.getDescription());
+    verify(postProcessor).postProcess(argThat(repository -> repository.getId().equals(heartOfGold.getId())));
+    verify(postProcessor).postProcess(argThat(repository -> repository.getId().equals(happyVerticalPeopleTransporter.getId())));
   }
 
   @Test
@@ -551,7 +555,7 @@ public class DefaultRepositoryManagerTest extends ManagerTestBase<Repository> {
     when(namespaceStrategy.createNamespace(Mockito.any(Repository.class))).thenAnswer(invocation -> mockedNamespace);
 
     return new DefaultRepositoryManager(contextProvider,
-      keyGenerator, repositoryDAO, handlerSet, Providers.of(namespaceStrategy));
+      keyGenerator, repositoryDAO, handlerSet, Providers.of(namespaceStrategy), postProcessor);
   }
 
   private RepositoryDAO createRepositoryDaoMock() {
@@ -618,9 +622,6 @@ public class DefaultRepositoryManagerTest extends ManagerTestBase<Repository> {
   private Repository createRepository(Repository repository) {
     manager.create(repository);
     assertNotNull(repository.getId());
-    assertNotNull(manager.get(repository.getId()));
-    assertTrue(repository.getCreationDate() > 0);
-
     return repository;
   }
 

scm-webapp/src/test/java/sonia/scm/repository/HealthCheckerTest.java

@@ -0,0 +1,278 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.repository;
+
+import org.apache.shiro.authz.AuthorizationException;
+import org.apache.shiro.subject.Subject;
+import org.apache.shiro.util.ThreadContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import sonia.scm.NotFoundException;
+import sonia.scm.repository.api.Command;
+import sonia.scm.repository.api.FullHealthCheckCommandBuilder;
+import sonia.scm.repository.api.RepositoryService;
+import sonia.scm.repository.api.RepositoryServiceFactory;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static com.google.common.collect.ImmutableSet.of;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.lenient;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class HealthCheckerTest {
+
+  private final Repository repository = RepositoryTestData.createHeartOfGold();
+  private final String repositoryId = repository.getId();
+
+  @Mock
+  private HealthCheck healthCheck1;
+  @Mock
+  private HealthCheck healthCheck2;
+
+  @Mock
+  private RepositoryManager repositoryManager;
+  @Mock
+  private RepositoryServiceFactory repositoryServiceFactory;
+  @Mock
+  private RepositoryService repositoryService;
+  @Mock
+  private RepositoryPostProcessor postProcessor;
+
+  @Mock
+  private Subject subject;
+
+  private HealthChecker checker;
+
+  @BeforeEach
+  void initializeChecker() {
+    this.checker = new HealthChecker(of(healthCheck1, healthCheck2), repositoryManager, repositoryServiceFactory, postProcessor);
+  }
+
+  @BeforeEach
+  void initSubject() {
+    ThreadContext.bind(subject);
+  }
+
+  @AfterEach
+  void cleanupSubject() {
+    ThreadContext.unbindSubject();
+  }
+
+  @Test
+  void shouldFailForNotExistingRepositoryId() {
+    assertThrows(NotFoundException.class, () -> checker.lightCheck("no-such-id"));
+  }
+
+  @Nested
+  class WithRepository {
+    @BeforeEach
+    void setUpRepository() {
+      doReturn(repository).when(repositoryManager).get(repositoryId);
+    }
+
+    @Test
+    void shouldComputeLightChecks() {
+      when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.unhealthy(createFailure("error1")));
+      when(healthCheck2.check(repository)).thenReturn(HealthCheckResult.unhealthy(createFailure("error2")));
+
+      checker.lightCheck(repositoryId);
+
+      verify(postProcessor).setCheckResults(eq(repository), argThat(failures -> {
+        assertThat(failures)
+          .hasSize(2)
+          .extracting("id").containsExactly("error1", "error2");
+        return true;
+      }));
+    }
+
+    @Test
+    void shouldLockWhileLightCheckIsRunning() throws InterruptedException {
+      CountDownLatch waitUntilSecondCheckHasRun = new CountDownLatch(1);
+      CountDownLatch waitForFirstCheckStarted = new CountDownLatch(1);
+
+      when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.healthy());
+      when(healthCheck2.check(repository)).thenAnswer(invocation -> {
+        waitForFirstCheckStarted.countDown();
+        waitUntilSecondCheckHasRun.await();
+        return HealthCheckResult.healthy();
+      });
+
+      new Thread(() -> checker.lightCheck(repositoryId)).start();
+
+      waitForFirstCheckStarted.await();
+      await().until(() -> {
+        checker.lightCheck(repositoryId);
+        return true;
+      });
+
+      waitUntilSecondCheckHasRun.countDown();
+
+      verify(healthCheck1).check(repository);
+    }
+
+    @Test
+    void shouldShowRunningCheck() throws InterruptedException {
+      CountDownLatch waitUntilVerification = new CountDownLatch(1);
+      CountDownLatch waitForFirstCheckStarted = new CountDownLatch(1);
+
+      assertThat(checker.checkRunning(repositoryId)).isFalse();
+
+      when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.healthy());
+      when(healthCheck2.check(repository)).thenAnswer(invocation -> {
+        waitForFirstCheckStarted.countDown();
+        waitUntilVerification.await();
+        return HealthCheckResult.healthy();
+      });
+
+      new Thread(() -> checker.lightCheck(repositoryId)).start();
+
+      waitForFirstCheckStarted.await();
+
+      assertThat(checker.checkRunning(repositoryId)).isTrue();
+
+      waitUntilVerification.countDown();
+
+      await().until(() -> !checker.checkRunning(repositoryId));
+    }
+
+    @Nested
+    class ForFullChecks {
+
+      @Mock
+      private FullHealthCheckCommandBuilder fullHealthCheckCommand;
+
+      @BeforeEach
+      void setUpRepository() {
+        when(repositoryServiceFactory.create(repository)).thenReturn(repositoryService);
+        lenient().when(repositoryService.getFullCheckCommand()).thenReturn(fullHealthCheckCommand);
+      }
+
+      @Test
+      void shouldComputeLightChecksForFullChecks() {
+        when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.healthy());
+        when(healthCheck2.check(repository)).thenReturn(HealthCheckResult.unhealthy(createFailure("error")));
+
+        checker.fullCheck(repositoryId);
+
+        verify(postProcessor).setCheckResults(eq(repository), argThat(failures -> {
+          assertThat(failures)
+            .hasSize(1)
+            .extracting("id").containsExactly("error");
+          return true;
+        }));
+      }
+
+      @Test
+      void shouldLockWhileFullCheckIsRunning() throws InterruptedException {
+        CountDownLatch waitUntilSecondCheckHasRun = new CountDownLatch(1);
+        CountDownLatch waitForFirstCheckStarted = new CountDownLatch(1);
+
+        when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.healthy());
+        when(healthCheck2.check(repository)).thenAnswer(invocation -> {
+          waitForFirstCheckStarted.countDown();
+          waitUntilSecondCheckHasRun.await();
+          return HealthCheckResult.healthy();
+        });
+
+        new Thread(() -> checker.fullCheck(repositoryId)).start();
+
+        waitForFirstCheckStarted.await();
+        await().until(() -> {
+          checker.fullCheck(repositoryId);
+          return true;
+        });
+
+        waitUntilSecondCheckHasRun.countDown();
+
+        verify(healthCheck1).check(repository);
+      }
+
+      @Test
+      void shouldComputeFullChecks() throws IOException {
+        when(healthCheck1.check(repository)).thenReturn(HealthCheckResult.healthy());
+        when(healthCheck2.check(repository)).thenReturn(HealthCheckResult.healthy());
+        when(repositoryService.isSupported(Command.FULL_HEALTH_CHECK)).thenReturn(true);
+        when(fullHealthCheckCommand.check()).thenReturn(HealthCheckResult.unhealthy(createFailure("error")));
+
+        checker.fullCheck(repositoryId);
+
+        verify(postProcessor).setCheckResults(eq(repository), argThat(failures -> {
+          assertThat(failures)
+            .hasSize(1)
+            .extracting("id").containsExactly("error");
+          return true;
+        }));
+      }
+    }
+  }
+
+  @Nested
+  class WithoutPermission {
+
+    @BeforeEach
+    void setMissingPermission() {
+      doThrow(AuthorizationException.class).when(subject).checkPermission("repository:healthCheck:" + repositoryId);
+    }
+
+    @Test
+    void shouldFailToRunLightChecksWithoutPermissionForId() {
+      assertThrows(AuthorizationException.class, () -> checker.lightCheck(repositoryId));
+    }
+
+    @Test
+    void shouldFailToRunLightChecksWithoutPermissionForRepository() {
+      assertThrows(AuthorizationException.class, () -> checker.lightCheck(repository));
+    }
+
+    @Test
+    void shouldFailToRunFullChecksWithoutPermissionForId() {
+      assertThrows(AuthorizationException.class, () -> checker.fullCheck(repositoryId));
+    }
+
+    @Test
+    void shouldFailToRunFullChecksWithoutPermissionForRepository() {
+      assertThrows(AuthorizationException.class, () -> checker.fullCheck(repository));
+    }
+  }
+
+  private HealthCheckFailure createFailure(String text) {
+    return new HealthCheckFailure(text, text, text);
+  }
+}

scm-webapp/src/test/java/sonia/scm/repository/RepositoryPostProcessorTest.java

@@ -0,0 +1,121 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020-present Cloudogu GmbH and Contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package sonia.scm.repository;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import sonia.scm.event.ScmEventBus;
+
+import static java.util.Collections.singleton;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+class RepositoryPostProcessorTest {
+
+  @Mock
+  private ScmEventBus eventBus;
+
+  @InjectMocks
+  RepositoryPostProcessor repositoryPostProcessor;
+
+  @Test
+  void shouldSetHealthChecksForRepository() {
+    Repository repository = RepositoryTestData.createHeartOfGold();
+    repositoryPostProcessor.setCheckResults(repository.clone(), singleton(new HealthCheckFailure("HOG", "improbable", "This is not very probable")));
+
+    repositoryPostProcessor.postProcess(repository);
+
+    assertThat(repository.getHealthCheckFailures())
+      .extracting("id")
+      .containsExactly("HOG");
+  }
+
+  @Test
+  void shouldSetEmptyListOfHealthChecksWhenNoResultsExist() {
+    Repository repository = RepositoryTestData.createHeartOfGold();
+
+    repositoryPostProcessor.postProcess(repository);
+
+    assertThat(repository.getHealthCheckFailures())
+      .isNotNull()
+      .isEmpty();
+  }
+
+  @Test
+  void shouldSetHealthChecksForRepositoryInSetter() {
+    Repository repository = RepositoryTestData.createHeartOfGold();
+    repositoryPostProcessor.setCheckResults(repository, singleton(new HealthCheckFailure("HOG", "improbable", "This is not very probable")));
+
+    assertThat(repository.getHealthCheckFailures())
+      .extracting("id")
+      .containsExactly("HOG");
+  }
+
+  @Test
+  void shouldTriggerHealthCheckEventForNewFailure() {
+    Repository repository = RepositoryTestData.createHeartOfGold();
+    repositoryPostProcessor.setCheckResults(repository, singleton(new HealthCheckFailure("HOG", "improbable", "This is not very probable")));
+
+    verify(eventBus).post(argThat(event -> {
+      HealthCheckEvent healthCheckEvent = (HealthCheckEvent) event;
+      assertThat(healthCheckEvent.getRepository())
+        .isEqualTo(repository);
+      assertThat(healthCheckEvent.getPreviousFailures())
+        .isEmpty();
+      assertThat(((HealthCheckEvent) event).getCurrentFailures())
+        .extracting("id")
+        .containsExactly("HOG");
+      return true;
+    }));
+  }
+
+  @Test
+  void shouldTriggerHealthCheckEventForDifferentFailure() {
+    Repository repository = RepositoryTestData.createHeartOfGold();
+    repositoryPostProcessor.setCheckResults(repository, singleton(new HealthCheckFailure("HOG", "improbable", "This is not very probable")));
+    repositoryPostProcessor.setCheckResults(repository, singleton(new HealthCheckFailure("VOG", "vogons", "Erased by Vogons")));
+
+    verify(eventBus).post(argThat(event -> {
+      HealthCheckEvent healthCheckEvent = (HealthCheckEvent) event;
+      if (healthCheckEvent.getPreviousFailures().isEmpty()) {
+        return false; // ignore event from first checks
+      }
+      assertThat((healthCheckEvent).getRepository())
+        .isEqualTo(repository);
+      assertThat((healthCheckEvent).getPreviousFailures())
+        .extracting("id")
+        .containsExactly("HOG");
+      assertThat((healthCheckEvent).getCurrentFailures())
+        .extracting("id")
+        .containsExactly("VOG");
+      return true;
+    }));
+  }
+}