From d6fff29a721cca2dc022b938be02129fe6bf8ca5 Mon Sep 17 00:00:00 2001 From: shimamoto Date: Wed, 19 Dec 2018 17:07:13 +0900 Subject: [PATCH 1/3] Run database tests in a Docker container --- build.sbt | 3 +- .../core/GitBucketCoreModuleSpec.scala | 118 +++++++++++++----- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/build.sbt b/build.sbt index 873e9d540..a24b89f88 100644 --- a/build.sbt +++ b/build.sbt @@ -66,8 +66,7 @@ libraryDependencies ++= Seq( "junit" % "junit" % "4.12" % "test", "org.scalatra" %% "scalatra-scalatest" % ScalatraVersion % "test", "org.mockito" % "mockito-core" % "2.23.4" % "test", - "com.wix" % "wix-embedded-mysql" % "4.2.0" % "test", - "ru.yandex.qatools.embed" % "postgresql-embedded" % "2.10" % "test", + "com.dimafeng" %% "testcontainers-scala" % "0.22.0" % "test", "net.i2p.crypto" % "eddsa" % "0.3.0", "is.tagomor.woothee" % "woothee-java" % "1.8.0", "org.ec4j.core" % "ec4j-core" % "0.0.3" diff --git a/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala b/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala index 4c253f276..a2c2a3409 100644 --- a/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala +++ b/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala @@ -1,19 +1,17 @@ package gitbucket.core import java.sql.DriverManager +import java.time.Duration +import java.time.temporal.ChronoUnit.SECONDS +import com.dimafeng.testcontainers.GenericContainer import io.github.gitbucket.solidbase.Solidbase import io.github.gitbucket.solidbase.model.Module import liquibase.database.core.{H2Database, MySQLDatabase, PostgresDatabase} +import org.junit.runner.Description import org.scalatest.{FunSuite, Tag} -import com.wix.mysql.EmbeddedMysql._ -import com.wix.mysql.config.Charset -import com.wix.mysql.config.MysqldConfig._ -import com.wix.mysql.distribution.Version._ -import ru.yandex.qatools.embed.postgresql.PostgresStarter -import ru.yandex.qatools.embed.postgresql.config.AbstractPostgresConfig.{Credentials, Net, Storage, Timeout} -import ru.yandex.qatools.embed.postgresql.config.PostgresConfig -import ru.yandex.qatools.embed.postgresql.distribution.Version.Main.PRODUCTION +import org.testcontainers.containers.ContainerLaunchException +import org.testcontainers.containers.wait.strategy.{HostPortWaitStrategy, Wait} object ExternalDBTest extends Tag("ExternalDBTest") @@ -28,52 +26,106 @@ class GitBucketCoreModuleSpec extends FunSuite { ) } - test("Migration MySQL", ExternalDBTest) { - val config = aMysqldConfig(v5_7_latest) - .withPort(3306) - .withUser("sa", "sa") - .withCharset(Charset.UTF8) - .withServerVariable("bind-address", "127.0.0.1") - .build() + implicit private val suiteDescription = Description.createSuiteDescription(getClass) - val mysqld = anEmbeddedMysql(config) - .addSchema("gitbucket") - .start() + test("Migration MySQL 5.7", ExternalDBTest) { + val container = GenericContainer( + "mysql:5.7", + env = Map("MYSQL_ROOT_PASSWORD" -> "my-secret-pw", "MYSQL_DATABASE" -> "gitbucket"), + waitStrategy = new HostPortWaitStrategy { + override def waitUntilReady(): Unit = { + super.waitUntilReady() + def readyForConnections(retry: Int = 0): Boolean = { + var con: java.sql.Connection = null + try { + con = DriverManager.getConnection( + s"jdbc:mysql://${waitStrategyTarget.getContainerIpAddress}:${waitStrategyTarget.getMappedPort(3306)}/gitbucket?useSSL=false", + "root", + "my-secret-pw" + ) + con.createStatement().execute("SELECT 1") + } catch { + case _: Exception if retry < 3 => + Thread.sleep(10000) + readyForConnections(retry + 1) + case _: Exception => false + } finally { + Option(con).foreach(_.close()) + } + } + + if (!readyForConnections()) throw new ContainerLaunchException("Timed out") + } + } + ) + + container.starting() try { new Solidbase().migrate( - DriverManager.getConnection("jdbc:mysql://localhost:3306/gitbucket?useSSL=false", "sa", "sa"), + DriverManager.getConnection( + s"jdbc:mysql://${container.containerIpAddress}:${container.mappedPort(3306)}/gitbucket?useSSL=false", + "root", + "my-secret-pw" + ), Thread.currentThread().getContextClassLoader(), new MySQLDatabase(), new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) ) } finally { - mysqld.stop() + container.finished() } } - test("Migration PostgreSQL", ExternalDBTest) { - val runtime = PostgresStarter.getDefaultInstance() - val config = new PostgresConfig( - PRODUCTION, - new Net("localhost", 5432), - new Storage("gitbucket"), - new Timeout(), - new Credentials("sa", "sa") + test("Migration PostgreSQL 11", ExternalDBTest) { + val container = GenericContainer( + "postgres:11", + env = Map("POSTGRES_PASSWORD" -> "mysecretpassword", "POSTGRES_DB" -> "gitbucket"), + waitStrategy = Wait + .forLogMessage(".*database system is ready to accept connections.*\\s", 2) + .withStartupTimeout(Duration.of(60, SECONDS)) ) - val exec = runtime.prepare(config) - val process = exec.start() - + container.starting() try { new Solidbase().migrate( - DriverManager.getConnection("jdbc:postgresql://localhost:5432/gitbucket", "sa", "sa"), + DriverManager.getConnection( + s"jdbc:postgresql://${container.containerIpAddress}:${container.mappedPort(5432)}/gitbucket", + "postgres", + "mysecretpassword" + ), Thread.currentThread().getContextClassLoader(), new PostgresDatabase(), new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) ) } finally { - process.stop() + container.finished() + } + } + + test("Migration PostgreSQL 10", ExternalDBTest) { + val container = GenericContainer( + "postgres:10", + env = Map("POSTGRES_PASSWORD" -> "mysecretpassword", "POSTGRES_DB" -> "gitbucket"), + waitStrategy = Wait + .forLogMessage(".*database system is ready to accept connections.*\\s", 2) + .withStartupTimeout(Duration.of(60, SECONDS)) + ) + + container.starting() + try { + new Solidbase().migrate( + DriverManager.getConnection( + s"jdbc:postgresql://${container.containerIpAddress}:${container.mappedPort(5432)}/gitbucket", + "postgres", + "mysecretpassword" + ), + Thread.currentThread().getContextClassLoader(), + new PostgresDatabase(), + new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) + ) + } finally { + container.finished() } } From f8013c0ec01d8f6d777e594056fbb47a9beaf32d Mon Sep 17 00:00:00 2001 From: shimamoto Date: Wed, 19 Dec 2018 19:45:11 +0900 Subject: [PATCH 2/3] Use Scala wrapper for testcontainers-java --- build.sbt | 2 + .../core/GitBucketCoreModuleSpec.scala | 130 +++++------------- 2 files changed, 35 insertions(+), 97 deletions(-) diff --git a/build.sbt b/build.sbt index a24b89f88..c2a58bf43 100644 --- a/build.sbt +++ b/build.sbt @@ -67,6 +67,8 @@ libraryDependencies ++= Seq( "org.scalatra" %% "scalatra-scalatest" % ScalatraVersion % "test", "org.mockito" % "mockito-core" % "2.23.4" % "test", "com.dimafeng" %% "testcontainers-scala" % "0.22.0" % "test", + "org.testcontainers" % "mysql" % "1.10.3" % "test", + "org.testcontainers" % "postgresql" % "1.10.3" % "test", "net.i2p.crypto" % "eddsa" % "0.3.0", "is.tagomor.woothee" % "woothee-java" % "1.8.0", "org.ec4j.core" % "ec4j-core" % "0.0.3" diff --git a/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala b/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala index a2c2a3409..b5dfdf961 100644 --- a/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala +++ b/src/test/scala/gitbucket/core/GitBucketCoreModuleSpec.scala @@ -1,17 +1,13 @@ package gitbucket.core import java.sql.DriverManager -import java.time.Duration -import java.time.temporal.ChronoUnit.SECONDS -import com.dimafeng.testcontainers.GenericContainer +import com.dimafeng.testcontainers.{MySQLContainer, PostgreSQLContainer} import io.github.gitbucket.solidbase.Solidbase import io.github.gitbucket.solidbase.model.Module import liquibase.database.core.{H2Database, MySQLDatabase, PostgresDatabase} import org.junit.runner.Description import org.scalatest.{FunSuite, Tag} -import org.testcontainers.containers.ContainerLaunchException -import org.testcontainers.containers.wait.strategy.{HostPortWaitStrategy, Wait} object ExternalDBTest extends Tag("ExternalDBTest") @@ -28,104 +24,44 @@ class GitBucketCoreModuleSpec extends FunSuite { implicit private val suiteDescription = Description.createSuiteDescription(getClass) - test("Migration MySQL 5.7", ExternalDBTest) { - val container = GenericContainer( - "mysql:5.7", - env = Map("MYSQL_ROOT_PASSWORD" -> "my-secret-pw", "MYSQL_DATABASE" -> "gitbucket"), - waitStrategy = new HostPortWaitStrategy { - override def waitUntilReady(): Unit = { - super.waitUntilReady() - - def readyForConnections(retry: Int = 0): Boolean = { - var con: java.sql.Connection = null - try { - con = DriverManager.getConnection( - s"jdbc:mysql://${waitStrategyTarget.getContainerIpAddress}:${waitStrategyTarget.getMappedPort(3306)}/gitbucket?useSSL=false", - "root", - "my-secret-pw" - ) - con.createStatement().execute("SELECT 1") - } catch { - case _: Exception if retry < 3 => - Thread.sleep(10000) - readyForConnections(retry + 1) - case _: Exception => false - } finally { - Option(con).foreach(_.close()) - } - } - - if (!readyForConnections()) throw new ContainerLaunchException("Timed out") + Seq("8.0", "5.7").foreach { tag => + test(s"Migration MySQL $tag", ExternalDBTest) { + val container = new MySQLContainer() { + override val container = new org.testcontainers.containers.MySQLContainer(s"mysql:$tag") { + override def getDriverClassName = "org.mariadb.jdbc.Driver" } + // TODO https://github.com/testcontainers/testcontainers-java/issues/736 + container.withCommand("mysqld --default-authentication-plugin=mysql_native_password") + } + container.starting() + try { + new Solidbase().migrate( + DriverManager.getConnection(s"${container.jdbcUrl}?useSSL=false", container.username, container.password), + Thread.currentThread().getContextClassLoader(), + new MySQLDatabase(), + new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) + ) + } finally { + container.finished() } - ) - - container.starting() - try { - new Solidbase().migrate( - DriverManager.getConnection( - s"jdbc:mysql://${container.containerIpAddress}:${container.mappedPort(3306)}/gitbucket?useSSL=false", - "root", - "my-secret-pw" - ), - Thread.currentThread().getContextClassLoader(), - new MySQLDatabase(), - new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) - ) - } finally { - container.finished() } } - test("Migration PostgreSQL 11", ExternalDBTest) { - val container = GenericContainer( - "postgres:11", - env = Map("POSTGRES_PASSWORD" -> "mysecretpassword", "POSTGRES_DB" -> "gitbucket"), - waitStrategy = Wait - .forLogMessage(".*database system is ready to accept connections.*\\s", 2) - .withStartupTimeout(Duration.of(60, SECONDS)) - ) + Seq("11", "10").foreach { tag => + test(s"Migration PostgreSQL $tag", ExternalDBTest) { + val container = PostgreSQLContainer(s"postgres:$tag") - container.starting() - try { - new Solidbase().migrate( - DriverManager.getConnection( - s"jdbc:postgresql://${container.containerIpAddress}:${container.mappedPort(5432)}/gitbucket", - "postgres", - "mysecretpassword" - ), - Thread.currentThread().getContextClassLoader(), - new PostgresDatabase(), - new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) - ) - } finally { - container.finished() - } - } - - test("Migration PostgreSQL 10", ExternalDBTest) { - val container = GenericContainer( - "postgres:10", - env = Map("POSTGRES_PASSWORD" -> "mysecretpassword", "POSTGRES_DB" -> "gitbucket"), - waitStrategy = Wait - .forLogMessage(".*database system is ready to accept connections.*\\s", 2) - .withStartupTimeout(Duration.of(60, SECONDS)) - ) - - container.starting() - try { - new Solidbase().migrate( - DriverManager.getConnection( - s"jdbc:postgresql://${container.containerIpAddress}:${container.mappedPort(5432)}/gitbucket", - "postgres", - "mysecretpassword" - ), - Thread.currentThread().getContextClassLoader(), - new PostgresDatabase(), - new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) - ) - } finally { - container.finished() + container.starting() + try { + new Solidbase().migrate( + DriverManager.getConnection(container.jdbcUrl, container.username, container.password), + Thread.currentThread().getContextClassLoader(), + new PostgresDatabase(), + new Module(GitBucketCoreModule.getModuleId, GitBucketCoreModule.getVersions) + ) + } finally { + container.finished() + } } } From a6d682fdeecb4fdd1ed45a180d72e80523de916a Mon Sep 17 00:00:00 2001 From: Naoki Takezoe Date: Wed, 19 Dec 2018 20:42:54 +0900 Subject: [PATCH 3/3] Fix for MySQL8 --- src/main/resources/update/gitbucket-core_4.0.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/update/gitbucket-core_4.0.xml b/src/main/resources/update/gitbucket-core_4.0.xml index 04f33fe53..e22660fff 100644 --- a/src/main/resources/update/gitbucket-core_4.0.xml +++ b/src/main/resources/update/gitbucket-core_4.0.xml @@ -206,7 +206,7 @@ - + @@ -348,4 +348,4 @@ - \ No newline at end of file +