From 146a1b2700447a9be76dcd8b0e1b2e55d0256803 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Wed, 11 Jul 2018 16:59:55 +0200 Subject: [PATCH 01/15] Maven: Use the same and latest javadoc version in all modules. Unfortunately, not fixing our failing build. --- pom.xml | 13 +++++++++++-- scm-core/pom.xml | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2416371d48..9e38672db2 100644 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,17 @@ + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.1 + + + + @@ -254,7 +265,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9 true ${project.build.sourceEncoding} @@ -439,7 +449,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.8.1 org.jboss.apiviz.APIviz diff --git a/scm-core/pom.xml b/scm-core/pom.xml index ddd443afb4..a06eb544dc 100644 --- a/scm-core/pom.xml +++ b/scm-core/pom.xml @@ -159,7 +159,6 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9 true ${project.build.sourceEncoding} From 2588dc927198a498adc303f35a9acf9d32e35e67 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Wed, 11 Jul 2018 18:07:26 +0200 Subject: [PATCH 02/15] Jenkins: Use a defined SDK > 1.8.0 u101, as required by maven. --- Jenkinsfile | 61 +++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 64c881183a..71c5f8c6ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,44 +4,49 @@ import com.cloudogu.ces.cesbuildlib.* node() { // No specific label - properties([ - // Keep only the last 10 build to preserve space - buildDiscarder(logRotator(numToKeepStr: '10')), - // Don't run concurrent builds for a branch, because they use the same workspace directory - disableConcurrentBuilds() - ]) + properties([ + // Keep only the last 10 build to preserve space + buildDiscarder(logRotator(numToKeepStr: '10')), + // Don't run concurrent builds for a branch, because they use the same workspace directory + disableConcurrentBuilds() + ]) - String defaultEmailRecipients = env.EMAIL_SCM_RECIPIENTS + String defaultEmailRecipients = env.EMAIL_SCM_RECIPIENTS - catchError { + catchError { - Maven mvn = new MavenWrapper(this) + Maven mvn = new MavenWrapper(this) + // Maven build specified it must be 1.8.0-101 or newer + def javaHome = tool 'JDK-1.8.0-101+' - stage('Checkout') { - checkout scm - } + withEnv(["JAVA_HOME=${javaHome}", "PATH=${env.JAVA_HOME}/bin:${env.PATH}"]) { - stage('Build') { - mvn 'clean install -DskipTests -DperformRelease' - archive '**/target/*.jar,**/target/*.zip' - } + stage('Checkout') { + checkout scm + } - stage('Unit Test') { - mvn 'test -Dsonia.scm.test.skip.hg=true' - } + stage('Build') { + mvn 'clean install -DskipTests -DperformRelease' + archive '**/target/*.jar,**/target/*.zip' + } - stage('SonarQube') { - def sonarQube = new SonarQube(this, 'ces-sonar') + stage('Unit Test') { + mvn 'test -Dsonia.scm.test.skip.hg=true' + } - sonarQube.analyzeWith(mvn) - } + stage('SonarQube') { + def sonarQube = new SonarQube(this, 'ces-sonar') + + sonarQube.analyzeWith(mvn) + } } + } - // Archive Unit and integration test results, if any - junit allowEmptyResults: true, testResults: '**/target/failsafe-reports/TEST-*.xml,**/target/surefire-reports/TEST-*.xml,**/target/jest-reports/TEST-*.xml' + // Archive Unit and integration test results, if any + junit allowEmptyResults: true, testResults: '**/target/failsafe-reports/TEST-*.xml,**/target/surefire-reports/TEST-*.xml,**/target/jest-reports/TEST-*.xml' - // Find maven warnings and visualize in job - warnings consoleParsers: [[parserName: 'Maven']], canRunOnFailed: true + // Find maven warnings and visualize in job + warnings consoleParsers: [[parserName: 'Maven']], canRunOnFailed: true - mailIfStatusChanged(defaultEmailRecipients) + mailIfStatusChanged(defaultEmailRecipients) } From 7cfcc48995b1a93490f264ef4c3da334a44020ec Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Wed, 11 Jul 2018 18:09:06 +0200 Subject: [PATCH 03/15] Jenkins: Ignore JavaDoc errors for now. JDK8 is more strict, we should fix this before the next release. But not at the point where we're migrating our Jenkins. --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 71c5f8c6ad..c5d1adf192 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -26,7 +26,7 @@ node() { // No specific label } stage('Build') { - mvn 'clean install -DskipTests -DperformRelease' + mvn 'clean install -DskipTests -DperformRelease -Dmaven.javadoc.failOnError=false' archive '**/target/*.jar,**/target/*.zip' } From e15787631c994a90223a4fb59434d93def7aabbc Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Wed, 11 Jul 2018 18:10:29 +0200 Subject: [PATCH 04/15] Jenkins: No longer archive artifacts. This consume a lot of space on Jenkins and is not needed very often. If needed we should deploy SNAPSHOTs to nexus. --- Jenkinsfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index c5d1adf192..f566478a63 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,6 @@ node() { // No specific label stage('Build') { mvn 'clean install -DskipTests -DperformRelease -Dmaven.javadoc.failOnError=false' - archive '**/target/*.jar,**/target/*.zip' } stage('Unit Test') { From 5697c850f3581f5c833cd78c56169358ee00b52f Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Wed, 11 Jul 2018 18:23:49 +0200 Subject: [PATCH 05/15] Jenkins: Updates SonarQube analysis props for SonarCloud and BitBucket. --- Jenkinsfile | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f566478a63..ae4b0767cf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ #!groovy -@Library('github.com/cloudogu/ces-build-lib@ac17d45') +@Library('github.com/cloudogu/ces-build-lib@9aadeeb') import com.cloudogu.ces.cesbuildlib.* node() { // No specific label @@ -19,13 +19,16 @@ node() { // No specific label // Maven build specified it must be 1.8.0-101 or newer def javaHome = tool 'JDK-1.8.0-101+' - withEnv(["JAVA_HOME=${javaHome}", "PATH=${env.JAVA_HOME}/bin:${env.PATH}"]) { + withEnv(["JAVA_HOME=${javaHome}", "PATH=${env.JAVA_HOME}/bin:${env.PATH}", + // Give Maven enough memory to do SonarQube analysis + "MAVEN_OPTS=-Xmx1g"]) { stage('Checkout') { checkout scm } stage('Build') { + // TODO release build only on default? or 2.0.0-M3 -> JavaDoc takes ages mvn 'clean install -DskipTests -DperformRelease -Dmaven.javadoc.failOnError=false' } @@ -34,9 +37,15 @@ node() { // No specific label } stage('SonarQube') { - def sonarQube = new SonarQube(this, 'ces-sonar') - sonarQube.analyzeWith(mvn) + def sonarQube = new SonarQube(this, 'sonarcloud.io') + + // TODO move this to ces-build-lib so we can use "sonarqube.analyzeWith(mvn)" here + analyzeWith(mvn) + + if (!sonarQube.waitForQualityGateWebhookToBeCalled()) { + currentBuild.result = 'UNSTABLE' + } } } } @@ -49,3 +58,30 @@ node() { // No specific label mailIfStatusChanged(defaultEmailRecipients) } + +void analyzeWith(Maven mvn) { + + withSonarQubeEnv('sonarcloud.io') { + + String mvnArgs = "${env.SONAR_MAVEN_GOAL} " + + "-Dsonar.host.url=${env.SONAR_HOST_URL} " + + "-Dsonar.login=${env.SONAR_AUTH_TOKEN} " + + if (isPullRequest()) { + echo "Analysing SQ in PR mode" + mvnArgs += "-Dsonar.pullrequest.base=${env.CHANGE_TARGET} " + + "-Dsonar.pullrequest.branch=${env.CHANGE_BRANCH} " + + "-Dsonar.pullrequest.key=${env.CHANGE_ID} " + + "-Dsonar.pullrequest.provider=bitbucketcloud " + + "-Dsonar.pullrequest.bitbucketcloud.owner=sdorra " + + "-Dsonar.pullrequest.bitbucketcloud.repository=sonarcloudtest " + } else { + mvnArgs += " -Dsonar.branch.name=${env.BRANCH_NAME} " + if (!"default".equals(env.BRANCH_NAME)) { + // Avoid exception "The main branch must not have a target" on master branch + mvnArgs += " -Dsonar.branch.target=default " + } + } + mvn "${mvnArgs}" + } +} From b4a0578ad332e4f482ae5af86d46f0b38b036381 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 10:32:24 +0200 Subject: [PATCH 06/15] Jenkins: Send emails to last committer only --- Jenkinsfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ae4b0767cf..09b785286c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,7 +56,7 @@ node() { // No specific label // Find maven warnings and visualize in job warnings consoleParsers: [[parserName: 'Maven']], canRunOnFailed: true - mailIfStatusChanged(defaultEmailRecipients) + mailIfStatusChanged(commitAuthorEmail) } void analyzeWith(Maven mvn) { @@ -85,3 +85,12 @@ void analyzeWith(Maven mvn) { mvn "${mvnArgs}" } } + +String getCommitAuthorComplete() { + new Sh(this).returnStdOut 'hg log --branch . --limit 1 --template "{author}"' +} + +String getCommitAuthorEmail() { + def matcher = getCommitAuthorComplete() =~ "<(.*?)>" + matcher ? matcher[0][1] : "" +} From 461645ede9793f1eeea4717a47899b201e6a571e Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 10:55:03 +0200 Subject: [PATCH 07/15] Jenkins: Tell SQ 2.0.0-m3 is the base branch for now --- Jenkinsfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 09b785286c..8467ce5804 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,6 @@ node() { // No specific label def sonarQube = new SonarQube(this, 'sonarcloud.io') - // TODO move this to ces-build-lib so we can use "sonarqube.analyzeWith(mvn)" here analyzeWith(mvn) if (!sonarQube.waitForQualityGateWebhookToBeCalled()) { @@ -59,6 +58,9 @@ node() { // No specific label mailIfStatusChanged(commitAuthorEmail) } +// Change this as when we go back to default - necessary for proper SonarQube analysis +String mainBranch = "2.0.0-m3" + void analyzeWith(Maven mvn) { withSonarQubeEnv('sonarcloud.io') { @@ -77,9 +79,9 @@ void analyzeWith(Maven mvn) { "-Dsonar.pullrequest.bitbucketcloud.repository=sonarcloudtest " } else { mvnArgs += " -Dsonar.branch.name=${env.BRANCH_NAME} " - if (!"default".equals(env.BRANCH_NAME)) { - // Avoid exception "The main branch must not have a target" on master branch - mvnArgs += " -Dsonar.branch.target=default " + if (!mainBranch.equals(env.BRANCH_NAME)) { + // Avoid exception "The main branch must not have a target" on main branch + mvnArgs += " -Dsonar.branch.target=${mainBranch} " } } mvn "${mvnArgs}" From f8b8400dce200b3e869a554de248ce2ea6a6ba42 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 11:01:37 +0200 Subject: [PATCH 08/15] Jenkins: Remove maven opts. Randomly failing builds (during JavaDoc or SonarQube) were more likely to be caused by using "small" instances by default on cloudbees. --- Jenkinsfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8467ce5804..2cfa04cbb0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,17 +11,13 @@ node() { // No specific label disableConcurrentBuilds() ]) - String defaultEmailRecipients = env.EMAIL_SCM_RECIPIENTS - catchError { Maven mvn = new MavenWrapper(this) // Maven build specified it must be 1.8.0-101 or newer def javaHome = tool 'JDK-1.8.0-101+' - withEnv(["JAVA_HOME=${javaHome}", "PATH=${env.JAVA_HOME}/bin:${env.PATH}", - // Give Maven enough memory to do SonarQube analysis - "MAVEN_OPTS=-Xmx1g"]) { + withEnv(["JAVA_HOME=${javaHome}", "PATH=${env.JAVA_HOME}/bin:${env.PATH}"]) { stage('Checkout') { checkout scm From 41a937c15c6d14ff8f191453fc70df3c84b4167b Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 11:06:54 +0200 Subject: [PATCH 09/15] Enables syntax completion for ces-build-lib in Jenkinsfile --- Jenkinsfile | 2 ++ pom.xml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 2cfa04cbb0..30a09458f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,6 @@ #!groovy + +// Keep the version in sync with the one used in pom.xml in order to get correct syntax completion. @Library('github.com/cloudogu/ces-build-lib@9aadeeb') import com.cloudogu.ces.cesbuildlib.* diff --git a/pom.xml b/pom.xml index 9e38672db2..8d3b3fb0b2 100644 --- a/pom.xml +++ b/pom.xml @@ -145,6 +145,16 @@ ${mokito.version} test + + + + com.github.cloudogu + ces-build-lib + + 9aadeeb + + true + From 9519255f8c0200075f238d2e015fd710d54d879e Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 11:15:24 +0200 Subject: [PATCH 10/15] Jenkins: Do release build only on main branch. JavaDoc takes ages, we want faster feedback! --- Jenkinsfile | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 30a09458f8..ff9c5ae4d3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -15,7 +15,7 @@ node() { // No specific label catchError { - Maven mvn = new MavenWrapper(this) + Maven mvn = setupMavenBuild() // Maven build specified it must be 1.8.0-101 or newer def javaHome = tool 'JDK-1.8.0-101+' @@ -26,8 +26,7 @@ node() { // No specific label } stage('Build') { - // TODO release build only on default? or 2.0.0-M3 -> JavaDoc takes ages - mvn 'clean install -DskipTests -DperformRelease -Dmaven.javadoc.failOnError=false' + mvn 'clean install -DskipTests' } stage('Unit Test') { @@ -59,6 +58,18 @@ node() { // No specific label // Change this as when we go back to default - necessary for proper SonarQube analysis String mainBranch = "2.0.0-m3" +Maven setupMavenBuild() { + Maven mvn = new MavenWrapper(this) + + if (env.BRANCH_NAME == "master") { + // Release starts javadoc, which takes very long, so do only for certain branches + mvn.additionalArgs += ' -DperformRelease' + // JDK8 is more strict, we should fix this before the next release. Right now, this is just not the focus, yet. + mvn.additionalArgs += ' -Dmaven.javadoc.failOnError=false' + } + return mvn +} + void analyzeWith(Maven mvn) { withSonarQubeEnv('sonarcloud.io') { From f3374255b5723bb205cdfa5f95dc899ac5c460c1 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 11:46:50 +0200 Subject: [PATCH 11/15] Jenkins: Declares main branch earlier in the build. Fixes No such property: mainBranch for class: groovy.lang.Binding --- Jenkinsfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index ff9c5ae4d3..5d8a652c17 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -6,6 +6,9 @@ import com.cloudogu.ces.cesbuildlib.* node() { // No specific label + // Change this as when we go back to default - necessary for proper SonarQube analysis + mainBranch = "2.0.0-m3" + properties([ // Keep only the last 10 build to preserve space buildDiscarder(logRotator(numToKeepStr: '10')), @@ -55,13 +58,12 @@ node() { // No specific label mailIfStatusChanged(commitAuthorEmail) } -// Change this as when we go back to default - necessary for proper SonarQube analysis -String mainBranch = "2.0.0-m3" +String mainBranch Maven setupMavenBuild() { Maven mvn = new MavenWrapper(this) - if (env.BRANCH_NAME == "master") { + if (mainBranch.equals(env.BRANCH_NAME)) { // Release starts javadoc, which takes very long, so do only for certain branches mvn.additionalArgs += ' -DperformRelease' // JDK8 is more strict, we should fix this before the next release. Right now, this is just not the focus, yet. From 4859be47b711ee2f63acf53ff2a11f1cad14fbb7 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 12:05:59 +0200 Subject: [PATCH 12/15] Sonar: Exclude complete legacy webapp. This will be deleted soon and the new webapp will have its own module. --- scm-webapp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scm-webapp/pom.xml b/scm-webapp/pom.xml index 98c70ff82a..d02db4d203 100644 --- a/scm-webapp/pom.xml +++ b/scm-webapp/pom.xml @@ -566,7 +566,7 @@ e1 javascript:S3827 **.js - src/main/webapp/resources/extjs/**,src/main/webapp/resources/moment/**,src/main/webapp/resources/syntaxhighlighter/** + src/main/webapp** From 36e5a5fc8bad66e5dce080ecc5c3570f6801c24f Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 13:04:06 +0200 Subject: [PATCH 13/15] Jenkins: Allows for concurrent builds. On Cloudbees every build is started on a new worker. So no need to restrict execution. --- Jenkinsfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5d8a652c17..145e7f2691 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,8 +12,6 @@ node() { // No specific label properties([ // Keep only the last 10 build to preserve space buildDiscarder(logRotator(numToKeepStr: '10')), - // Don't run concurrent builds for a branch, because they use the same workspace directory - disableConcurrentBuilds() ]) catchError { From 73e6983ad6d79755b90a4bda69086d068cb2dfa1 Mon Sep 17 00:00:00 2001 From: Johannes Schnatterer Date: Thu, 12 Jul 2018 13:05:42 +0200 Subject: [PATCH 14/15] Jenkins: Fixes repo name for SonarQube PR analysis --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 145e7f2691..f139f54906 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -85,7 +85,7 @@ void analyzeWith(Maven mvn) { "-Dsonar.pullrequest.key=${env.CHANGE_ID} " + "-Dsonar.pullrequest.provider=bitbucketcloud " + "-Dsonar.pullrequest.bitbucketcloud.owner=sdorra " + - "-Dsonar.pullrequest.bitbucketcloud.repository=sonarcloudtest " + "-Dsonar.pullrequest.bitbucketcloud.repository=scm-manager " } else { mvnArgs += " -Dsonar.branch.name=${env.BRANCH_NAME} " if (!mainBranch.equals(env.BRANCH_NAME)) { From dba8b5faeb6351704ec45a67c5f5f4ed6bc402bf Mon Sep 17 00:00:00 2001 From: Sebastian Sdorra Date: Fri, 13 Jul 2018 09:04:09 +0000 Subject: [PATCH 15/15] Close branch feature/jenkins_on_cloudbees