Merge pull request #30 from abayer/infra-1263

[INFRA-1263] Test data and logic for Jenkinsfile for PR builds
jenkins-infra · Jul 5, 2017 · 83b6d51 · 83b6d51
2 parents 34db9e8 + 8beef3c
commit 83b6d51
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 8 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -9,7 +9,7 @@ final String JAVA_TOOL   = 'jdk8'
 final String GROOVY_TOOL = 'groovy'
 final String USAGE_HOST  = 'usage.jenkins.io'
 final String CENSUS_HOST = 'census.jenkins.io'
-
+final boolean isPRTest = !infra.isTrusted() && env.BRANCH_NAME
 
 if (!infra.isTrusted()) {
     echo 'TODO: Implement viable pull request validation code here :)'
@@ -33,27 +33,45 @@ else {
             "JAVA_HOME=${javaHome}",
         ]
 
-        final String usagestats_dir = './usage-stats'
-        final String census_dir = './census'
+        String usagestats_dir = './usage-stats'
+        String census_dir = './census'
+        String mongoDataDir = "mongo-data"
 
+        if (isPRTest) {
+            // If we're running for a PR build, use a fresh testing directory and nuke whatever was there previously.
+            sh "rm -rf testing"
+            sh "mkdir -p testing/census"
+            sh "cd testing && tar -xvzf ../test-data/usage.tar.gz ."
+            usagestats_dir = './testing/usage'
+            census_dir = './testing/census'
+            mongoDataDir = "testing/mongo-data"
+        }
         stage 'Sync raw data and census files'
-        sh "rsync -avz --delete ${USAGE_HOST}:/srv/usage/usage-stats ."
-        sh "rsync -avz --delete ${CENSUS_HOST}:/srv/census/census ."
-
+        if (infra.isTrusted()) {
+            sh "rsync -avz --delete ${USAGE_HOST}:/srv/usage/usage-stats ."
+            sh "rsync -avz --delete ${CENSUS_HOST}:/srv/census/census ."
+        }
 
         stage 'Process raw logs'
         // TODO: Fix ownership!
         // The mongo-data directory will end up containing files and dirs owned by 999:docker that we can't do much about for the moment.
         // Needs a better fix going forward, but for the moment...
-        sh "mkdir -p mongo-data"
+        sh "mkdir -p ${mongoDataDir}"
 
         // Use the Mongo data directory in the workspace.
-        docker.image('mongo:2').withRun('-p 27017:27017 -v ' + pwd() + "/mongo-data:/data/db") { container ->
+        docker.image('mongo:2').withRun('-p 27017:27017 -v ' + pwd() + "/" + mongoDataDir + ":/data/db") { container ->
             withEnv(customEnv) {
                 sh "groovy parseUsage.groovy --logs ${usagestats_dir} --output ${census_dir} --incremental"
             }
         }
 
+        if (isPRTest) {
+            // Make sure the expected files exist.
+            sh "test -f ${pwd()}/testing/census/201103.json.gz"
+            sh "test -f ${pwd()}/testing/census/201104.json.gz"
+            sh "test -f ${pwd()}/testing/census/201105.json.gz"
+        }
+
         stage 'Generate census data'
         withEnv(customEnv) {
             sh 'mkdir -p target'

diff --git a/test-data/usage.tar.gz b/test-data/usage.tar.gz