From 98162194e4d2df6cc9ce6fbd91f2e2e2c4ee6409 Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 1/6] major refactoring of the c3i global variable and supporting classes


This change addresses a number of deficiencies with the c3i library.

Previously, the library was forced to make "openshift.withCluster()" calls because the
context created in the calling pipeline was lost when calling "openshift" from a library
method. The library methods have been refactored to take a "script" arg, which is a
reference to the calling pipeline. The "openshift" variable on that pipeline can be called
directly, which retains the context from the pipeline. Because of this, library methods can
now be used to interact with multiple OpenShift project from the same pipeline.

This also enables the return values of library methods to be OpenShift "selectors", rather
than simple strings, which allows the pipeline to perform more complex actions on objects
created by library methods.

Unit test coverage is also significantly expanded. Additional convenience methods for
using the "Deployer" class have also been added to the c3i variable.

---

diff --git a/src/com/redhat/c3i/util/Builder.groovy b/src/com/redhat/c3i/util/Builder.groovy
index 3cea805..6f1d635 100644
--- a/src/com/redhat/c3i/util/Builder.groovy
+++ b/src/com/redhat/c3i/util/Builder.groovy
@@ -3,65 +3,76 @@
 
 package com.redhat.c3i.util
 
-def build(List models, boolean wait, Object... args) {
-  openshift.withCluster() {
-    def objects = openshift.apply(models)
+class Builder implements Serializable {
+  def script
+
+  def build(Map model, String... args) {
+    def bc = script.openshift.apply(model)
+    return build(bc, args)
+  }
+
+  def build(List models, String... args) {
+    def objects = script.openshift.apply(models)
     def bc = objects.narrow("bc")
-    return _build(bc, wait, args)
+    return build(bc, args)
   }
-}
 
-def build(bcname, boolean wait, Object... args) {
-  openshift.withCluster() {
-    def bc = openshift.selector(bcname)
-    return _build(bc, wait, args)
+  def build(String bcname, String... args) {
+    def bc = script.openshift.selector(bcname)
+    return build(bc, args)
   }
-}
 
-def _build(bc, boolean wait, Object... args) {
-  def build = bc.startBuild(args as String[]).narrow("build")
-  if (wait) {
-    _wait(build)
+  def build(selector, String... args) {
+    return selector.startBuild(args as String[]).narrow("build")
   }
-  return build.name()
-}
 
-def wait(buildsel) {
-  openshift.withCluster() {
-    def build = openshift.selector(buildsel)
-    return _wait(build)
+  def waitForStart(String buildname, Integer timeout=5) {
+    def build = script.openshift.selector(buildname)
+    return waitForStart(build, timeout)
   }
-}
 
-def _wait(build) {
-  echo "Waiting for ${build.name()} to start..."
-  timeout(5) {
-    build.watch {
-      return !(it.object().status.phase in ["New", "Pending", "Unknown"])
+  def waitForStart(build, Integer timeout=5) {
+    script.echo "Waiting for ${build.name()} to start..."
+    script.timeout(time: timeout) {
+      build.watch {
+        return !(it.object().status.phase in ["New", "Pending", "Unknown"])
+      }
     }
+    return build
   }
-  def buildobj = build.object()
-  def buildurl = buildobj.metadata.annotations['openshift.io/jenkins-build-uri']
-  if (buildurl) {
-    echo "Details: ${buildurl}"
+
+  def wait(String buildname, Integer timeout=60) {
+    def build = script.openshift.selector(buildname)
+    return wait(build, timeout)
   }
-  if (buildobj.spec.strategy.type == "JenkinsPipeline") {
-    echo "Waiting for ${build.name()} to complete..."
-    build.logs("--tail=1")
-    timeout(60) {
-      build.watch {
-        it.object().status.phase != "Running"
+
+  def wait(build, Integer timeout=60) {
+    waitForStart(build)
+    def buildobj = build.object()
+    def buildurl = buildobj.metadata?.annotations?.get('openshift.io/jenkins-build-uri')
+    if (buildurl) {
+      script.echo "Details: ${buildurl}"
+    }
+    if (buildobj.spec.strategy.type == "JenkinsPipeline") {
+      script.echo "Waiting for ${build.name()} to complete..."
+      build.logs("--tail=1")
+      script.timeout(time: timeout) {
+        build.watch {
+          it.object().status.phase != "Running"
+        }
+      }
+    } else {
+      script.echo "Following build logs..."
+      script.timeout(time: timeout) {
+        while (build.object().status.phase == "Running") {
+          build.logs("--tail=1", "--timestamps=true", "-f")
+        }
       }
     }
-  } else {
-    echo "Following build logs..."
-    while (build.object().status.phase == "Running") {
-      build.logs("--tail=1", "--timestamps=true", "-f")
+    buildobj = build.object()
+    if (buildobj.status.phase != "Complete") {
+      script.error "Build ${buildobj.metadata.name} ${buildobj.status.phase}"
     }
+    return build
   }
-  buildobj = build.object()
-  if (buildobj.status.phase != "Complete") {
-    error "Build ${buildobj.metadata.name} ${buildobj.status.phase}"
-  }
-  return build.name()
 }
diff --git a/src/com/redhat/c3i/util/Deployer.groovy b/src/com/redhat/c3i/util/Deployer.groovy
index 6be4006..3560bb3 100644
--- a/src/com/redhat/c3i/util/Deployer.groovy
+++ b/src/com/redhat/c3i/util/Deployer.groovy
@@ -5,24 +5,43 @@ package com.redhat.c3i.util
 
 import java.text.*
 
-def run(models) {
-  openshift.withCluster() {
-    def objects = openshift.apply(models)
+class Deployer implements Serializable {
+  def script
+
+  def run(Map model) {
+    def dc = script.openshift.apply(model)
+    return run(dc)
+  }
+
+  def run(List models) {
+    def objects = script.openshift.apply(models)
     def dcs = objects.narrow('dc')
+    return run(dcs)
+  }
+
+  def run(String dcname) {
+    def dcs = script.openshift.selector(dcname)
+    return run(dcs)
+  }
+
+  def run(dcs) {
     def rm = dcs.rollout()
-    def replicas = 0
-    dcs.withEach {
-      replicas += it.object().spec.replicas
-    }
-    return replicas
+    return dcs
   }
-}
 
-def wait(num, selector) {
-  echo "Waiting for ${num} test pods matching ${selector} to become Ready"
-  openshift.withCluster() {
-    def pods = openshift.selector('pods', selector)
-    timeout(10) {
+  def waitForPods(Integer num, String name, Integer timeout=10) {
+    def pods = script.openshift.selector('pods', name)
+    return waitForPods(num, pods, timeout)
+  }
+
+  def waitForPods(Integer num, Map labels, Integer timeout=10) {
+    def pods = script.openshift.selector('pods', labels)
+    return waitForPods(num, pods, timeout)
+  }
+
+  def waitForPods(Integer num, pods, Integer timeout=10) {
+    script.echo "Waiting for ${num} pods to be ready..."
+    script.timeout(time: timeout) {
       pods.untilEach(num) {
         def pod = it.object()
         if (pod.status.phase in ["New", "Pending", "Unknown"]) {
@@ -31,33 +50,55 @@ def wait(num, selector) {
         if (pod.status.phase == "Running") {
           for (cond in pod.status.conditions) {
             if (cond.type == 'Ready' && cond.status == 'True') {
-              echo "Pod ${pod.metadata.name} is ready"
+              script.echo "Pod ${pod.metadata.name} is ready"
               return true
             }
           }
           return false
         }
-        error("Test pod ${pod.metadata.name} is not running. Current phase is ${pod.status.phase}.")
+        script.error("Pod ${pod.metadata.name} is not running. Current phase is ${pod.status.phase}.")
       }
     }
+    return pods
   }
-}
 
-def runUntilReady(models, selector) {
-  def numpods = run(models)
-  wait(numpods, selector)
-}
+  def wait(String name, Integer timeout=10) {
+    def dc = script.openshift.selector('dc', name)
+    return wait(dc, timeout)
+  }
+
+  def wait(Map labels, Integer timeout=10) {
+    def dc = script.openshift.selector('dc', labels)
+    return wait(dc, timeout)
+  }
+
+  def wait(dcs, Integer timeout=10) {
+    script.timeout(time: timeout) {
+      dcs.withEach {
+        def dc = it.object()
+        def pods = it.related('pod')
+        def replicas = dc.spec.replicas
+        if (replicas < 1) {
+          return
+        }
+        script.echo "Waiting for deployment ${dc.metadata.name} to be ready..."
+        waitForPods(replicas, pods, timeout)
+        script.echo "Deployment ${dc.metadata.name} is ready"
+      }
+    }
+    return dcs
+  }
 
-def cleanup(int age=60, String... apps) {
-  // age is specified in minutes
-  openshift.withCluster() {
+  def Integer cleanup(Integer age=60, String... apps) {
+    // age is specified in minutes
     def df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'")
     df.setTimeZone(TimeZone.getTimeZone("UTC"))
     def oldobjs = []
     for (app in apps) {
-      def selected = openshift.selector("all,pvc,configmap,secret", ["app": app])
+      def selected = script.openshift.selector("all,pvc,configmap,secret", ["app": app])
       oldobjs.addAll(selected.objects())
     }
+    def deleted = 0
     def now = new Date()
     // Delete all objects that are older than 1 hour
     for (obj in oldobjs) {
@@ -66,9 +107,11 @@ def cleanup(int age=60, String... apps) {
       }
       def creationTime = df.parse(obj.metadata.creationTimestamp)
       if (now.getTime() - creationTime.getTime() > (1000 * 60 * age)) {
-        echo "Deleting ${obj.kind} ${obj.metadata.name}..."
-        openshift.delete(obj.kind, obj.metadata.name, "--ignore-not-found=true")
+        script.echo "Deleting ${obj.kind} ${obj.metadata.name}..."
+        script.openshift.delete(obj.kind, obj.metadata.name, "--ignore-not-found=true")
+        deleted += 1
       }
     }
+    return deleted
   }
 }
diff --git a/test/BuilderTest.groovy b/test/BuilderTest.groovy
new file mode 100644
index 0000000..052d2db
--- /dev/null
+++ b/test/BuilderTest.groovy
@@ -0,0 +1,276 @@
+import org.junit.*
+import com.lesfurets.jenkins.unit.cps.BasePipelineTestCPS
+import static groovy.test.GroovyAssert.*
+
+class BuilderTest extends BasePipelineTestCPS {
+    def c3i
+
+    @Before
+    void setUp() {
+        super.setUp()
+        c3i = loadScript('vars/c3i.groovy')
+        binding.setVariable('script', c3i)
+        binding.setVariable('openshift', c3i)
+    }
+
+    @Test
+    void testBuildMapNoWait() {
+        helper.registerAllowedMethod('apply', [Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'bc'],
+        ])})
+        c3i.build(script: c3i, objs: [name: 'model1'], 'arg1', 'arg2')
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(0, helper.methodCallCount('wait'))
+    }
+
+    @Test
+    void testBuildListNoWait() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'test'],
+            [name: 'obj2', kind: 'bc'],
+        ])})
+        c3i.build(script: c3i, objs: ['model1'], 'arg1', 'arg2')
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(2, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(0, helper.methodCallCount('wait'))
+    }
+
+    @Test
+    void testBuildByNameNoWait() {
+        helper.registerAllowedMethod('selector', [String.class], { new SelectorMock(test: this, objs: [
+            [name: 'bc1', kind: 'bc']
+        ])})
+        c3i.build(script: c3i, objs: 'bc/name', 'arg1', 'arg2')
+        assertEquals(0, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['bc/name']
+        })
+        assertEquals(1, helper.methodCallCount('narrow'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'narrow' && call.args == ['build']
+        })
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(0, helper.methodCallCount('wait'))
+    }
+
+    @Test
+    void testBuildSelectorNoWait() {
+        c3i.build(script: c3i, objs: new SelectorMock(test: this, objs: [
+            [name: 'bc1', kind: 'bc']
+        ]), 'arg1', 'arg2')
+        assertEquals(0, helper.methodCallCount('apply'))
+        assertEquals(0, helper.methodCallCount('selector'))
+        assertEquals(1, helper.methodCallCount('narrow'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'narrow' && call.args == ['build']
+        })
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(0, helper.methodCallCount('wait'))
+    }
+
+    @Test
+    void testBuildAndWaitList() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'test'],
+            [name: 'obj2', kind: 'bc'],
+        ])})
+        c3i.buildAndWait(script: c3i, objs: ['model1'], 'arg1', 'arg2')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(2, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 60
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testBuildAndWaitListTimeout() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'test'],
+            [name: 'obj2', kind: 'bc'],
+        ])})
+        c3i.buildAndWait(script: c3i, objs: ['model1'], timeout: 30, 'arg1', 'arg2')
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(2, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 30
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testBuildAndWaitMap() {
+        helper.registerAllowedMethod('apply', [Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'bc'],
+        ])})
+        c3i.buildAndWait(script: c3i, objs: [name: 'model1'], 'arg1', 'arg2')
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2']
+        })
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testBuildAndWaitListPipeline() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'test'],
+            [name: 'obj2', kind: 'bc'],
+        ])})
+        c3i.buildAndWait(script: c3i, objs: ['model1'], 'arg1', 'arg2', 'pipeline')
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(2, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('startBuild'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'startBuild' && call.args == ['arg1', 'arg2', 'pipeline']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Details: https://pipeline.example.com/']
+        })
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertEquals(2, helper.methodCallCount('watch'))
+        assertEquals(1, helper.methodCallCount('logs'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'logs' && call.args == ['--tail=1']
+        })
+    }
+
+    @Test
+    void testWaitForBuildStartFail() {
+        shouldFail {
+            c3i.waitForBuildStart(script: c3i, build: new SelectorMock(test: this, objs: [
+                [name: 'build1', kind: 'build', phase: 'New']
+            ]))
+        }
+    }
+
+    @Test
+    void testWaitForBuildStart() {
+        c3i.waitForBuildStart(script: c3i, build: new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Running']
+        ]))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Waiting for build1 to start...']
+        })
+        assertEquals(1, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 5
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+    }
+
+    @Test
+    void testWaitForBuildStartByName() {
+        helper.registerAllowedMethod('selector', [String.class], { new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Running']
+        ])})
+        c3i.waitForBuildStart(script: c3i, build: 'build/build1')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['build/build1']
+        })
+    }
+
+    @Test
+    void testWaitForBuildStartTimeout() {
+        c3i.waitForBuildStart(script: c3i, build: new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Running']
+        ]), timeout: 10)
+        assertEquals(1, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 10
+        })
+    }
+
+    @Test
+    void testWaitArgsBuildName() {
+        helper.registerAllowedMethod('selector', [String.class], { new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Complete']
+        ])})
+        c3i.waitForBuild(script: c3i, build: 'build/foo')
+        printCallStack()
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 60
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testWaitArgsBuildNameTimeout() {
+        helper.registerAllowedMethod('selector', [String.class], { new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Complete']
+        ])})
+        c3i.waitForBuild(script: c3i, build: 'build/foo', timeout: 30)
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 30
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testWaitArgsBuildSelector() {
+        c3i.waitForBuild(script: c3i, build: new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Complete']
+        ]))
+        assertEquals(0, helper.methodCallCount('selector'))
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 60
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+    @Test
+    void testWaitArgsBuildSelectorTimeout() {
+        c3i.waitForBuild(script: c3i, build: new SelectorMock(test: this, objs: [
+            [name: 'build1', kind: 'build', phase: 'Complete']
+        ]), timeout: 30)
+        assertEquals(0, helper.methodCallCount('selector'))
+        assertEquals(2, helper.methodCallCount('timeout'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 30
+        })
+        assertEquals(1, helper.methodCallCount('watch'))
+        assertEquals(0, helper.methodCallCount('logs'))
+    }
+
+}
diff --git a/test/DeployerTest.groovy b/test/DeployerTest.groovy
new file mode 100644
index 0000000..6cc57e1
--- /dev/null
+++ b/test/DeployerTest.groovy
@@ -0,0 +1,422 @@
+import org.junit.*
+import com.lesfurets.jenkins.unit.cps.BasePipelineTestCPS
+import static groovy.test.GroovyAssert.*
+
+class DeployerTest extends BasePipelineTestCPS {
+    def c3i
+
+    @Before
+    void setUp() {
+        super.setUp()
+        c3i = loadScript('vars/c3i.groovy')
+        binding.setVariable('script', c3i)
+        binding.setVariable('openshift', c3i)
+    }
+
+    @Test
+    void testCleanupNoApps() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this) })
+        c3i.cleanup(script: c3i)
+        printCallStack()
+        assertEquals(0, helper.methodCallCount('selector'))
+    }
+
+    @Test
+    void testCleanupOneAppNoObjs() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this) })
+        c3i.cleanup(script: c3i, 'foo')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertEquals(0, helper.methodCallCount('delete'))
+    }
+
+    @Test
+    void testCleanupIgnoreRecent() {
+        def dateStr = new Date().format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', ts: dateStr]
+        ])})
+        helper.registerAllowedMethod('delete', [String.class, String.class, String.class], null)
+        c3i.cleanup(script: c3i, 'foo')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertEquals(0, helper.methodCallCount('delete'))
+    }
+
+    @Test
+    void testCleanupOneObj() {
+        def dateStr = new Date().minus(1).format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', ts: dateStr]
+        ])})
+        helper.registerAllowedMethod('delete', [String.class, String.class, String.class], null)
+        def cleaned = c3i.cleanup(script: c3i, 'foo')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertEquals(1, helper.methodCallCount('delete'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == "delete" && call.args == ['test', 'obj1', '--ignore-not-found=true']
+        })
+        assertEquals(1, cleaned)
+    }
+
+    @Test
+    void testCleanupMulti() {
+        def dateStr = new Date().minus(1).format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', ts: dateStr]
+        ])})
+        helper.registerAllowedMethod('delete', [String.class, String.class, String.class], null)
+        def cleaned = c3i.cleanup(script: c3i, 'foo', 'bar', 'baz')
+        printCallStack()
+        assertEquals(3, helper.methodCallCount('selector'))
+        assertEquals(3, helper.methodCallCount('delete'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == "delete" && call.args == ['test', 'obj1', '--ignore-not-found=true']
+        })
+        assertEquals(3, cleaned)
+    }
+
+    @Test
+    void testCleanupAge() {
+        def dateStr = new Date().minus(1).format("yyyy-MM-dd'T'HH:mm:ss'Z'", TimeZone.getTimeZone("UTC"))
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', ts: dateStr]
+        ])})
+        helper.registerAllowedMethod('delete', [String.class, String.class, String.class], null)
+        def cleaned = c3i.cleanup(script: c3i, 'foo', age: 60 * 25)
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertEquals(0, helper.methodCallCount('delete'))
+        assertEquals(0, cleaned)
+    }
+
+    @Test
+    void testRunList() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'obj1', kind: 'test', replicas: 1],
+            [name: 'dc2', kind: 'dc', replicas: 2],
+            [name: 'dc3', kind: 'dc', replicas: 4]
+        ])})
+        def dcs = c3i.deploy(script: c3i, objs: [])
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(2, dcs.count())
+    }
+
+    @Test
+    void testRunMap() {
+        helper.registerAllowedMethod('apply', [Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc', replicas: 2],
+        ])})
+        def dcs = c3i.deploy(script: c3i, objs: [:])
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(0, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(1, dcs.count())
+    }
+
+    @Test
+    void testRunByName() {
+        helper.registerAllowedMethod('selector', [String.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc', replicas: 4],
+        ])})
+        def dcs = c3i.deploy(script: c3i, objs: '')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('selector'))
+        assertEquals(0, helper.methodCallCount('apply'))
+        assertEquals(0, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(['dc1'], dcs.names())
+    }
+
+    @Test
+    void testRunDC() {
+        def dcs = c3i.deploy(script: c3i, objs: new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc', replicas: 1],
+            [name: 'dc2', kind: 'dc', replicas: 2],
+        ]))
+        printCallStack()
+        assertEquals(0, helper.methodCallCount('apply'))
+        assertEquals(0, helper.methodCallCount('narrow'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(2, dcs.count())
+        assertEquals(['dc1', 'dc2'], dcs.names())
+    }
+
+    @Test
+    void testWaitForPodsUnknown() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1'],
+            [name: 'pod2']
+        ])})
+        shouldFail {
+            c3i.waitForPods(script: c3i, num: 2, objs: [:])
+        }
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('echo'))
+    }
+
+    @Test
+    void testWaitForPodsCrashed() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1'],
+            [name: 'pod2', phase: 'Crashed']
+        ])})
+        shouldFail {
+            c3i.waitForPods(script: c3i, num: 2, objs: [:])
+        }
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('echo'))
+        assertEquals(1, helper.methodCallCount('error'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'error' && call.args == ['Pod pod2 is not running. Current phase is Crashed.']
+        })
+    }
+
+    @Test
+    void testWaitForPodsPartialReady() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Pending', status: 'True']]]
+        ])})
+        shouldFail {
+            c3i.waitForPods(script: c3i, num: 2, objs: [:])
+        }
+        printCallStack()
+        assertEquals(2, helper.methodCallCount('echo'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod1 is ready']
+        })
+    }
+
+    @Test
+    void testWaitForPodsReady() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ])})
+        c3i.waitForPods(script: c3i, num: 2, objs: [:])
+        printCallStack()
+        assertEquals(3, helper.methodCallCount('echo'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod2 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 10
+        })
+    }
+
+    @Test
+    void testWaitForPodsTimeout() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ])})
+        c3i.waitForPods(script: c3i, num: 2, objs: [:], timeout: 20)
+        printCallStack()
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 20
+        })
+    }
+
+    @Test
+    void testWaitForPodsNoTimeout() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ])})
+        c3i.waitForPods(script: c3i, num: 2, objs: [:])
+        printCallStack()
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 10
+        })
+    }
+
+    @Test
+    void testWaitForPodsString() {
+        helper.registerAllowedMethod('selector', [String.class, String.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ])})
+        c3i.waitForPods(script: c3i, num: 2, objs: 'foo')
+        printCallStack()
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['pods', 'foo']
+        })
+    }
+
+    @Test
+    void testWaitForPodsMap() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ])})
+        c3i.waitForPods(script: c3i, num: 2, objs: [foo: 'bar'])
+        printCallStack()
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['pods', [foo: 'bar']]
+        })
+    }
+
+    @Test
+    void testWaitForPodsSelector() {
+        c3i.waitForPods(script: c3i, num: 2, objs: new SelectorMock(test: this, objs: [
+            [name: 'pod1', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]],
+            [name: 'pod2', phase: 'Running', conditions: [[type: 'Ready', status: 'True']]]
+        ]))
+        printCallStack()
+        assertEquals(0, helper.methodCallCount('selector'))
+    }
+
+    @Test
+    void testWaitForDeploymentString() {
+        helper.registerAllowedMethod('selector', [String.class, String.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc']
+        ])})
+        c3i.waitForDeployment(script: c3i, objs: 'dc1')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['dc', 'dc1']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'untilEach' && call.args[0] == 1
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc1 is ready']
+        })
+    }
+
+    @Test
+    void testWaitForDeploymentMap() {
+        helper.registerAllowedMethod('selector', [String.class, Map.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc']
+        ])})
+        c3i.waitForDeployment(script: c3i, objs: [kind: 'dc'])
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'selector' && call.args == ['dc', [kind: 'dc']]
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'untilEach' && call.args[0] == 1
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc1 is ready']
+        })
+    }
+
+    @Test
+    void testWaitForDeploymentSelector() {
+        c3i.waitForDeployment(script: c3i, objs: new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc']
+        ]))
+        assertEquals(0, helper.methodCallCount('selector'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'related' && call.args == ['pod']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'untilEach' && call.args[0] == 1
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc1 is ready']
+        })
+    }
+
+    @Test
+    void testWaitForDeploymentTimeout() {
+        c3i.waitForDeployment(script: c3i, objs: new SelectorMock(test: this, objs: [
+            [name: 'dc1', kind: 'dc']
+        ]), timeout: 20)
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 20
+        })
+    }
+
+    @Test
+    void testDeployAndWait() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc0', kind: 'dc', replicas: 1],
+            [name: 'test0', kind: 'test', replicas: 2],
+            [name: 'dc1', kind: 'dc', replicas: 3]
+        ])})
+        def dcs = c3i.deployAndWait(script: c3i, objs: [])
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(2, dcs.count())
+        assertEquals(['dc0', 'dc1'], dcs.names())
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Deployment dc0 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Deployment dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc0 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod1-from-dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod2-from-dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 10
+        })
+    }
+
+    @Test
+    void testDeployAndWaitTimeout() {
+        helper.registerAllowedMethod('apply', [List.class], { new SelectorMock(test: this, objs: [
+            [name: 'dc0', kind: 'dc', replicas: 2],
+            [name: 'dc1', kind: 'dc', replicas: 1],
+            [name: 'test0', kind: 'test', replicas: 4]
+        ])})
+        def dcs = c3i.deployAndWait(script: c3i, objs: [], timeout: 20)
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('apply'))
+        assertEquals(1, helper.methodCallCount('rollout'))
+        assertEquals(2, dcs.count())
+        assertEquals(['dc0', 'dc1'], dcs.names())
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Deployment dc0 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Deployment dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc0 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod1-from-dc0 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'echo' && call.args == ['Pod pod0-from-dc1 is ready']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'timeout' && call.args[0].time == 20
+        })
+    }
+}
diff --git a/test/SelectorMock.groovy b/test/SelectorMock.groovy
new file mode 100644
index 0000000..33311c9
--- /dev/null
+++ b/test/SelectorMock.groovy
@@ -0,0 +1,117 @@
+import com.lesfurets.jenkins.unit.BasePipelineTest
+import static groovy.test.GroovyAssert.*
+
+class SelectorMock {
+    BasePipelineTest test
+    List objs
+
+    def objects() {
+        return objs.collect {
+            [
+               kind: it.kind ?: 'test',
+               metadata: [
+                            creationTimestamp: it.ts ?: '2019-09-01T00:00:00Z',
+                            name: it.name,
+                            annotations: it.strategyType == 'JenkinsPipeline'
+                              ? ['openshift.io/jenkins-build-uri': 'https://pipeline.example.com/']
+                              : null
+                         ],
+               spec: [
+                        replicas: it.replicas ?: 1,
+                        strategy: [
+                                     type: it.strategyType ?: ''
+                                  ]
+                     ],
+               status: [
+                          phase: it.phase ?: 'Unknown',
+                          conditions: it.conditions ?: []
+                       ]
+            ]
+        }
+    }
+
+    def getDepth() {
+        return Thread.currentThread().stackTrace.findAll {
+            it.className in [this.class.name, test.class.name]
+        }.size()
+    }
+
+    def object() {
+        test.helper.registerMethodCall(this, getDepth(), 'object')
+        return objects()[0]
+    }
+
+    def rollout() {
+        test.helper.registerMethodCall(this, getDepth(), 'rollout')
+        return this
+    }
+
+    def narrow(kind) {
+        test.helper.registerMethodCall(this, getDepth(), 'narrow', kind)
+        return new SelectorMock(test: test, objs: objs.findAll { it.kind == kind })
+    }
+
+    def withEach(Closure body) {
+        test.helper.registerMethodCall(this, getDepth(), 'withEach', body)
+        objs.each {
+            body(new SelectorMock(test: test, objs: [it]))
+        }
+    }
+
+    def untilEach(int num, Closure body) {
+        test.helper.registerMethodCall(this, getDepth(), 'untilEach', num, body)
+        assertEquals(num, objs.size())
+        def result = true
+        objs.each {
+            result &= body(new SelectorMock(test: test, objs: [it]))
+        }
+        assertTrue(result)
+    }
+
+    def startBuild(String[] args) {
+        test.helper.registerMethodCall(this, getDepth(), 'startBuild', args)
+        return new SelectorMock(test: test, objs: [
+            [name: 'build1', kind: 'build', phase: 'Complete',
+             strategyType: 'pipeline' in args ? 'JenkinsPipeline' : '']
+        ])
+    }
+
+    def watch(Closure body) {
+        test.helper.registerMethodCall(this, getDepth(), 'watch', body)
+        assertTrue(body(this))
+    }
+
+    def logs(String[] args) {
+        test.helper.registerMethodCall(this, getDepth(), 'logs', args)
+    }
+
+    def count() {
+        return objs.size()
+    }
+
+    def names() {
+        test.helper.registerMethodCall(this, getDepth(), 'names')
+        return objs.collect {
+            it.name
+        }
+    }
+
+    def name() {
+        test.helper.registerMethodCall(this, getDepth(), 'name')
+        return objects()[0].metadata.name
+    }
+
+    def related(String kind) {
+        test.helper.registerMethodCall(this, getDepth(), 'related', kind)
+        def rel = []
+        objs.each { obj ->
+            (obj.replicas ?: 1).times {
+                rel.add([
+                    kind: kind, name: "${kind}${it}-from-${obj.name}", phase: 'Running',
+                    conditions: [[type: 'Ready', status: 'True']]
+                ])
+            }
+        }
+        return new SelectorMock(test: test, objs: rel)
+    }
+}
diff --git a/vars/c3i.groovy b/vars/c3i.groovy
index 6fbbd4c..acfaa47 100644
--- a/vars/c3i.groovy
+++ b/vars/c3i.groovy
@@ -1,26 +1,161 @@
 // Useful functions for deploying and managing resources
 // Mike Bonnet (mikeb@redhat.com), 2019-01-04
 
-def cleanup(String... apps) {
-  stage("Cleanup old resources") {
-    def deployer = new com.redhat.c3i.util.Deployer()
-    deployer.cleanup(apps)
+/**
+ * Cleanup objects from previous pipeline runs.
+ * @param args.script The script calling the method.
+ * @param args.age Objects created more than this number of minutes ago will
+ *        be candidates for cleanup. Defaults to 60.
+ * @param apps One or more {@code String} app labels. Any objects with a matching
+ *        "app:" label will be a candidate for cleanup.
+ * @return The number of objects deleted.
+ */
+def cleanup(Map args, String... apps) {
+  def deployer = new com.redhat.c3i.util.Deployer(script: args.script)
+  return deployer.cleanup(args.age ?: 60, apps as String[])
+}
+
+/**
+ * Run a build.
+ * @param args.script The script calling the method.
+ * @param args.objs A {@code Map} modeling a single OpenShift {@code BuildConfig} object,
+ *        or a {@code List} modeling a number of OpenShift objects, at least one of which
+ *        is a {@code BuildConfig} object, or a {@code String} that identifies a single
+ *        {@code BuildConfig} object in "bc/name" format, or a {@code Selector} that identifies
+ *        a {@code BuildConfig}.
+ * @param params A {@code List} of {@code String} parameters to be passed to {@code startBuild()}.
+ * @return A {@code Selector} identifying the build.
+ */
+def build(Map args, String... params) {
+  def builder = new com.redhat.c3i.util.Builder(script: args.script)
+  return builder.build(args.objs, params)
+}
+
+/**
+ * Run a build and wait for it to complete.
+ * @param args.script The script calling the method.
+ * @param args.objs A {@code Map} modeling a single OpenShift {@code BuildConfig} object,
+ *        or a {@code List} modeling a number of OpenShift objects, at least one of which
+ *        is a {@code BuildConfig} object, or a {@code String} that identifies a single
+ *        {@code BuildConfig} object in "bc/name" format, or a {@code Selector} that identifies
+ *        a {@code BuildConfig}.
+ * @param args.timeout The length of time to wait for the build to complete, in minutes. If not
+ *        specified, defaults to 60.
+ * @param params A {@code List} of {@code String} parameters to be passed to {@code startBuild()}.
+ * @return A {@code Selector} identifying the build.
+ */
+def buildAndWait(Map args, String... params) {
+  def builder = new com.redhat.c3i.util.Builder(script: args.script)
+  def build = builder.build(args.objs, params)
+  return builder.wait(build, args.timeout ?: 60)
+}
+
+/**
+ * Wait for a build to complete.
+ * @deprecated Use {@code waitForBuild(Map)} instead.
+ * @param buildsel {@code String} name of the build, in "build/name-N" format.
+ * @return String name of the build that was waited for.
+ */
+def wait(String buildsel) {
+  openshift.withCluster() {
+    def build = waitForBuild(script: this, build: buildsel)
+    return build.name()
   }
 }
 
-def build(modelsOrName, Object... args) {
-  def builder = new com.redhat.c3i.util.Builder()
-  def buildname = builder.build(modelsOrName, false, args)
-  return buildname
+/**
+ * Wait for a build to complete.
+ * @param args.script The script calling the method.
+ * @param args.build A OpenShift {@code Selector} that represents a single build, or a
+ *        {@code String} containing the name of a single build in progress,
+ *        in "build/name-N" format.
+ * @param args.timeout The length of time to wait for the build to complete, in minutes.
+ *        If not specified, defaults to 60.
+ * @return A OpenShift {@code Selector} that represents the build that was waited for.
+ */
+def waitForBuild(Map args) {
+  def builder = new com.redhat.c3i.util.Builder(script: args.script)
+  return builder.wait(args.build, args.timeout ?: 60)
+}
+
+/**
+ * Wait for a build to start (to no longer be in the "New", "Pending", or "Unknown" states).
+ * @param args.script The script calling the method.
+ * @param args.build A OpenShift {@code Selector} that represents a single build, or a
+ *        {@code String} containing the name of a single build in progress, in "build/name-N"
+ *        format.
+ * @param args.timeout The length of time to wait in minutes. Defaults to 5.
+ * @return A OpenShift {@code Selector} that represents the build that was waited for.
+ */
+def waitForBuildStart(Map args) {
+  def builder = new com.redhat.c3i.util.Builder(script: args.script)
+  return builder.waitForStart(args.build, args.timeout ?: 5)
+}
+
+/**
+ * Deploy a set of OpenShift objects.
+ * @param args.script The script calling the method.
+ * @param args.objs A {@code Map} modeling a single OpenShift {@code DeploymentConfig} object,
+ *        or a {@code List} modeling a number of OpenShift objects, at least one of which
+ *        is a {@code DeploymentConfig} object, or a {@code String} which identifies a
+ *        {@code DeploymentConfig} in "dc/name" format, or a {@code Selector} that matches
+ *        a set of {@code DeploymentConfig} objects.
+ * @return A OpenShift {@code Selector} that represents the set of {@code DeploymentConfig}s
+ *         that were rolled out.
+ */
+def deploy(Map args) {
+  def deployer = new com.redhat.c3i.util.Deployer(script: args.script)
+  return deployer.run(args.objs)
+}
+
+/**
+ * Wait for pods to be Ready.
+ * @param args.script The script calling the method.
+ * @param args.num The number of pods to wait for.
+ * @param args.objs A {@code Map} that contains the label:value pairs used match the pods being
+ *        deployed, or a {@code String} identifying the name of a pod being deployed, or a
+ *        {@code Selector} that matches the pods being deployed.
+ * @param args.timeout The length of time to wait for the pods to become Ready, in minutes.
+ *        If not specified, defaults to 10.
+ * @return A OpenShift {@code Selector} that represents the set of pods that were waited on.
+ */
+def waitForPods(Map args) {
+  def deployer = new com.redhat.c3i.util.Deployer(script: args.script)
+  return deployer.waitForPods(args.num, args.objs, args.timeout ?: 10)
 }
 
-def buildAndWait(modelsOrName, Object... args) {
-  def builder = new com.redhat.c3i.util.Builder()
-  def buildname = builder.build(modelsOrName, true, args)
-  return buildname
+/**
+ * Wait for a deployment to complete.
+ * @param args.script The script calling the method.
+ * @param args.objs A {@code Map} that contains the label:value pairs used match the
+ *        {@code DeploymentConfig}s being deployed, or a {@code String} identifying the name of
+ *        the {@code DeploymentConfig} being deployed, or a {@code Selector} that matches a set
+ *        of {@code DeploymentConfig} objects.
+ * @param args.timeout The length of time to wait for the deployment to complete, in minutes.
+ *        If not specified, defaults to 10.
+ * @return A OpenShift {@code Selector} that represents the set of {@code DeploymentConfig}s that
+ *         were waited on.
+ */
+def waitForDeployment(Map args) {
+  def deployer = new com.redhat.c3i.util.Deployer(script: args.script)
+  return deployer.wait(args.objs, args.timeout ?: 10)
 }
 
-def wait(buildsel) {
-  def builder = new com.redhat.c3i.util.Builder()
-  return builder.wait(buildsel)
+/**
+ * Deploy a set of OpenShift objects and wait for them to be ready.
+ * @param args.script The script calling the method.
+ * @param args.objs A {@code Map} modeling a single OpenShift {@code DeploymentConfig} object,
+ *        or a {@code List} modeling a number of OpenShift objects, at least one of which
+ *        is a {@code DeploymentConfig} object, or a {@code String} which identifies a
+ *        {@code DeploymentConfig} in "dc/name" format, or a {@code Selector} that matches
+ *        a set of {@code DeploymentConfig} objects.
+ * @param args.timeout The length of time to wait for the deployment to complete, in minutes.
+ *        If not specified, defaults to 10.
+ * @return A OpenShift {@code Selector} that represents the set of {@code DeploymentConfig}s that
+ *         were waited on.
+ */
+def deployAndWait(Map args) {
+  def deployer = new com.redhat.c3i.util.Deployer(script: args.script)
+  def dcs = deployer.run(args.objs)
+  return deployer.wait(dcs, args.timeout ?: 10)
 }
diff --git a/vars/ca.groovy b/vars/ca.groovy
index 2dd8c17..66b2511 100644
--- a/vars/ca.groovy
+++ b/vars/ca.groovy
@@ -1,16 +1,14 @@
 def gen_ca() {
-    stage('Generate certificate authority') {
-        def conftmpl = libraryResource 'ca/ssl.cnf.in'
-        def conf = conftmpl.replace('${servername}', 'Test Certificate Authority')
-        conf = conf.replace('${commonName}', 'Test Certificate Authority')
-        writeFile file: 'ssl.cnf', text: conf
-        writeFile file: 'ca/index.txt', text: ''
-        writeFile file: 'ca/index.txt.attr', text: ''
-        writeFile file: 'ca/serial', text: '01\n'
-        sh 'openssl genrsa -out ca/ca-key.pem'
-        sh 'openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem'
-        stash name: 'ca', includes: 'ca/**'
-    }
+    def conftmpl = libraryResource 'ca/ssl.cnf.in'
+    def conf = conftmpl.replace('${servername}', 'Test Certificate Authority')
+    conf = conf.replace('${commonName}', 'Test Certificate Authority')
+    writeFile file: 'ssl.cnf', text: conf
+    writeFile file: 'ca/index.txt', text: ''
+    writeFile file: 'ca/index.txt.attr', text: ''
+    writeFile file: 'ca/serial', text: '01\n'
+    sh 'openssl genrsa -out ca/ca-key.pem'
+    sh 'openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem'
+    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_ca() {
@@ -28,18 +26,16 @@ def get_ca_cert() {
 }
 
 def gen_ssl_cert(servername) {
-    stage("Generate SSL cert for ${servername}") {
-        get_ca()
-        def conftmpl = libraryResource 'ca/ssl.cnf.in'
-        def conf = conftmpl.replace('${servername}', servername)
-        conf = conf.replace('${commonName}', servername.tokenize('.')[0])
-        writeFile file: 'ssl.cnf', text: conf
-        sh """openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes \
-                -keyout ca/${servername}-key.pem -out ca/${servername}-req.pem"""
-        sh """openssl ca -config ssl.cnf -batch -in ca/${servername}-req.pem \
-                -notext -out ca/${servername}-cert.pem"""
-        stash name: 'ca', includes: 'ca/**'
-    }
+    get_ca()
+    def conftmpl = libraryResource 'ca/ssl.cnf.in'
+    def conf = conftmpl.replace('${servername}', servername)
+    conf = conf.replace('${commonName}', servername.tokenize('.')[0])
+    writeFile file: 'ssl.cnf', text: conf
+    sh """openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes \
+            -keyout ca/${servername}-key.pem -out ca/${servername}-req.pem"""
+    sh """openssl ca -config ssl.cnf -batch -in ca/${servername}-req.pem \
+            -notext -out ca/${servername}-cert.pem"""
+    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_ssl_cert(servername) {
@@ -52,42 +48,35 @@ def get_ssl_cert(servername) {
     return ['cert': cert, 'key': key]
 }
 
-def _gen_keystore(servername, passwd) {
-    stage("Generate keystore for ${servername}") {
-        data = get_ssl_cert(servername)
-        writeFile file: "ca/${servername}.pem", text: data['cert'] + data['key']
-        sh """openssl pkcs12 -export -in ca/${servername}.pem -out ca/${servername}.ks \
-                -name ${servername} -passout pass:${passwd}"""
-        stash name: 'ca', includes: 'ca/**'
-    }
+def gen_keystore(servername, passwd) {
+    data = get_ssl_cert(servername)
+    writeFile file: "ca/${servername}.pem", text: data['cert'] + data['key']
+    sh """openssl pkcs12 -export -in ca/${servername}.pem -out ca/${servername}.ks \
+            -name ${servername} -passout pass:${passwd}"""
+    stash name: 'ca', includes: 'ca/**'
 }
+
 def get_keystore(servername, passwd) {
     get_ca()
     if (!fileExists("ca/${servername}.ks")) {
-        _gen_keystore(servername, passwd)
+        gen_keystore(servername, passwd)
     }
-    // convert to readFile(encoding: 'Base64') when using workflow-basic-steps plugin >= 2.8.1
-    // return readFile(file: "ca/${servername}.ks", encoding: 'Base64')
-    return sh(script: "base64 -w0 ca/${servername}.ks", returnStdout: true).trim()
+    return readFile(file: "ca/${servername}.ks", encoding: 'Base64')
 }
 
-def _gen_truststore(passwd) {
+def gen_truststore(passwd) {
     // Java 7 doesn't support PKCS12 truststores.
     // Switch this to using PKCS12 when everything is running on Java 8+
-    stage("Generate truststore") {
-        sh """keytool -importcert -file ca/ca-cert.pem -alias testca \
-                -keystore ca/truststore.ts -storetype jks \
-                -storepass ${passwd} -trustcacerts -noprompt"""
-        stash name: 'ca', includes: 'ca/**'
-    }
+    sh """keytool -importcert -file ca/ca-cert.pem -alias testca \
+            -keystore ca/truststore.ts -storetype jks \
+            -storepass ${passwd} -trustcacerts -noprompt"""
+    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_truststore(passwd) {
     get_ca()
     if (!fileExists('ca/truststore.ts')) {
-        _gen_truststore(passwd)
+        gen_truststore(passwd)
     }
-    // convert to readFile(encoding: 'Base64') when using workflow-basic-steps plugin >= 2.8.1
-    // return readFile(file: 'ca/truststore.ts', encoding: 'Base64')
-    return sh(script: 'base64 -w0 ca/truststore.ts', returnStdout: true).trim()
+    return readFile(file: 'ca/truststore.ts', encoding: 'Base64')
 }
diff --git a/vars/koji.groovy b/vars/koji.groovy
index 4b0cf43..9c3c3df 100644
--- a/vars/koji.groovy
+++ b/vars/koji.groovy
@@ -8,26 +8,42 @@ import groovy.transform.Field
 @Field _kojicmd = "koji -q -c ${_confdir}/config -p ${_confname}"
 @Field _config = null
 
-def deploy(test_id, hubca, hubcert, msgurl, msgcert, admin_user,
-           hub_image="quay.io/factory2/koji:latest") {
-  stage("Deploy Koji") {
-    openshift.withCluster() {
-      def yaml = libraryResource "openshift/templates/koji.yaml"
-      def template = readYaml text: yaml
-      def models = openshift.process(template,
-        '-p', "TEST_ID=${test_id}",
-        '-p', "KOJI_CA_CERT=" + hubca.cert.bytes.encodeBase64().toString(),
-        '-p', "KOJI_HUB_CERT=" + hubcert.cert.bytes.encodeBase64().toString(),
-        '-p', "KOJI_HUB_KEY=" + hubcert.key.bytes.encodeBase64().toString(),
-        '-p', "KOJI_MESSAGING_URL=${msgurl}",
-        '-p', "KOJI_MESSAGING_CERT_AND_KEY=" + (msgcert.cert + msgcert.key).bytes.encodeBase64().toString(),
-        '-p', "KOJI_ADMIN_USER=${admin_user}",
-        '-p', "KOJI_HUB_IMAGE=${hub_image}",
-        )
-      def deployer = new com.redhat.c3i.util.Deployer()
-      deployer.runUntilReady(models, ["app": "koji", "environment": "test-${test_id}"])
-    }
+/**
+ * Deploy a Koji instance suitable for testing.
+ * @param args.script The script calling the method.
+ * @param args.test_id A unique {@code String} used to identify this instance.
+ * @param args.hubca A {@code Map} containing certificate data for the CA certificate. The "cert" entry
+ *        must contain the certificate in text (PEM) format.
+ * @param args.hubcert A {@code Map} containing certificate data for the HTTPS certificate. The "cert" entry
+ *        must contain the certificate in text (PEM) format. The "key" entry must contain the
+ *        private key in text (PEM) format.
+ * @param args.brokerurl The URL to the ActiveMQ messaging broker.
+ * @param args.brokercert A {@code Map} containing the certificate data for the broker client certificate.
+ *        The "cert" entry must contain the certificate in text (PEM) format. The "key" entry must contain
+ *        the private key in text (PEM) format.
+ * @param args.admin_user The name of the user to be created and configured as the Koji admin.
+ * @param args.hub_image The pull spec of the Koji container image to use.
+ * @return An OpenShift selector representing the DeploymentConfigs rolled out.
+ */
+def deploy(Map args) {
+  if (!args.hub_image) {
+    args.hub_image = 'quay.io/factory2/koji:latest'
   }
+  def yaml = libraryResource "openshift/templates/koji.yaml"
+  def template = readYaml text: yaml
+  def models = args.script.openshift.process(template,
+    '-p', "TEST_ID=${args.test_id}",
+    '-p', "KOJI_CA_CERT=" + args.hubca.cert.bytes.encodeBase64().toString(),
+    '-p', "KOJI_HUB_CERT=" + args.hubcert.cert.bytes.encodeBase64().toString(),
+    '-p', "KOJI_HUB_KEY=" + args.hubcert.key.bytes.encodeBase64().toString(),
+    '-p', "KOJI_MESSAGING_URL=${args.brokerurl}",
+    '-p', "KOJI_MESSAGING_CERT_AND_KEY=" + (args.brokercert.cert + args.brokercert.key).bytes.encodeBase64().toString(),
+    '-p', "KOJI_ADMIN_USER=${args.admin_user}",
+    '-p', "KOJI_HUB_IMAGE=${args.hub_image}",
+    '-l', 'c3i.redhat.com/app=koji',
+    '-l', "c3i.redhat.com/test=${args.test_id}",
+  )
+  return c3i.deploy(script: args.script, objs: models)
 }
 
 def _writeConfig() {
diff --git a/vars/mbs.groovy b/vars/mbs.groovy
index 270199d..f2a2bc9 100644
--- a/vars/mbs.groovy
+++ b/vars/mbs.groovy
@@ -1,30 +1,60 @@
 // Functions to deploy a containerized MBS
 // Mike Bonnet (mikeb@redhat.com), 2019-01-07
 
-def deploy(test_id, kojicert, kojica, msgcert, frontendcert, frontendca, cacerts, kojiurl, stompuri,
-           backend_image="quay.io/factory2/mbs-backend:latest",
-           frontend_image="quay.io/factory2/mbs-frontend:latest") {
-  stage("Deploy MBS") {
-    openshift.withCluster() {
-      def yaml = libraryResource "openshift/templates/mbs.yaml"
-      def template = readYaml text: yaml
-      def models = openshift.process(template,
-        '-p', "TEST_ID=${test_id}",
-        '-p', "KOJI_CERT=" + (kojicert.cert + kojicert.key).bytes.encodeBase64().toString(),
-        '-p', "KOJI_SERVERCA=" + kojica.cert.bytes.encodeBase64().toString(),
-        '-p', "MESSAGING_CERT=" + msgcert.cert.bytes.encodeBase64().toString(),
-        '-p', "MESSAGING_KEY=" + msgcert.key.bytes.encodeBase64().toString(),
-        '-p', "FRONTEND_CERT=" + frontendcert.cert.bytes.encodeBase64().toString(),
-        '-p', "FRONTEND_KEY=" + frontendcert.key.bytes.encodeBase64().toString(),
-        '-p', "FRONTEND_CA=" + frontendca.cert.bytes.encodeBase64().toString(),
-        '-p', "CA_CERTS=" + cacerts.bytes.encodeBase64().toString(),
-        '-p', "KOJI_URL=${kojiurl}",
-        '-p', "STOMP_URI=${stompuri}",
-        '-p', "MBS_BACKEND_IMAGE=${backend_image}",
-        '-p', "MBS_FRONTEND_IMAGE=${frontend_image}",
-        )
-      def deployer = new com.redhat.c3i.util.Deployer()
-      deployer.runUntilReady(models, ["app": "mbs", "environment": "test-${test_id}"])
-    }
+/**
+ * Deploy a MBS instance suitable for testing.
+ * @param args.script The script calling the method.
+ * @param args.test_id A unique {@code String} used to identify this instance.
+ * @param args.kojicert A {@code Map} containing client certificate data for authenticating to Koji.
+ *        The "cert" entry must contain the certificate in text (PEM) format. The "key" entry
+ *        must contain the private key in text (PEM) format.
+ * @param args.kojica A {@code Map} containing certificate data for the CA certificate that
+ *        issued the Koji client certificate. The "cert" entry must contain the certificate in
+ *        text (PEM) format.
+ * @param args.brokercert A {@code Map} containing client certificate data for authenticating to the
+ *        ActiveMQ messaging broker. The "cert" entry must contain the certificate in text (PEM)
+ *        format. The "key" entry must contain the private key in text (PEM) format.
+ * @param args.frontendcert A {@code Map} containing certificate data for the HTTPS certificate for
+ *        the MBS frontend. The "cert" entry must contain the certificate in text (PEM) format.
+ *        The "key" entry must contain the private key in text (PEM) format.
+ * @param args.frontendca A {@code Map} containing certificate data for the CA certificate that
+ *        issued the frontend certificate. The "cert" entry must contain the certificate in
+ *        text (PEM) format.
+ * @param args.cacerts A {@code Map} containing certificate data for the CA certificates that
+ *        should be trusted by MBS. The "cert" entry must contain the certificates in text (PEM)
+ *        format.
+ * @param args.kojiurl The URL to the Koji instance.
+ * @param args.stompuri The URI used to connect to the ActiveMQ broker via the STOMP protocol.
+ * @param args.backend_image The pull spec of the MBS backend container image to use.
+ * @param args.frontend_image The pull spec of the MBS frontend container image to use.
+ * @return An OpenShift selector representing the DeploymentConfigs rolled out.
+ * @return The number of pods deployed.
+ */
+def deploy(Map args) {
+  if (!args.backend_image) {
+    args.backend_image = 'quay.io/factory2/mbs-backend:latest'
   }
+  if (!args.frontend_image) {
+    args.frontend_image = 'quay.io/factory2/mbs-frontend:latest'
+  }
+  def yaml = libraryResource "openshift/templates/mbs.yaml"
+  def template = readYaml text: yaml
+  def models = args.script.openshift.process(template,
+    '-p', "TEST_ID=${args.test_id}",
+    '-p', "KOJI_CERT=" + (args.kojicert.cert + args.kojicert.key).bytes.encodeBase64().toString(),
+    '-p', "KOJI_SERVERCA=" + args.kojica.cert.bytes.encodeBase64().toString(),
+    '-p', "MESSAGING_CERT=" + args.brokercert.cert.bytes.encodeBase64().toString(),
+    '-p', "MESSAGING_KEY=" + args.brokercert.key.bytes.encodeBase64().toString(),
+    '-p', "FRONTEND_CERT=" + args.frontendcert.cert.bytes.encodeBase64().toString(),
+    '-p', "FRONTEND_KEY=" + args.frontendcert.key.bytes.encodeBase64().toString(),
+    '-p', "FRONTEND_CA=" + args.frontendca.cert.bytes.encodeBase64().toString(),
+    '-p', "CA_CERTS=" + args.cacerts.bytes.encodeBase64().toString(),
+    '-p', "KOJI_URL=${args.kojiurl}",
+    '-p', "STOMP_URI=${args.stompuri}",
+    '-p', "MBS_BACKEND_IMAGE=${args.backend_image}",
+    '-p', "MBS_FRONTEND_IMAGE=${args.frontend_image}",
+    '-l', 'c3i.redhat.com/app=mbs',
+    '-l', "c3i.redhat.com/test=${args.test_id}",
+  )
+  return c3i.deploy(script: args.script, objs: models)
 }
diff --git a/vars/umb.groovy b/vars/umb.groovy
index f79039e..e466857 100644
--- a/vars/umb.groovy
+++ b/vars/umb.groovy
@@ -1,22 +1,32 @@
 // Functions to deploy a containerized UMB instance.
 // Mike Bonnet (mikeb@redhat.com), 2019-01-04
 
-def deploy(test_id, keystore_data, keystore_password, truststore_data, truststore_password,
-           broker_image="docker-registry.engineering.redhat.com/factory2/umb:latest") {
-  stage("Deploy UMB") {
-    openshift.withCluster() {
-      def yaml = libraryResource "openshift/templates/umb.yaml"
-      def template = readYaml text: yaml
-      def models = openshift.process(template,
-        '-p', "TEST_ID=${test_id}",
-        '-p', "BROKER_KEYSTORE_DATA=${keystore_data}",
-        '-p', "BROKER_KEYSTORE_PASSWORD=${keystore_password}",
-        '-p', "BROKER_TRUSTSTORE_DATA=${truststore_data}",
-        '-p', "BROKER_TRUSTSTORE_PASSWORD=${truststore_password}",
-        '-p', "UMB_IMAGE=${broker_image}",
-        )
-      def deployer = new com.redhat.c3i.util.Deployer()
-      deployer.runUntilReady(models, ["app": "umb", "environment": "test-${test_id}"])
-    }
+/**
+ * Deploy a UMB instance suitable for testing.
+ * @param args.script The script calling the method.
+ * @param args.test_id A unique {@code String} used to identify this instance.
+ * @param args.keystore_data A Base64-encoded Java keystore.
+ * @param args.keystore_password The password to the keystore encoded in {@code keystore_data}.
+ * @param args.truststore_data A Base64-encoded Java truststore.
+ * @param args.truststore_password The password to the truststore encoded in {@code truststore_data}.
+ * @param args.broker_image The pull spec of the UMB container image to use.
+ * @return An OpenShift selector representing the DeploymentConfigs rolled out.
+ */
+def deploy(Map args) {
+  if (!args.broker_image) {
+    args.broker_image = 'docker-registry.engineering.redhat.com/factory2/umb:latest'
   }
+  def yaml = libraryResource "openshift/templates/umb.yaml"
+  def template = readYaml text: yaml
+  def models = args.script.openshift.process(template,
+    '-p', "TEST_ID=${args.test_id}",
+    '-p', "BROKER_KEYSTORE_DATA=${args.keystore_data}",
+    '-p', "BROKER_KEYSTORE_PASSWORD=${args.keystore_password}",
+    '-p', "BROKER_TRUSTSTORE_DATA=${args.truststore_data}",
+    '-p', "BROKER_TRUSTSTORE_PASSWORD=${args.truststore_password}",
+    '-p', "UMB_IMAGE=${args.broker_image}",
+    '-l', 'c3i.redhat.com/app=umb',
+    '-l', "c3i.redhat.com/test=${args.test_id}",
+  )
+  return c3i.deploy(script: args.script, objs: models)
 }

From e40c570a3e4983a0b4433ac5b135ed39f6019d0f Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 2/6] add a c3i.clone() method for cloning a git repo


---

diff --git a/test/CloneTest.groovy b/test/CloneTest.groovy
new file mode 100644
index 0000000..9fbbd99
--- /dev/null
+++ b/test/CloneTest.groovy
@@ -0,0 +1,123 @@
+import org.junit.*
+import com.lesfurets.jenkins.unit.cps.BasePipelineTestCPS
+import static groovy.test.GroovyAssert.*
+
+class CloneTest extends BasePipelineTestCPS {
+    def c3i
+
+    @Before
+    void setUp() {
+        super.setUp()
+        c3i = loadScript('vars/c3i.groovy')
+        binding.setVariable('script', c3i)
+        helper.registerAllowedMethod('retry', [Integer.class, Closure.class], null)
+    }
+
+    @Test
+    void testCloneDefaults() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git')
+        printCallStack()
+        assertEquals(1, helper.methodCallCount('retry'))
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'master']], args[0].branches)
+        def remote = args[0].userRemoteConfigs[0]
+        assertEquals('origin', remote.name)
+        assertEquals('https://example.com/repo.git', remote.url)
+        assertEquals('+refs/heads/master:refs/remotes/origin/master', remote.refspec as String)
+        def exts = args[0].extensions
+        assertEquals('CleanBeforeCheckout', exts[0]['$class'])
+        def opts = exts[1]
+        assertEquals('CloneOption', opts['$class'])
+        assertEquals(true, opts.honorRefspec)
+        assertEquals(true, opts.noTags)
+        assertEquals(true, opts.shallow)
+        assertEquals(10, opts.depth)
+    }
+
+    @Test
+    void testCloneBranch() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', branch: 'testbranch')
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'testbranch']], args[0].branches)
+        def remote = args[0].userRemoteConfigs[0]
+        assertEquals('+refs/heads/testbranch:refs/remotes/origin/testbranch', remote.refspec as String)
+        def opts = args[0].extensions[1]
+        assertEquals(2, opts.depth)
+    }
+
+    @Test
+    void testClonePRBranch() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', branch: 'pull/123/head')
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'pull/123/head']], args[0].branches)
+        def remote = args[0].userRemoteConfigs[0]
+        assertEquals('+refs/pull/123/head:refs/remotes/origin/pull/123/head', remote.refspec as String)
+        def opts = args[0].extensions[1]
+        assertEquals(2, opts.depth)
+    }
+
+    @Test
+    void testCloneRev() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', rev: 'a1b2c3')
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'a1b2c3']], args[0].branches)
+        def remote = args[0].userRemoteConfigs[0]
+        assertEquals('+refs/heads/master:refs/remotes/origin/master', remote.refspec as String)
+    }
+
+    @Test
+    void testCloneBranchRev() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', branch: 'testbranch', rev: 'a1b2c3')
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'a1b2c3']], args[0].branches)
+        def remote = args[0].userRemoteConfigs[0]
+        assertEquals('+refs/heads/testbranch:refs/remotes/origin/testbranch', remote.refspec as String)
+    }
+
+    @Test
+    void testCloneDepth() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', depth: 5)
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'master']], args[0].branches)
+        def opts = args[0].extensions[1]
+        assertEquals(true, opts.shallow)
+        assertEquals(5, opts.depth)
+    }
+
+    @Test
+    void testCloneNotShallow() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', depth: -1)
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'master']], args[0].branches)
+        def opts = args[0].extensions[1]
+        assertEquals(false, opts.shallow)
+        assertEquals(null, opts.depth)
+    }
+
+    @Test
+    void testCloneTags() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', tags: true)
+        assertEquals(1, helper.methodCallCount('checkout'))
+        def args = helper.callStack.find { call -> call.methodName == 'checkout' }.args
+        assertEquals([[name: 'master']], args[0].branches)
+        def opts = args[0].extensions[1]
+        assertEquals(false, opts.noTags)
+    }
+
+    @Test
+    void testCloneRetries() {
+        c3i.clone(script: c3i, repo: 'https://example.com/repo.git', retries: 10)
+        assertEquals(1, helper.methodCallCount('retry'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'retry' && call.args[0] == 10
+        })
+    }
+
+}
diff --git a/vars/c3i.groovy b/vars/c3i.groovy
index acfaa47..7ffb750 100644
--- a/vars/c3i.groovy
+++ b/vars/c3i.groovy
@@ -159,3 +159,53 @@ def deployAndWait(Map args) {
   def dcs = deployer.run(args.objs)
   return deployer.wait(dcs, args.timeout ?: 10)
 }
+
+/**
+ * Clone a git repo.
+ * @param args.repo The URL to the repo to checkout.
+ * @param args.branch The branch to checkout. Defaults to "master".
+ * @param args.rev The specific revision to checkout. Usually a 40-byte hex string identifying
+ *        a commit, but it could be any format accepted by git. If not specified, the HEAD of
+ *        the branch is checked out.
+ * @param args.depth The depth of the checkout. Defaults to 10 for "master", 2 for other branches.
+ *        Specify -1 for a full checkout.
+ * @param args.tags Whether or not to fetch tags. Defaults to false.
+ * @param args.retries The number of times to retry the checkout. Defaults to 5.
+ * @return A {@code Map} of variables set by the SCM plugin.
+ */
+def clone(Map args) {
+  def branch = args.branch ?: 'master'
+  def srcref
+  if (branch.contains('/')) {
+    // Assume they've specified the full path to the ref
+    srcref = branch
+  } else {
+    srcref = "heads/${branch}"
+  }
+  def opts = [
+    $class: 'CloneOption',
+    honorRefspec: true,
+    noTags: !args.tags,
+    shallow: args.depth != -1,
+  ]
+  if (opts.shallow) {
+    def depth = args.depth ?: branch == 'master' ? 10 : 2
+    opts.depth = depth
+  }
+  retry(args.retries ?: 5) {
+    return checkout([$class: 'GitSCM',
+      branches: [[name: args.rev ?: branch]],
+      userRemoteConfigs: [
+        [
+          name: 'origin',
+          url: args.repo,
+          refspec: "+refs/${srcref}:refs/remotes/origin/${branch}",
+        ],
+      ],
+      extensions: [
+        [$class: 'CleanBeforeCheckout'],
+        opts,
+      ],
+    ])
+  }
+}

From 78705fdbbb8d625f0ea6e3b6f3c46ca5b50ba346 Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 3/6] support adding multiple subjectAltNames to a cert


Also add test coverage for the "ca" variable

---

diff --git a/resources/ca/ssl.cnf.in b/resources/ca/ssl.cnf.in
index c6fc8f9..d341028 100644
--- a/resources/ca/ssl.cnf.in
+++ b/resources/ca/ssl.cnf.in
@@ -53,7 +53,7 @@ nsCertType              = server, client, email, objsign
 nsComment               = "Certificate for testing purposes only"
 subjectKeyIdentifier    = hash
 authorityKeyIdentifier  = keyid,issuer
-subjectAltName          = DNS:${servername}
+subjectAltName          = ${sans}
 
 [ v3_req ]
 basicConstraints  = CA:FALSE
diff --git a/test/CATest.groovy b/test/CATest.groovy
new file mode 100644
index 0000000..5b367b3
--- /dev/null
+++ b/test/CATest.groovy
@@ -0,0 +1,201 @@
+import org.junit.*
+import com.lesfurets.jenkins.unit.cps.BasePipelineTestCPS
+import static groovy.test.GroovyAssert.*
+
+class CATest extends BasePipelineTestCPS {
+    def ca
+
+    @Before
+    void setUp() {
+        super.setUp()
+        helper.gse.getGroovyClassLoader().addClasspath('resources')
+        helper.registerAllowedMethod('writeFile', [Map.class], null)
+        helper.registerAllowedMethod('stash', [Map.class], null)
+        ca = loadScript('vars/ca.groovy')
+    }
+
+    @Test
+    void testGenCa() {
+        ca.gen_ca()
+        assertEquals(3, helper.methodCallCount('sh'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem']
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGetCa() {
+        helper.registerAllowedMethod('unstash', [String.class], { raise new RuntimeException('nothing to unstash') })
+        ca.get_ca()
+        assertEquals(3, helper.methodCallCount('sh'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem']
+        })
+        assertEquals(1, helper.methodCallCount('unstash'))
+    }
+
+    @Test
+    void testGetCaAgain() {
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        ca.get_ca()
+        assertEquals(1, helper.methodCallCount('unstash'))
+        assertEquals(0, helper.methodCallCount('sh'))
+        assertEquals(0, helper.methodCallCount('writeFile'))
+        assertEquals(0, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGetCaCert() {
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
+        def result = ca.get_ca_cert()
+        assertEquals(1, helper.methodCallCount('unstash'))
+        assertEquals(1, helper.methodCallCount('readFile'))
+        assertEquals(['cert': 'certdata'], result)
+    }
+
+    @Test
+    void testGenSslCert() {
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        ca.gen_ssl_cert('testcert')
+        assertEquals(1, helper.methodCallCount('unstash'))
+        assertEquals(1, helper.methodCallCount('libraryResource'))
+        assertEquals(1, helper.methodCallCount('writeFile'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'writeFile' &&
+            call.args[0].file == 'ssl.cnf' &&
+            call.args[0].text =~ /(?m)^subjectAltName\s+=\s+DNS:testcert$/
+        })
+        assertEquals(2, helper.methodCallCount('sh'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args[0].split() == [
+                'openssl', 'ca', '-config', 'ssl.cnf', '-batch', '-in', 'ca/testcert-req.pem',
+                '-notext', '-out', 'ca/testcert-cert.pem'
+            ]
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGenSslCertSans() {
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        ca.gen_ssl_cert('testcert', 'san1', 'san2')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'writeFile' &&
+            call.args[0].file == 'ssl.cnf' &&
+            call.args[0].text =~ /(?m)^subjectAltName\s+=\s+DNS:testcert,DNS:san1,DNS:san2$/
+        })
+    }
+
+    @Test
+    void testGetSslCert() {
+        helper.registerAllowedMethod('fileExists', [String.class], { false })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
+        def result = ca.get_ssl_cert('testcert')
+        assertEquals(2, helper.methodCallCount('unstash'))
+        assertEquals(1, helper.methodCallCount('libraryResource'))
+        assertEquals(1, helper.methodCallCount('writeFile'))
+        assertEquals(2, helper.methodCallCount('sh'))
+        assertEquals(1, helper.methodCallCount('stash'))
+        assertEquals(['cert': 'certdata', 'key': 'certdata'], result)
+    }
+
+    @Test
+    void testGetSslCertSans() {
+        helper.registerAllowedMethod('fileExists', [String.class], { false })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        ca.get_ssl_cert('testcert', 'san1', 'san2')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'writeFile' &&
+            call.args[0].file == 'ssl.cnf' &&
+            call.args[0].text =~ /(?m)^subjectAltName\s+=\s+DNS:testcert,DNS:san1,DNS:san2$/
+        })
+    }
+
+    @Test
+    void testGetSslCertExists() {
+        helper.registerAllowedMethod('fileExists', [String.class], { true })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
+        def result = ca.get_ssl_cert('testcert')
+        assertEquals(1, helper.methodCallCount('unstash'))
+        assertEquals(0, helper.methodCallCount('libraryResource'))
+        assertEquals(0, helper.methodCallCount('writeFile'))
+        assertEquals(0, helper.methodCallCount('sh'))
+        assertEquals(0, helper.methodCallCount('stash'))
+        assertEquals(2, helper.methodCallCount('readFile'))
+        assertEquals(['cert': 'certdata', 'key': 'certdata'], result)
+    }
+
+    @Test
+    void testGenKeystore() {
+        helper.registerAllowedMethod('fileExists', [String.class], { true })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
+        ca.gen_keystore('testserver', 'testpass')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'writeFile' &&
+            call.args[0].file == 'ca/testserver.pem'
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args[0].split() == [
+                'openssl', 'pkcs12', '-export',
+                '-in', 'ca/testserver.pem', '-out', 'ca/testserver.ks',
+                '-name', 'testserver', '-passout', 'pass:testpass'
+            ]
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGetKeystore() {
+        helper.registerAllowedMethod('fileExists', [String.class], { true })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [Map.class], { 'certdata' })
+        ca.get_keystore('testserver', 'testpass')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'readFile' &&
+            call.args[0].file == 'ca/testserver.ks' &&
+            call.args[0].encoding == 'Base64'
+        })
+        assertEquals(0, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGenTruststore() {
+        helper.registerAllowedMethod('fileExists', [String.class], { true })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
+        ca.gen_truststore('testpass')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args[0].split() == [
+                'keytool', '-importcert', '-file', 'ca/ca-cert.pem', '-alias', 'testca',
+                '-keystore', 'ca/truststore.ts', '-storetype', 'jks',
+                '-storepass', 'testpass', '-trustcacerts', '-noprompt'
+            ]
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
+    }
+
+    @Test
+    void testGetTruststore() {
+        helper.registerAllowedMethod('fileExists', [String.class], { true })
+        helper.registerAllowedMethod('unstash', [String.class], null)
+        helper.registerAllowedMethod('readFile', [Map.class], { 'certdata' })
+        ca.get_truststore('testpass')
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'readFile' &&
+            call.args[0].file == 'ca/truststore.ts' &&
+            call.args[0].encoding == 'Base64'
+        })
+        assertEquals(0, helper.methodCallCount('stash'))
+    }
+
+}
diff --git a/vars/ca.groovy b/vars/ca.groovy
index 66b2511..9f6b615 100644
--- a/vars/ca.groovy
+++ b/vars/ca.groovy
@@ -1,7 +1,9 @@
 def gen_ca() {
+    sh 'rm -rf ./ca'
     def conftmpl = libraryResource 'ca/ssl.cnf.in'
     def conf = conftmpl.replace('${servername}', 'Test Certificate Authority')
     conf = conf.replace('${commonName}', 'Test Certificate Authority')
+    conf = conf.replace('${sans}', '')
     writeFile file: 'ssl.cnf', text: conf
     writeFile file: 'ca/index.txt', text: ''
     writeFile file: 'ca/index.txt.attr', text: ''
@@ -25,11 +27,16 @@ def get_ca_cert() {
     return ['cert': cert]
 }
 
-def gen_ssl_cert(servername) {
+def gen_ssl_cert(servername, String... sans) {
     get_ca()
+    def sanslist = "DNS:${servername}"
+    for (san in sans) {
+      sanslist += ",DNS:${san}"
+    }
     def conftmpl = libraryResource 'ca/ssl.cnf.in'
     def conf = conftmpl.replace('${servername}', servername)
     conf = conf.replace('${commonName}', servername.tokenize('.')[0])
+    conf = conf.replace('${sans}', sanslist)
     writeFile file: 'ssl.cnf', text: conf
     sh """openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes \
             -keyout ca/${servername}-key.pem -out ca/${servername}-req.pem"""
@@ -38,10 +45,10 @@ def gen_ssl_cert(servername) {
     stash name: 'ca', includes: 'ca/**'
 }
 
-def get_ssl_cert(servername) {
+def get_ssl_cert(servername, String... sans) {
     get_ca()
     if (!fileExists("ca/${servername}-cert.pem")) {
-        gen_ssl_cert(servername)
+        gen_ssl_cert(servername, sans)
     }
     def cert = readFile "ca/${servername}-cert.pem"
     def key = readFile "ca/${servername}-key.pem"
@@ -65,6 +72,7 @@ def get_keystore(servername, passwd) {
 }
 
 def gen_truststore(passwd) {
+    get_ca()
     // Java 7 doesn't support PKCS12 truststores.
     // Switch this to using PKCS12 when everything is running on Java 8+
     sh """keytool -importcert -file ca/ca-cert.pem -alias testca \

From 77ce2e3ac3452c4f0642faf1d82de997e44673ef Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 4/6] use a more descriptive name for the subjectAltName variable in the OpenSSL config template


---

diff --git a/resources/ca/ssl.cnf.in b/resources/ca/ssl.cnf.in
index d341028..39116d9 100644
--- a/resources/ca/ssl.cnf.in
+++ b/resources/ca/ssl.cnf.in
@@ -53,7 +53,7 @@ nsCertType              = server, client, email, objsign
 nsComment               = "Certificate for testing purposes only"
 subjectKeyIdentifier    = hash
 authorityKeyIdentifier  = keyid,issuer
-subjectAltName          = ${sans}
+subjectAltName          = ${subjectAltNames}
 
 [ v3_req ]
 basicConstraints  = CA:FALSE
diff --git a/vars/ca.groovy b/vars/ca.groovy
index 9f6b615..a967a5c 100644
--- a/vars/ca.groovy
+++ b/vars/ca.groovy
@@ -3,7 +3,7 @@ def gen_ca() {
     def conftmpl = libraryResource 'ca/ssl.cnf.in'
     def conf = conftmpl.replace('${servername}', 'Test Certificate Authority')
     conf = conf.replace('${commonName}', 'Test Certificate Authority')
-    conf = conf.replace('${sans}', '')
+    conf = conf.replace('${subjectAltNames}', '')
     writeFile file: 'ssl.cnf', text: conf
     writeFile file: 'ca/index.txt', text: ''
     writeFile file: 'ca/index.txt.attr', text: ''
@@ -36,7 +36,7 @@ def gen_ssl_cert(servername, String... sans) {
     def conftmpl = libraryResource 'ca/ssl.cnf.in'
     def conf = conftmpl.replace('${servername}', servername)
     conf = conf.replace('${commonName}', servername.tokenize('.')[0])
-    conf = conf.replace('${sans}', sanslist)
+    conf = conf.replace('${subjectAltNames}', sanslist)
     writeFile file: 'ssl.cnf', text: conf
     sh """openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes \
             -keyout ca/${servername}-key.pem -out ca/${servername}-req.pem"""

From 36aa0ae3c10a38d67eed8f423a8703e362fe973b Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 5/6] use environment variables to pass arguments through to "sh"


This should make shell injection attacks more difficult.

---

diff --git a/test/CATest.groovy b/test/CATest.groovy
index 5b367b3..73a0a14 100644
--- a/test/CATest.groovy
+++ b/test/CATest.groovy
@@ -11,6 +11,7 @@ class CATest extends BasePipelineTestCPS {
         helper.gse.getGroovyClassLoader().addClasspath('resources')
         helper.registerAllowedMethod('writeFile', [Map.class], null)
         helper.registerAllowedMethod('stash', [Map.class], null)
+        helper.registerAllowedMethod('withEnv', [List.class, Closure.class], null)
         ca = loadScript('vars/ca.groovy')
     }
 
@@ -73,8 +74,15 @@ class CATest extends BasePipelineTestCPS {
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
             call.args[0].split() == [
-                'openssl', 'ca', '-config', 'ssl.cnf', '-batch', '-in', 'ca/testcert-req.pem',
-                '-notext', '-out', 'ca/testcert-cert.pem'
+                'openssl', 'req', '-config', 'ssl.cnf', '-batch', '-new', '-newkey', 'rsa:2048', '-nodes',
+                '-keyout', '"ca/${SERVERNAME}-key.pem"', '-out', '"ca/${SERVERNAME}-req.pem"'
+            ]
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args[0].split() == [
+                'openssl', 'ca', '-config', 'ssl.cnf', '-batch', '-notext',
+                '-in', '"ca/${SERVERNAME}-req.pem"', '-out', '"ca/${SERVERNAME}-cert.pem"'
             ]
         })
         assertEquals(1, helper.methodCallCount('stash'))
@@ -146,8 +154,8 @@ class CATest extends BasePipelineTestCPS {
             call.methodName == 'sh' &&
             call.args[0].split() == [
                 'openssl', 'pkcs12', '-export',
-                '-in', 'ca/testserver.pem', '-out', 'ca/testserver.ks',
-                '-name', 'testserver', '-passout', 'pass:testpass'
+                '-in', '"ca/${SERVERNAME}.pem"', '-out', '"ca/${SERVERNAME}.ks"',
+                '-name', '"${SERVERNAME}"', '-passout', 'env:PASSWD'
             ]
         })
         assertEquals(1, helper.methodCallCount('stash'))
@@ -178,7 +186,7 @@ class CATest extends BasePipelineTestCPS {
             call.args[0].split() == [
                 'keytool', '-importcert', '-file', 'ca/ca-cert.pem', '-alias', 'testca',
                 '-keystore', 'ca/truststore.ts', '-storetype', 'jks',
-                '-storepass', 'testpass', '-trustcacerts', '-noprompt'
+                '-storepass', '"${PASSWD}"', '-trustcacerts', '-noprompt'
             ]
         })
         assertEquals(1, helper.methodCallCount('stash'))
diff --git a/vars/ca.groovy b/vars/ca.groovy
index a967a5c..24b552a 100644
--- a/vars/ca.groovy
+++ b/vars/ca.groovy
@@ -38,10 +38,12 @@ def gen_ssl_cert(servername, String... sans) {
     conf = conf.replace('${commonName}', servername.tokenize('.')[0])
     conf = conf.replace('${subjectAltNames}', sanslist)
     writeFile file: 'ssl.cnf', text: conf
-    sh """openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes \
-            -keyout ca/${servername}-key.pem -out ca/${servername}-req.pem"""
-    sh """openssl ca -config ssl.cnf -batch -in ca/${servername}-req.pem \
-            -notext -out ca/${servername}-cert.pem"""
+    withEnv(["SERVERNAME=${servername}"]) {
+        sh 'openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes' + \
+           ' -keyout "ca/${SERVERNAME}-key.pem" -out "ca/${SERVERNAME}-req.pem"'
+        sh 'openssl ca -config ssl.cnf -batch -notext' + \
+           ' -in "ca/${SERVERNAME}-req.pem" -out "ca/${SERVERNAME}-cert.pem"'
+    }
     stash name: 'ca', includes: 'ca/**'
 }
 
@@ -58,8 +60,10 @@ def get_ssl_cert(servername, String... sans) {
 def gen_keystore(servername, passwd) {
     data = get_ssl_cert(servername)
     writeFile file: "ca/${servername}.pem", text: data['cert'] + data['key']
-    sh """openssl pkcs12 -export -in ca/${servername}.pem -out ca/${servername}.ks \
-            -name ${servername} -passout pass:${passwd}"""
+    withEnv(["SERVERNAME=${servername}", "PASSWD=${passwd}"]) {
+        sh 'openssl pkcs12 -export -in "ca/${SERVERNAME}.pem" -out "ca/${SERVERNAME}.ks"' + \
+           ' -name "${SERVERNAME}" -passout env:PASSWD'
+    }
     stash name: 'ca', includes: 'ca/**'
 }
 
@@ -75,9 +79,11 @@ def gen_truststore(passwd) {
     get_ca()
     // Java 7 doesn't support PKCS12 truststores.
     // Switch this to using PKCS12 when everything is running on Java 8+
-    sh """keytool -importcert -file ca/ca-cert.pem -alias testca \
-            -keystore ca/truststore.ts -storetype jks \
-            -storepass ${passwd} -trustcacerts -noprompt"""
+    withEnv(["PASSWD=${passwd}"]) {
+        sh 'keytool -importcert -file ca/ca-cert.pem -alias testca' + \
+           ' -keystore ca/truststore.ts -storetype jks' + \
+           ' -storepass "${PASSWD}" -trustcacerts -noprompt'
+    }
     stash name: 'ca', includes: 'ca/**'
 }
 

From a174a00ebe5f9ea30531132dbb0a127b684c59d3 Mon Sep 17 00:00:00 2001
From: Mike Bonnet <mikeb@redhat.com>
Date: Oct 16 2019 23:48:11 +0000
Subject: [PATCH 6/6] use a consistent directory for storing CA-related files


Previously, the "ca" variable operated relative to the current working directory. If the
pipeline had changed directory, this could lead to unexpected behavior, and to files being
created at various locations on the filesystem. This change ensures all files are stored in
a single, consistent directory.

---

diff --git a/resources/ca/ssl.cnf.in b/resources/ca/ssl.cnf.in
index 39116d9..2ccedce 100644
--- a/resources/ca/ssl.cnf.in
+++ b/resources/ca/ssl.cnf.in
@@ -2,7 +2,7 @@
 default_ca       = CA_default        # The default ca section
 
 [ CA_default ]
-dir              = $ENV::PWD/ca        # Where everything is kept
+dir              = $ENV::PWD           # Where everything is kept
 certs            = $dir                # Where the issued certs are kept
 database         = $dir/index.txt      # database index file.
 new_certs_dir    = $dir                # default place for new certs.
diff --git a/test/CATest.groovy b/test/CATest.groovy
index 73a0a14..5e974e1 100644
--- a/test/CATest.groovy
+++ b/test/CATest.groovy
@@ -12,34 +12,63 @@ class CATest extends BasePipelineTestCPS {
         helper.registerAllowedMethod('writeFile', [Map.class], null)
         helper.registerAllowedMethod('stash', [Map.class], null)
         helper.registerAllowedMethod('withEnv', [List.class, Closure.class], null)
+        helper.registerAllowedMethod('deleteDir', [], null)
         ca = loadScript('vars/ca.groovy')
+        binding.setVariable('env', [WORKSPACE: '/workspace', BUILD_NUMBER: 123])
     }
 
     @Test
     void testGenCa() {
         ca.gen_ca()
-        assertEquals(3, helper.methodCallCount('sh'))
+        assertEquals(1, helper.methodCallCount('deleteDir'))
+        assertEquals(2, helper.methodCallCount('sh'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl genrsa -out ca-key.pem']
+        })
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
-            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem']
+            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca-key.pem -out ca-cert.pem']
         })
         assertEquals(1, helper.methodCallCount('stash'))
+        assertEquals('/workspace/.ca-123', ca.env.CA_PATH as String)
+    }
+
+    @Test
+    void testGenCaPath() {
+        ca.gen_ca('/test/ca')
+        assertEquals(1, helper.methodCallCount('deleteDir'))
+        assertEquals(2, helper.methodCallCount('sh'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl genrsa -out ca-key.pem']
+        })
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca-key.pem -out ca-cert.pem']
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
+        assertEquals('/test/ca', ca.env.CA_PATH)
     }
 
     @Test
     void testGetCa() {
-        helper.registerAllowedMethod('unstash', [String.class], { raise new RuntimeException('nothing to unstash') })
         ca.get_ca()
-        assertEquals(3, helper.methodCallCount('sh'))
+        assertEquals(2, helper.methodCallCount('sh'))
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
-            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem']
+            call.args == ['openssl genrsa -out ca-key.pem']
         })
-        assertEquals(1, helper.methodCallCount('unstash'))
+        assertTrue(helper.callStack.any { call ->
+            call.methodName == 'sh' &&
+            call.args == ['openssl req -config ssl.cnf -batch -new -x509 -key ca-key.pem -out ca-cert.pem']
+        })
+        assertEquals(1, helper.methodCallCount('stash'))
     }
 
     @Test
     void testGetCaAgain() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('unstash', [String.class], null)
         ca.get_ca()
         assertEquals(1, helper.methodCallCount('unstash'))
@@ -50,6 +79,7 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGetCaCert() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
         def result = ca.get_ca_cert()
@@ -60,6 +90,7 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGenSslCert() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('unstash', [String.class], null)
         ca.gen_ssl_cert('testcert')
         assertEquals(1, helper.methodCallCount('unstash'))
@@ -75,14 +106,14 @@ class CATest extends BasePipelineTestCPS {
             call.methodName == 'sh' &&
             call.args[0].split() == [
                 'openssl', 'req', '-config', 'ssl.cnf', '-batch', '-new', '-newkey', 'rsa:2048', '-nodes',
-                '-keyout', '"ca/${SERVERNAME}-key.pem"', '-out', '"ca/${SERVERNAME}-req.pem"'
+                '-keyout', '"${SERVERNAME}-key.pem"', '-out', '"${SERVERNAME}-req.pem"'
             ]
         })
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
             call.args[0].split() == [
                 'openssl', 'ca', '-config', 'ssl.cnf', '-batch', '-notext',
-                '-in', '"ca/${SERVERNAME}-req.pem"', '-out', '"ca/${SERVERNAME}-cert.pem"'
+                '-in', '"${SERVERNAME}-req.pem"', '-out', '"${SERVERNAME}-cert.pem"'
             ]
         })
         assertEquals(1, helper.methodCallCount('stash'))
@@ -101,6 +132,7 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGetSslCert() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { false })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
@@ -127,6 +159,7 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGetSslCertExists() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { true })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
@@ -142,19 +175,20 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGenKeystore() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { true })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
         ca.gen_keystore('testserver', 'testpass')
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'writeFile' &&
-            call.args[0].file == 'ca/testserver.pem'
+            call.args[0].file == 'testserver.pem'
         })
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
             call.args[0].split() == [
                 'openssl', 'pkcs12', '-export',
-                '-in', '"ca/${SERVERNAME}.pem"', '-out', '"ca/${SERVERNAME}.ks"',
+                '-in', '"${SERVERNAME}.pem"', '-out', '"${SERVERNAME}.ks"',
                 '-name', '"${SERVERNAME}"', '-passout', 'env:PASSWD'
             ]
         })
@@ -163,13 +197,14 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGetKeystore() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { true })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [Map.class], { 'certdata' })
         ca.get_keystore('testserver', 'testpass')
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'readFile' &&
-            call.args[0].file == 'ca/testserver.ks' &&
+            call.args[0].file == 'testserver.ks' &&
             call.args[0].encoding == 'Base64'
         })
         assertEquals(0, helper.methodCallCount('stash'))
@@ -177,6 +212,7 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGenTruststore() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { true })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [String.class], { 'certdata' })
@@ -184,8 +220,8 @@ class CATest extends BasePipelineTestCPS {
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'sh' &&
             call.args[0].split() == [
-                'keytool', '-importcert', '-file', 'ca/ca-cert.pem', '-alias', 'testca',
-                '-keystore', 'ca/truststore.ts', '-storetype', 'jks',
+                'keytool', '-importcert', '-file', 'ca-cert.pem', '-alias', 'testca',
+                '-keystore', 'truststore.ts', '-storetype', 'jks',
                 '-storepass', '"${PASSWD}"', '-trustcacerts', '-noprompt'
             ]
         })
@@ -194,13 +230,14 @@ class CATest extends BasePipelineTestCPS {
 
     @Test
     void testGetTruststore() {
+        ca.env.CA_PATH = '/workspace/ca'
         helper.registerAllowedMethod('fileExists', [String.class], { true })
         helper.registerAllowedMethod('unstash', [String.class], null)
         helper.registerAllowedMethod('readFile', [Map.class], { 'certdata' })
         ca.get_truststore('testpass')
         assertTrue(helper.callStack.any { call ->
             call.methodName == 'readFile' &&
-            call.args[0].file == 'ca/truststore.ts' &&
+            call.args[0].file == 'truststore.ts' &&
             call.args[0].encoding == 'Base64'
         })
         assertEquals(0, helper.methodCallCount('stash'))
diff --git a/vars/ca.groovy b/vars/ca.groovy
index 24b552a..78fda61 100644
--- a/vars/ca.groovy
+++ b/vars/ca.groovy
@@ -1,30 +1,37 @@
-def gen_ca() {
-    sh 'rm -rf ./ca'
+def gen_ca(path='') {
     def conftmpl = libraryResource 'ca/ssl.cnf.in'
     def conf = conftmpl.replace('${servername}', 'Test Certificate Authority')
     conf = conf.replace('${commonName}', 'Test Certificate Authority')
     conf = conf.replace('${subjectAltNames}', '')
-    writeFile file: 'ssl.cnf', text: conf
-    writeFile file: 'ca/index.txt', text: ''
-    writeFile file: 'ca/index.txt.attr', text: ''
-    writeFile file: 'ca/serial', text: '01\n'
-    sh 'openssl genrsa -out ca/ca-key.pem'
-    sh 'openssl req -config ssl.cnf -batch -new -x509 -key ca/ca-key.pem -out ca/ca-cert.pem'
-    stash name: 'ca', includes: 'ca/**'
+    env.CA_PATH = path ?: "${env.WORKSPACE}/.ca-${env.BUILD_NUMBER}"
+    dir(env.CA_PATH) {
+        deleteDir()
+        writeFile file: 'ssl.cnf', text: conf
+        writeFile file: 'index.txt', text: ''
+        writeFile file: 'index.txt.attr', text: ''
+        writeFile file: 'serial', text: '01\n'
+        sh 'openssl genrsa -out ca-key.pem'
+        sh 'openssl req -config ssl.cnf -batch -new -x509 -key ca-key.pem -out ca-cert.pem'
+        stash name: 'ca'
+    }
 }
 
 def get_ca() {
-    try {
-        unstash 'ca'
-    } catch (e) {
+    if (env.CA_PATH) {
+        dir(env.CA_PATH) {
+            unstash 'ca'
+        }
+    } else {
         gen_ca()
     }
 }
 
 def get_ca_cert() {
     get_ca()
-    def cert = readFile 'ca/ca-cert.pem'
-    return ['cert': cert]
+    dir(env.CA_PATH) {
+        def cert = readFile 'ca-cert.pem'
+        return ['cert': cert]
+    }
 }
 
 def gen_ssl_cert(servername, String... sans) {
@@ -37,60 +44,72 @@ def gen_ssl_cert(servername, String... sans) {
     def conf = conftmpl.replace('${servername}', servername)
     conf = conf.replace('${commonName}', servername.tokenize('.')[0])
     conf = conf.replace('${subjectAltNames}', sanslist)
-    writeFile file: 'ssl.cnf', text: conf
-    withEnv(["SERVERNAME=${servername}"]) {
-        sh 'openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes' + \
-           ' -keyout "ca/${SERVERNAME}-key.pem" -out "ca/${SERVERNAME}-req.pem"'
-        sh 'openssl ca -config ssl.cnf -batch -notext' + \
-           ' -in "ca/${SERVERNAME}-req.pem" -out "ca/${SERVERNAME}-cert.pem"'
+    dir(env.CA_PATH) {
+        writeFile file: 'ssl.cnf', text: conf
+        withEnv(["SERVERNAME=${servername}"]) {
+            sh 'openssl req -config ssl.cnf -batch -new -newkey rsa:2048 -nodes' + \
+               ' -keyout "${SERVERNAME}-key.pem" -out "${SERVERNAME}-req.pem"'
+            sh 'openssl ca -config ssl.cnf -batch -notext' + \
+               ' -in "${SERVERNAME}-req.pem" -out "${SERVERNAME}-cert.pem"'
+        }
+        stash name: 'ca'
     }
-    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_ssl_cert(servername, String... sans) {
     get_ca()
-    if (!fileExists("ca/${servername}-cert.pem")) {
-        gen_ssl_cert(servername, sans)
+    dir(env.CA_PATH) {
+        if (!fileExists("${servername}-cert.pem")) {
+            gen_ssl_cert(servername, sans)
+        }
+        def cert = readFile "${servername}-cert.pem"
+        def key = readFile "${servername}-key.pem"
+        return ['cert': cert, 'key': key]
     }
-    def cert = readFile "ca/${servername}-cert.pem"
-    def key = readFile "ca/${servername}-key.pem"
-    return ['cert': cert, 'key': key]
 }
 
 def gen_keystore(servername, passwd) {
     data = get_ssl_cert(servername)
-    writeFile file: "ca/${servername}.pem", text: data['cert'] + data['key']
-    withEnv(["SERVERNAME=${servername}", "PASSWD=${passwd}"]) {
-        sh 'openssl pkcs12 -export -in "ca/${SERVERNAME}.pem" -out "ca/${SERVERNAME}.ks"' + \
-           ' -name "${SERVERNAME}" -passout env:PASSWD'
+    dir(env.CA_PATH) {
+        writeFile file: "${servername}.pem", text: data['cert'] + data['key']
+        withEnv(["SERVERNAME=${servername}", "PASSWD=${passwd}"]) {
+            sh 'openssl pkcs12 -export -in "${SERVERNAME}.pem" -out "${SERVERNAME}.ks"' + \
+               ' -name "${SERVERNAME}" -passout env:PASSWD'
+        }
+        stash name: 'ca'
     }
-    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_keystore(servername, passwd) {
     get_ca()
-    if (!fileExists("ca/${servername}.ks")) {
-        gen_keystore(servername, passwd)
+    dir(env.CA_PATH) {
+        if (!fileExists("${servername}.ks")) {
+            gen_keystore(servername, passwd)
+        }
+        return readFile(file: "${servername}.ks", encoding: 'Base64')
     }
-    return readFile(file: "ca/${servername}.ks", encoding: 'Base64')
 }
 
 def gen_truststore(passwd) {
     get_ca()
     // Java 7 doesn't support PKCS12 truststores.
     // Switch this to using PKCS12 when everything is running on Java 8+
-    withEnv(["PASSWD=${passwd}"]) {
-        sh 'keytool -importcert -file ca/ca-cert.pem -alias testca' + \
-           ' -keystore ca/truststore.ts -storetype jks' + \
-           ' -storepass "${PASSWD}" -trustcacerts -noprompt'
+    dir(env.CA_PATH) {
+        withEnv(["PASSWD=${passwd}"]) {
+            sh 'keytool -importcert -file ca-cert.pem -alias testca' + \
+               ' -keystore truststore.ts -storetype jks' + \
+               ' -storepass "${PASSWD}" -trustcacerts -noprompt'
+        }
+        stash name: 'ca'
     }
-    stash name: 'ca', includes: 'ca/**'
 }
 
 def get_truststore(passwd) {
     get_ca()
-    if (!fileExists('ca/truststore.ts')) {
-        gen_truststore(passwd)
+    dir(env.CA_PATH) {
+        if (!fileExists('truststore.ts')) {
+            gen_truststore(passwd)
+        }
+        return readFile(file: 'truststore.ts', encoding: 'Base64')
     }
-    return readFile(file: 'ca/truststore.ts', encoding: 'Base64')
 }