Merge pull request #207 from mythi/lock-versions

kad · web-flow · commit 0cda452c4168 · 2019-08-29T15:24:23.000+03:00
Prepare for image publishing
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -7,6 +7,7 @@ pipeline {
   }
   environment {
     GO111MODULE="on"
+    REG = "cloud-native-image-registry.westus.cloudapp.azure.com/"
     RUNC_VERSION="v1.0.0-rc8"
     CRIO_VERSION="v1.14.6"
     BUILDAH_VERSION="v1.10.0"
@@ -134,4 +135,16 @@ pipeline {
       }
     }
   }
+  post {
+    success {
+      script {
+        if (env.CHANGE_ID == null) {
+          withDockerRegistry([ credentialsId: "57e4a8b2-ccf9-4da1-a787-76dd1aac8fd1", url: "https://${REG}" ]) {
+            sh "make images PUSH=1"
+            sh "make demos PUSH=1"
+          }
+        }
+      }
+    }
+  }
 }
diff --git a/Makefile b/Makefile
@@ -41,21 +41,35 @@ build: $(cmds)
 clean:
 	@for cmd in $(cmds) ; do pwd=$(shell pwd) ; cd cmd/$$cmd ; $(GO) clean ; cd $$pwd ; done
 
-TAG?=$(shell git rev-parse HEAD)
+ORG?=intel
+REG?=$(ORG)/
+TAG?=devel
+export TAG
 
 images = $(shell ls build/docker/*.Dockerfile | sed 's/.*\/\(.\+\)\.Dockerfile/\1/')
 
 $(images):
-	@build/docker/build-image.sh $@ $(BUILDER)
+ifndef PUSH
+	@build/docker/build-image.sh $(REG)$@ $(BUILDER)
+else
+	@docker push $(REG)$@
+endif
 
 images: $(images)
 
 demos = $(shell cd demo/ && ls -d */ | sed 's/\(.\+\)\//\1/g')
 
 $(demos):
-	@cd demo/ && ./build-image.sh $@ $(BUILDER)
+ifndef PUSH
+	@cd demo/ && ./build-image.sh $(REG)$@ $(BUILDER)
+else
+	@docker push ${REG}$@
+endif
 
 demos: $(demos)
 
+lock-images:
+	@scripts/update-clear-linux-base.sh clearlinux/golang:latest $(shell ls build/docker/*.Dockerfile)
+	@scripts/update-clear-linux-base.sh clearlinux:latest $(shell find demo -name Dockerfile)
 
-.PHONY: all format vet cyclomatic-check test lint build images $(cmds) $(images)
+.PHONY: all format vet cyclomatic-check test lint build images $(cmds) $(images) lock-images
diff --git a/build/docker/build-image.sh b/build/docker/build-image.sh
@@ -3,7 +3,7 @@
 IMG=$1
 BUILDER=$2
 
-DOCKERFILE="$(dirname $0)/${IMG}.Dockerfile"
+DOCKERFILE="$(dirname $0)/$(basename ${IMG}).Dockerfile"
 
 if [ -z "$IMG" ]; then
     (>&2 echo "Usage: $0 <Dockerfile>")
@@ -15,14 +15,15 @@ if [ ! -e "${DOCKERFILE}" ]; then
     exit 1
 fi
 
-TAG=$(git rev-parse HEAD)
+TAG=${TAG:-devel}
+SRCREV=$(git rev-parse HEAD)
 
 if [ -z "${BUILDER}" -o "${BUILDER}" = 'docker' ] ; then
     docker build --pull -t ${IMG}:${TAG} -f ${DOCKERFILE} .
-    docker tag ${IMG}:${TAG} ${IMG}:devel
+    docker tag ${IMG}:${TAG} ${IMG}:${SRCREV}
 elif [ "${BUILDER}" = 'buildah' ] ; then
     buildah bud --pull-always -t ${IMG}:${TAG} -f ${DOCKERFILE} .
-    buildah tag ${IMG}:${TAG} ${IMG}:devel
+    buildah tag ${IMG}:${TAG} ${IMG}:${SRCREV}
 else
     (>&2 echo "Unknown builder ${BUILDER}")
     exit 1
diff --git a/build/docker/intel-qat-plugin.Dockerfile b/build/docker/intel-qat-plugin.Dockerfile
@@ -45,4 +45,4 @@ RUN chmod a+x /go/bin/qat_plugin \
 FROM scratch as final
 COPY --from=builder /install_root /
 ENV PATH=/usr/local/bin
-CMD ["/usr/local/bin/intel_qat_device_plugin"]
+ENTRYPOINT ["/usr/local/bin/intel_qat_device_plugin"]
diff --git a/demo/build-image.sh b/demo/build-image.sh
@@ -2,26 +2,28 @@
 
 IMG=$1
 BUILDER=$2
+DIR=$(basename $IMG)
 
-if [ -z "$IMG" ]; then
+if [ -z "$DIR" ]; then
     (>&2 echo "Usage: $0 <image directory>")
     exit 1
 fi
 
-if [ ! -d "$IMG" ]; then
-    (>&2 echo "Directory $IMG doesn't exist")
+if [ ! -d "$DIR" ]; then
+    (>&2 echo "Directory $DIR doesn't exist")
     exit 1
 fi
 
 CWD=`dirname $0`
-TAG=`git rev-parse HEAD`
+TAG=${TAG:-devel}
+SRCREV=$(git rev-parse HEAD)
 
 if [ -z "$BUILDER" -o "$BUILDER" = 'docker' ] ; then
-    docker build --rm -t ${IMG}:${TAG} "$CWD/$IMG/"
-    docker tag ${IMG}:${TAG} ${IMG}:devel
+    docker build --pull -t ${IMG}:${TAG} "$CWD/$DIR/"
+    docker tag ${IMG}:${TAG} ${IMG}:${SRCREV}
 elif [ "$BUILDER" = 'buildah' ] ; then
-    buildah bud -t ${IMG}:${TAG} "$CWD/$IMG/"
-    buildah tag ${IMG}:${TAG} ${IMG}:devel
+    buildah bud  --pull-always -t ${IMG}:${TAG} "$CWD/$DIR/"
+    buildah tag ${IMG}:${TAG} ${IMG}:${SRCREV}
 else
     (>&2 echo "Unknown builder $BUILDER")
     exit 1
diff --git a/deployments/fpga_admissionwebhook/deployment-tpl.yaml b/deployments/fpga_admissionwebhook/deployment-tpl.yaml
@@ -13,7 +13,7 @@ spec:
     spec:
       containers:
         - name: fpga-mutator
-          image: intel-fpga-admissionwebhook:devel
+          image: intel/intel-fpga-admissionwebhook:devel
           imagePullPolicy: IfNotPresent
           args:
             - -tls-cert-file=/etc/webhook/certs/cert.pem
diff --git a/deployments/fpga_plugin/fpga_plugin.yaml b/deployments/fpga_plugin/fpga_plugin.yaml
@@ -17,7 +17,7 @@ spec:
       serviceAccountName: intel-fpga-plugin-controller
       initContainers:
       - name: intel-fpga-initcontainer
-        image: intel-fpga-initcontainer:devel
+        image: intel/intel-fpga-initcontainer:devel
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - mountPath: /opt/intel/fpga-sw
@@ -31,7 +31,7 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
-        image: intel-fpga-plugin:devel
+        image: intel/intel-fpga-plugin:devel
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: devfs
diff --git a/deployments/gpu_plugin/gpu_plugin.yaml b/deployments/gpu_plugin/gpu_plugin.yaml
@@ -21,7 +21,7 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
-        image: intel-gpu-plugin:devel
+        image: intel/intel-gpu-plugin:devel
         imagePullPolicy: IfNotPresent
         volumeMounts:
         - name: devfs
diff --git a/deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml b/deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml
@@ -5,7 +5,7 @@ metadata:
 spec:
   containers:
   - name: crypto-perf
-    image: crypto-perf:devel
+    image: intel/crypto-perf:devel
     imagePullPolicy: IfNotPresent
     command: [ "/bin/bash", "-c", "--" ]
     args: [ "while true; do sleep 300000; done;" ]
diff --git a/deployments/qat_plugin/qat_plugin.yaml b/deployments/qat_plugin/qat_plugin.yaml
@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: intel-qat-plugin
-        image: intel-qat-plugin:devel
+        image: intel/intel-qat-plugin:devel
         env:
         - name: DPDK_DRIVER
           valueFrom:
@@ -38,7 +38,7 @@ spec:
               name: intel-qat-plugin-config
               key: DEBUG
         imagePullPolicy: IfNotPresent
-        command: ["/usr/bin/intel_qat_device_plugin", "-dpdk-driver", "$(DPDK_DRIVER)", "-kernel-vf-drivers", "$(KERNEL_VF_DRIVERS)", "-max-num-devices", "$(MAX_NUM_DEVICES)", "-debug", "$(DEBUG)"]
+        args: ["-dpdk-driver", "$(DPDK_DRIVER)", "-kernel-vf-drivers", "$(KERNEL_VF_DRIVERS)", "-max-num-devices", "$(MAX_NUM_DEVICES)", "-debug", "$(DEBUG)"]
         volumeMounts:
         - name: pcidir
           mountPath: /sys/bus/pci
diff --git a/deployments/qat_plugin/qat_plugin_kernel_mode.yaml b/deployments/qat_plugin/qat_plugin_kernel_mode.yaml
@@ -17,9 +17,9 @@ spec:
       - name: intel-qat-kernel-plugin
         securityContext:
           privileged: true
-        image: intel-qat-plugin:devel
+        image: intel/intel-qat-plugin:devel
         imagePullPolicy: IfNotPresent
-        command: ["/usr/bin/intel_qat_device_plugin", "-mode", "kernel"]
+        args: ["-mode", "kernel"]
         volumeMounts:
         - name: devfs
           mountPath: /dev
diff --git a/scripts/set-version.sh b/scripts/set-version.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+#
+# Copyright 2019 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Invoke this script with a version as parameter
+# and it will update all hard-coded image versions
+# in the source code.
+#
+# Adapted from https://github.com/intel/pmem-csi/
+
+if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
+    echo "Usage: $0 <image version>" >&2
+    exit 1
+fi
+
+sed -i -e "s;\(^TAG?*=\|intel/crypto-perf:\|intel/intel-[^ ]*:\)[^ ]*;\1$1;g" $(git grep -l '^TAG?*=\|intel/crypto-perf:\|intel/intel-[^ ]*:' Makefile deployments)
diff --git a/scripts/update-clear-linux-base.sh b/scripts/update-clear-linux-base.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# Copyright 2019 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Invoke this script on a system that has Docker installed
+# such that it can be used by the current user. Then the script
+# will bump up the CLEAR_LINUX_BASE and CLEAR_LINUX_VERSION
+# parameters in the Dockerfiles such that they pick the
+# current version of Clear Linux.
+#
+# The script is adapted from https://github.com/intel/pmem-csi/
+
+die () {
+    echo "ERROR: $@"
+    exit 1
+}
+
+IMG=$1
+shift
+DOCKERFILES=$@
+
+docker image pull $IMG || die "pulling $IMG failed"
+base=$(docker inspect --format='{{index .RepoDigests 0}}' $IMG) || die "failed to inspect $IMG"
+echo "Base image: $base"
+
+# We rely on swupd to determine what this particular image can be
+# updated to with "swupd update --version". This might not be the very latest
+# Clear Linux, for example when there has been a format bump and the
+# base image is still using the older format.
+output=$(docker run $base swupd check-update) # will return non-zero exit code if there is nothing to update
+# The expected output on failure is one of:
+#     Current OS version: 30450
+#     Latest server version: 30450
+#     There are no updates available
+# or:
+#     Current OS version: 29940
+#     Latest server version: 29970
+#     There is a new OS version available: 29970
+version=$(echo "$output" | grep "Latest server version" | tail -n 1 | sed -e 's/.*: *//')
+if [ ! "$version" ]; then
+    die "failed to obtain information about available updates"
+fi
+echo "Update version: $version"
+
+# Do a trial-run with these parameters.
+docker run "$base" swupd update --version=$version || die "failed to update"
+
+# Now update the Dockerfile(s).
+sed -i -e 's;^\(ARG CLEAR_LINUX_BASE=\).*;\1'"$base"';' -e 's;^\(ARG CLEAR_LINUX_VERSION=\).*;\1"--version='"$version"'";' $DOCKERFILES || die "failed to patch Dockerfiles"
+
+echo "Done."

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@ pipeline {`
`7`	`7`	`}`
`8`	`8`	`environment {`
`9`	`9`	`GO111MODULE="on"`
	`10`	`+ REG = "cloud-native-image-registry.westus.cloudapp.azure.com/"`
`10`	`11`	`RUNC_VERSION="v1.0.0-rc8"`
`11`	`12`	`CRIO_VERSION="v1.14.6"`
`12`	`13`	`BUILDAH_VERSION="v1.10.0"`
`@@ -134,4 +135,16 @@ pipeline {`
`134`	`135`	`}`
`135`	`136`	`}`
`136`	`137`	`}`
	`138`	`+ post {`
	`139`	`+ success {`
	`140`	`+ script {`
	`141`	`+ if (env.CHANGE_ID == null) {`
	`142`	`+ withDockerRegistry([ credentialsId: "57e4a8b2-ccf9-4da1-a787-76dd1aac8fd1", url: "https://${REG}" ]) {`
	`143`	`+ sh "make images PUSH=1"`
	`144`	`+ sh "make demos PUSH=1"`
	`145`	`+ }`
	`146`	`+ }`
	`147`	`+ }`
	`148`	`+ }`
	`149`	`+ }`
`137`	`150`	`}`