Skip to content

Commit 155ad68

Browse files
rojkovrojkov
authored
Merge pull request #682 from mythi/PR-2021-041
e2e improvements
2 parents 865d385 + 8c6b8ce commit 155ad68

File tree

6 files changed

+254
-9
lines changed

6 files changed

+254
-9
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ test-with-kind:
6666
@$(KIND) create cluster --name "intel-device-plugins" --kubeconfig $(e2e_tmp_dir)/kubeconfig --image "kindest/node:v1.19.0"
6767
@$(KIND) load image-archive --name "intel-device-plugins" $(e2e_tmp_dir)/$(WEBHOOK_IMAGE_FILE)
6868
$(KUBECTL) --kubeconfig=$(e2e_tmp_dir)/kubeconfig apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml
69-
@$(GO) test -v ./test/e2e -args -kubeconfig $(e2e_tmp_dir)/kubeconfig -kubectl-path $(KUBECTL) -ginkgo.focus "Webhook" || rc=1; \
69+
@$(GO) test -v ./test/e2e -args -kubeconfig $(e2e_tmp_dir)/kubeconfig -kubectl-path $(KUBECTL) -ginkgo.focus "FPGA Admission" || rc=1; \
7070
$(KIND) delete cluster --name "intel-device-plugins"; \
7171
rm -rf $(e2e_tmp_dir); \
7272
exit $$rc

‎test/e2e/deviceplugins_suite_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/gpu"
2828
_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/qat"
2929
_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgx"
30+
_ "github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/sgxadmissionwebhook"
3031
v1 "k8s.io/api/core/v1"
3132
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3233
"k8s.io/component-base/logs"

‎test/e2e/fpgaadmissionwebhook/fpgaadmissionwebhook.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func checkPodMutation(f *framework.Framework, mappingsNamespace string, source,
7373
}
7474

7575
ginkgo.By("deploying webhook")
76-
utils.DeployFpgaWebhook(f, kustomizationPath)
76+
_ = utils.DeployWebhook(f, kustomizationPath)
7777

7878
ginkgo.By("deploying mappings")
7979
framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-n", mappingsNamespace, "-f", filepath.Dir(kustomizationPath)+"/../mappings-collection.yaml")

‎test/e2e/qat/qatplugin_dpdk.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,19 @@ func describeQatDpdkPlugin() {
5757
framework.RunKubectlOrDie(f.Namespace.Name, "--namespace", f.Namespace.Name, "apply", "-k", filepath.Dir(kustomizationPath))
5858

5959
ginkgo.By("waiting for QAT plugin's availability")
60-
if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
61-
labels.Set{"app": "intel-qat-plugin"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil {
60+
podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
61+
labels.Set{"app": "intel-qat-plugin"}.AsSelector(), 1 /* one replica */, 10*time.Second)
62+
if err != nil {
6263
framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
6364
kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
6465
framework.Failf("unable to wait for all pods to be running and ready: %v", err)
6566
}
6667

68+
ginkgo.By("checking QAT plugin's securityContext")
69+
if err := utils.TestPodsFileSystemInfo(podList.Items); err != nil {
70+
framework.Failf("container filesystem info checks failed: %v", err)
71+
}
72+
6773
ginkgo.By("checking the resource is allocatable")
6874
if err := utils.WaitForNodesWithResource(f.ClientSet, "qat.intel.com/generic", 30*time.Second); err != nil {
6975
framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)

‎test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
// Copyright 2021 Intel Corporation. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package sgxadmissionwebhook implements E2E tests for SGX admission webhook.
16+
package sgxadmissionwebhook
17+
18+
import (
19+
"context"
20+
"reflect"
21+
22+
"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
23+
"github.com/onsi/ginkgo"
24+
"github.com/onsi/gomega"
25+
v1 "k8s.io/api/core/v1"
26+
"k8s.io/apimachinery/pkg/api/resource"
27+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
28+
"k8s.io/kubernetes/test/e2e/framework"
29+
"k8s.io/kubernetes/test/e2e/framework/kubectl"
30+
imageutils "k8s.io/kubernetes/test/utils/image"
31+
)
32+
33+
const (
34+
kustomizationYaml = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
35+
)
36+
37+
func init() {
38+
ginkgo.Describe("SGX Admission Webhook", describe)
39+
}
40+
41+
func describe() {
42+
f := framework.NewDefaultFramework("sgxwebhook")
43+
var webhook v1.Pod
44+
45+
ginkgo.BeforeEach(func() {
46+
kustomizationPath, err := utils.LocateRepoFile(kustomizationYaml)
47+
if err != nil {
48+
framework.Failf("unable to locate %q: %v", kustomizationYaml, err)
49+
}
50+
webhook = utils.DeployWebhook(f, kustomizationPath)
51+
})
52+
53+
ginkgo.It("checks the webhook pod is safely configured", func() {
54+
err := utils.TestContainersRunAsNonRoot([]v1.Pod{webhook})
55+
gomega.Expect(err).To(gomega.BeNil())
56+
})
57+
ginkgo.It("mutates created pods when no quote generation is needed", func() {
58+
ginkgo.By("submitting the pod")
59+
pod := submitPod(f, []string{"test"}, "")
60+
61+
ginkgo.By("checking the container resources have been mutated")
62+
checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
63+
64+
ginkgo.By("checking the pod total EPC size annotation is correctly set")
65+
gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
66+
})
67+
ginkgo.It("mutates created pods when the container contains the quote generation libraries", func() {
68+
ginkgo.By("submitting the pod")
69+
pod := submitPod(f, []string{"test"}, "test")
70+
71+
ginkgo.By("checking the container resources have been mutated")
72+
checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave", "sgx.intel.com/provision"}, []v1.ResourceName{})
73+
74+
ginkgo.By("checking the pod total EPC size annotation is correctly set")
75+
gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
76+
})
77+
ginkgo.It("mutates created pods when the container uses aesmd from a side-car container to generate quotes", func() {
78+
ginkgo.By("submitting the pod")
79+
pod := submitPod(f, []string{"test", "aesmd"}, "aesmd")
80+
ginkgo.By("checking the container resources have been mutated")
81+
checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
82+
checkMutatedResources(f, pod.Spec.Containers[1].Resources, []v1.ResourceName{"sgx.intel.com/enclave", "sgx.intel.com/provision"}, []v1.ResourceName{})
83+
ginkgo.By("checking the container volumes have been mutated")
84+
checkMutatedVolumes(f, pod, "aesmd-socket", v1.EmptyDirVolumeSource{})
85+
ginkgo.By("checking the container envvars have been mutated")
86+
gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR"))
87+
gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
88+
ginkgo.By("checking the pod total EPC size annotation is correctly set")
89+
gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("2Mi"))
90+
})
91+
ginkgo.It("mutates created pods where one container uses host/daemonset aesmd to generate quotes", func() {
92+
ginkgo.By("submitting the pod")
93+
pod := submitPod(f, []string{"test"}, "aesmd")
94+
ginkgo.By("checking the container resources have been mutated")
95+
checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
96+
ginkgo.By("checking the container volumes have been mutated")
97+
checkMutatedVolumes(f, pod, "aesmd-socket", v1.HostPathVolumeSource{})
98+
ginkgo.By("checking the container envvars have been mutated")
99+
gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR"))
100+
gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
101+
ginkgo.By("checking the pod total EPC size annotation is correctly set")
102+
gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
103+
})
104+
ginkgo.It("mutates created pods where three containers use host/daemonset aesmd to generate quotes", func() {
105+
ginkgo.By("submitting the pod")
106+
pod := submitPod(f, []string{"test1", "test2", "test3"}, "aesmd")
107+
ginkgo.By("checking the container resources have been mutated")
108+
checkMutatedResources(f, pod.Spec.Containers[0].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
109+
checkMutatedResources(f, pod.Spec.Containers[1].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
110+
checkMutatedResources(f, pod.Spec.Containers[2].Resources, []v1.ResourceName{"sgx.intel.com/enclave"}, []v1.ResourceName{"sgx.intel.com/provision"})
111+
ginkgo.By("checking the container volumes have been mutated")
112+
checkMutatedVolumes(f, pod, "aesmd-socket", v1.HostPathVolumeSource{})
113+
ginkgo.By("checking the container envvars have been mutated")
114+
gomega.Expect(pod.Spec.Containers[0].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR"))
115+
gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
116+
gomega.Expect(pod.Spec.Containers[1].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR"))
117+
gomega.Expect(pod.Spec.Containers[1].Env[0].Value).To(gomega.Equal("1"))
118+
gomega.Expect(pod.Spec.Containers[2].Env[0].Name).To(gomega.Equal("SGX_AESM_ADDR"))
119+
gomega.Expect(pod.Spec.Containers[2].Env[0].Value).To(gomega.Equal("1"))
120+
ginkgo.By("checking the pod total EPC size annotation is correctly set")
121+
gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("3Mi"))
122+
})
123+
}
124+
125+
func checkMutatedVolumes(f *framework.Framework, pod *v1.Pod, volumeName string, volumeType interface{}) {
126+
switch reflect.TypeOf(volumeType).String() {
127+
case "v1.HostPathVolumeSource":
128+
gomega.Expect(pod.Spec.Volumes[0].HostPath).NotTo(gomega.BeNil())
129+
gomega.Expect(pod.Spec.Volumes[0].Name).To(gomega.Equal(volumeName))
130+
case "v1.EmptyDirVolumeSource":
131+
gomega.Expect(pod.Spec.Volumes[0].EmptyDir).NotTo(gomega.BeNil())
132+
gomega.Expect(pod.Spec.Volumes[0].Name).To(gomega.Equal(volumeName))
133+
}
134+
135+
for _, c := range pod.Spec.Containers {
136+
gomega.Expect(c.VolumeMounts[0].Name).To(gomega.Equal(volumeName))
137+
}
138+
}
139+
140+
func checkMutatedResources(f *framework.Framework, r v1.ResourceRequirements, expectedResources, forbiddenResources []v1.ResourceName) {
141+
for _, res := range expectedResources {
142+
q, ok := r.Limits[res]
143+
if !ok {
144+
framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
145+
kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
146+
framework.Fail("the pod has missing resources")
147+
}
148+
gomega.Expect(q.String()).To(gomega.Equal("1"))
149+
}
150+
for _, res := range forbiddenResources {
151+
_, ok := r.Limits[res]
152+
if ok {
153+
framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
154+
kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
155+
framework.Fail("the pod has extra resources")
156+
}
157+
}
158+
}
159+
160+
func submitPod(f *framework.Framework, containerNames []string, quoteProvider string) *v1.Pod {
161+
containers := make([]v1.Container, 0)
162+
163+
for _, c := range containerNames {
164+
containers = append(containers, v1.Container{
165+
Name: c,
166+
Image: imageutils.GetPauseImageName(),
167+
Resources: v1.ResourceRequirements{
168+
Requests: v1.ResourceList{"sgx.intel.com/epc": resource.MustParse("1Mi")},
169+
Limits: v1.ResourceList{"sgx.intel.com/epc": resource.MustParse("1Mi")},
170+
},
171+
})
172+
}
173+
174+
disabled := false
175+
176+
podSpec := &v1.Pod{
177+
ObjectMeta: metav1.ObjectMeta{
178+
Name: "webhook-tester-pod",
179+
Annotations: map[string]string{
180+
"sgx.intel.com/quote-provider": quoteProvider,
181+
},
182+
},
183+
Spec: v1.PodSpec{
184+
AutomountServiceAccountToken: &disabled,
185+
Containers: containers,
186+
},
187+
}
188+
189+
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(),
190+
podSpec, metav1.CreateOptions{})
191+
192+
framework.ExpectNoError(err, "pod Create API error")
193+
194+
return pod
195+
}

‎test/e2e/utils/utils.go‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,14 +126,14 @@ func CreateKustomizationOverlay(namespace, base, overlay string) error {
126126
return os.WriteFile(overlay+"/kustomization.yaml", []byte(content), 0600)
127127
}
128128

129-
// DeployFpgaWebhook deploys FPGA admission webhook to a framework-specific namespace.
130-
func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) {
129+
// DeployWebhook deploys an admission webhook to a framework-specific namespace.
130+
func DeployWebhook(f *framework.Framework, kustomizationPath string) v1.Pod {
131131
if _, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, "cert-manager",
132132
labels.Set{"app.kubernetes.io/name": "cert-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil {
133133
framework.Failf("unable to detect running cert-manager: %v", err)
134134
}
135135

136-
tmpDir, err := os.MkdirTemp("", "fpgawebhooke2etest-"+f.Namespace.Name)
136+
tmpDir, err := os.MkdirTemp("", "webhooke2etest-"+f.Namespace.Name)
137137
if err != nil {
138138
framework.Failf("unable to create temp directory: %v", err)
139139
}
@@ -145,10 +145,53 @@ func DeployFpgaWebhook(f *framework.Framework, kustomizationPath string) {
145145
}
146146

147147
framework.RunKubectlOrDie(f.Namespace.Name, "apply", "-k", tmpDir)
148-
if _, err = e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
149-
labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second); err != nil {
148+
podList, err := e2epod.WaitForPodsWithLabelRunningReady(f.ClientSet, f.Namespace.Name,
149+
labels.Set{"control-plane": "controller-manager"}.AsSelector(), 1 /* one replica */, 10*time.Second)
150+
if err != nil {
150151
framework.DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
151152
kubectl.LogFailedContainers(f.ClientSet, f.Namespace.Name, framework.Logf)
152153
framework.Failf("unable to wait for all pods to be running and ready: %v", err)
153154
}
155+
return podList.Items[0]
156+
}
157+
158+
// TestContainersRunAsNonRoot checks that all containers within the Pods run
159+
// with non-root UID/GID.
160+
func TestContainersRunAsNonRoot(pods []v1.Pod) error {
161+
for _, p := range pods {
162+
for _, c := range append(p.Spec.InitContainers, p.Spec.Containers...) {
163+
if !*c.SecurityContext.RunAsNonRoot {
164+
return fmt.Errorf("%s (container: %s): RunAsNonRoot is not true", p.Name, c.Name)
165+
}
166+
if *c.SecurityContext.RunAsGroup == 0 {
167+
return fmt.Errorf("%s (container: %s): RunAsGroup is root (0)", p.Name, c.Name)
168+
}
169+
if *c.SecurityContext.RunAsUser == 0 {
170+
return fmt.Errorf("%s (container: %s): RunAsUser is root (0)", p.Name, c.Name)
171+
}
172+
}
173+
}
174+
return nil
175+
}
176+
177+
func printVolumeMounts(vm []v1.VolumeMount) {
178+
for _, v := range vm {
179+
if !v.ReadOnly {
180+
framework.Logf("Available RW volume mounts: %v", v)
181+
}
182+
}
183+
}
184+
185+
// TestPodsFileSystemInfo checks that all containers within the Pods run
186+
// with ReadOnlyRootFileSystem. It also prints RW volume mounts.
187+
func TestPodsFileSystemInfo(pods []v1.Pod) error {
188+
for _, p := range pods {
189+
for _, c := range append(p.Spec.InitContainers, p.Spec.Containers...) {
190+
if !*c.SecurityContext.ReadOnlyRootFilesystem {
191+
return fmt.Errorf("%s (container: %s): Writable root filesystem", p.Name, c.Name)
192+
}
193+
printVolumeMounts(c.VolumeMounts)
194+
}
195+
}
196+
return nil
154197
}

0 commit comments

Comments
 (0)