Merge pull request #507 from mythi/sgx-v41

SGX v4x updates

Author: rojkov

cmd/sgx_plugin/README.md

@@ -5,6 +5,7 @@ Contents
 * [Introduction](#introduction)
 * [Installation](#installation)
     * [Prerequisites](#prerequisites)
+        * [Backwards compatiblity note](#backwards-compatibility-note)
     * [Pre-built images](#pre-built-images)
     * [Getting the source code](#getting-the-source-code)
     * [Verify node kubelet config](#verify-node-kubelet-config)
@@ -63,11 +64,44 @@ The component has the same basic dependancies as the
 [generic plugin framework dependencies](../../README.md#about).
 
 The SGX device plugin requires Linux Kernel SGX drivers to be available. These drivers
-are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git/tag/?h=v39).
-RFC *v39* was used to validate what is written in this document.
+are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git).
+RFC *v41* was used to validate what is written in this document.
 
 The hardware platform must support SGX Flexible Launch Control.
 
+#### Backwards compatibility note
+
+The SGX device nodes have changed from `/dev/sgx/[enclave|provision]`
+to `/dev/sgx_[enclave|provision]` in v4x RFC patches according to the
+LKML feedback.
+
+Backwards compatibility is provided by adding `/dev/sgx` directory volume
+mount to containers. This assumes the cluster admin has installed the
+udev rules provided below to make the old device nodes as symlinks to the
+new device nodes.
+
+**Note:** the symlinks become visible in all containers requesting SGX
+resources but are potentially dangling links if the device the corresponding
+device resource is not requested.
+
+```bash
+$ cat /etc/udev/rules/9*.rules
+SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666"
+SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave"
+SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666"
+SUBSYSTEM=="misc",KERNEL=="provision",MODE="0660"
+SUBSYSTEM=="misc",KERNEL=="sgx_provision",SYMLINK+="sgx/provision",MODE="0660"
+SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660"
+$ sudo udevadm trigger
+$ ls -la /dev/sgx/*
+lrwxrwxrwx 1 root root 14 Nov 18 01:01 /dev/sgx/enclave -> ../sgx_enclave
+lrwxrwxrwx 1 root root 16 Nov 18 01:01 /dev/sgx/provision -> ../sgx_provision
+```
+
+The backwards compatibility will be removed in the next release (v0.20) and
+from the main development branch once the SGX SDK and DCAP releases default to
+the new devices.
+
 ### Pre-built images
 
 [Pre-built images](https://hub.docker.com/u/intel/)

cmd/sgx_plugin/sgx_plugin.go

@@ -68,9 +68,9 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 	devTree := dpapi.NewDeviceTree()
 
-	// Assume that both /dev/sgx/enclave and /dev/sgx/provision must be present.
-	sgxEnclavePath := path.Join(dp.devfsDir, "sgx", "enclave")
-	sgxProvisionPath := path.Join(dp.devfsDir, "sgx", "provision")
+	// Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present.
+	sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave")
+	sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision")
 	if _, err := os.Stat(sgxEnclavePath); err != nil {
 		klog.Error("No SGX enclave file available: ", err)
 		return devTree, nil
@@ -80,15 +80,22 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 		return devTree, nil
 	}
 
+	deprecatedMounts := []pluginapi.Mount{
+		{
+			HostPath:      "/dev/sgx",
+			ContainerPath: "/dev/sgx",
+		},
+	}
+
 	for i := uint(0); i < dp.nEnclave; i++ {
 		devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
 		nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
+		devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
 	}
 	for i := uint(0); i < dp.nProvision; i++ {
 		devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
 		nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
-		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
+		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
 	}
 	return devTree, nil
 }

cmd/sgx_plugin/sgx_plugin_test.go

@@ -103,31 +103,31 @@ func TestScan(t *testing.T) {
 		},
 		{
 			name:                  "only enclave file",
-			enclaveDevice:         "enclave",
+			enclaveDevice:         "sgx_enclave",
 			requestedEnclaveDevs:  1,
 			expectedEnclaveDevs:   0,
 			expectedProvisionDevs: 0,
 		},
 		{
 			name:                   "only provision file",
-			provisionDevice:        "provision",
+			provisionDevice:        "sgx_provision",
 			requestedProvisionDevs: 1,
 			expectedEnclaveDevs:    0,
 			expectedProvisionDevs:  0,
 		},
 		{
 			name:                   "one device",
-			enclaveDevice:          "enclave",
-			provisionDevice:        "provision",
+			enclaveDevice:          "sgx_enclave",
+			provisionDevice:        "sgx_provision",
 			requestedEnclaveDevs:   1,
 			expectedEnclaveDevs:    1,
 			requestedProvisionDevs: 1,
 			expectedProvisionDevs:  1,
 		},
 		{
 			name:                   "one device",
-			enclaveDevice:          "enclave",
-			provisionDevice:        "provision",
+			enclaveDevice:          "sgx_enclave",
+			provisionDevice:        "sgx_provision",
 			requestedEnclaveDevs:   10,
 			expectedEnclaveDevs:    10,
 			requestedProvisionDevs: 20,
@@ -144,20 +144,20 @@ func TestScan(t *testing.T) {
 			defer func() { _ = os.RemoveAll(root) }()
 
 			devfs := path.Join(root, "dev")
-			err = os.MkdirAll(path.Join(devfs, "sgx"), 0750)
+			err = os.MkdirAll(devfs, 0750)
 			if err != nil {
 				t.Fatalf("Failed to create fake device directory: %+v", err)
 			}
 			if tc.enclaveDevice != "" {
-				err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.enclaveDevice), []byte{}, 0600)
+				err = ioutil.WriteFile(path.Join(devfs, tc.enclaveDevice), []byte{}, 0600)
 				if err != nil {
-					t.Fatalf("Failed to create fake vendor file: %+v", err)
+					t.Fatalf("Failed to create fake enclave file: %+v", err)
 				}
 			}
 			if tc.provisionDevice != "" {
-				err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.provisionDevice), []byte{}, 0600)
+				err = ioutil.WriteFile(path.Join(devfs, tc.provisionDevice), []byte{}, 0600)
 				if err != nil {
-					t.Fatalf("Failed to create fake vendor file: %+v", err)
+					t.Fatalf("Failed to create fake provision file: %+v", err)
 				}
 			}
 

demo/sgx-sdk-demo/Dockerfile

@@ -23,8 +23,8 @@ RUN apt-get update && \
 # SGX SDK is installed in /opt/intel directory.
 WORKDIR /opt/intel
 
-ARG SGX_SDK_INSTALLER=sgx_linux_x64_sdk_2.11.100.2.bin
-ARG DCAP_VERSION=DCAP_1.8
+ARG SGX_SDK_INSTALLER=sgx_linux_x64_sdk_2.12.100.3.bin
+ARG DCAP_VERSION=DCAP_1.9
 
 RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bionic main" >> /etc/apt/sources.list.d/intel-sgx.list \
  && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add - \
@@ -35,7 +35,7 @@ RUN echo "deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu bio
     libsgx-quote-ex-dev
 
 # Install SGX SDK
-RUN wget https://download.01.org/intel-sgx/sgx-linux/2.11/distro/ubuntu18.04-server/$SGX_SDK_INSTALLER \
+RUN wget https://download.01.org/intel-sgx/sgx-linux/2.12/distro/ubuntu18.04-server/$SGX_SDK_INSTALLER \
  && chmod +x  $SGX_SDK_INSTALLER \
  && echo "yes" | ./$SGX_SDK_INSTALLER \
  && rm $SGX_SDK_INSTALLER

deployments/sgx_plugin/base/intel-sgx-plugin.yaml

@@ -34,13 +34,28 @@ spec:
         - name: sgxdevices
           mountPath: /dev/sgx
           readOnly: true
+        - name: sgx-enclave
+          mountPath: /dev/sgx_enclave
+          readOnly: true
+        - name: sgx-provision
+          mountPath: /dev/sgx_provision
+          readOnly: true
       volumes:
       - name: kubeletsockets
         hostPath:
           path: /var/lib/kubelet/device-plugins
       - name: sgxdevices
         hostPath:
           path: /dev/sgx
+          type: DirectoryOrCreate
+      - name: sgx-enclave
+        hostPath:
+          path: /dev/sgx_enclave
+          type: CharDevice
+      - name: sgx-provision
+        hostPath:
+          path: /dev/sgx_provision
+          type: CharDevice
       - name: nfd-source-hooks
         hostPath:
           path: /etc/kubernetes/node-feature-discovery/source.d/

pkg/controllers/sgx/controller.go

@@ -81,6 +81,7 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	}
 
 	yes := true
+	charDevice := v1.HostPathCharDev
 	directoryOrCreate := v1.HostPathDirectoryOrCreate
 	return &apps.DaemonSet{
 		ObjectMeta: metav1.ObjectMeta{
@@ -134,6 +135,16 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 									MountPath: "/dev/sgx",
 									ReadOnly:  true,
 								},
+								{
+									Name:      "sgx-enclave",
+									MountPath: "/dev/sgx_enclave",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "sgx-provision",
+									MountPath: "/dev/sgx_provision",
+									ReadOnly:  true,
+								},
 								{
 									Name:      "kubeletsockets",
 									MountPath: "/var/lib/kubelet/device-plugins",
@@ -148,6 +159,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 							VolumeSource: v1.VolumeSource{
 								HostPath: &v1.HostPathVolumeSource{
 									Path: "/dev/sgx",
+									Type: &directoryOrCreate,
+								},
+							},
+						},
+						{
+							Name: "sgx-enclave",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev/sgx_enclave",
+									Type: &charDevice,
+								},
+							},
+						},
+						{
+							Name: "sgx-provision",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/dev/sgx_provision",
+									Type: &charDevice,
 								},
 							},
 						},