Skip to content

Commit d65cb90

Browse files
mythimythi
committed
sgx: move to RFC v4x device API
The SGX device nodes have changed from /dev/sgx/[enclave|provision] to /dev/sgx_[enclave|provision] in v4x RFC patches according to the LKML feedback. This changes moves to use the new device nodes. Backwards compatibility is provided by adding /dev/sgx directory mount to containers. This assumes the cluster admin has installed the udev rules provided in the README to make the old device nodes as symlinks to the new device nodes. Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>
1 parent e4dc4c1 commit d65cb90

File tree

5 files changed

+104
-18
lines changed

5 files changed

+104
-18
lines changed

‎cmd/sgx_plugin/README.md‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Contents
55
* [Introduction](#introduction)
66
* [Installation](#installation)
77
* [Prerequisites](#prerequisites)
8+
* [Backwards compatiblity note](#backwards-compatibility-note)
89
* [Pre-built images](#pre-built-images)
910
* [Getting the source code](#getting-the-source-code)
1011
* [Verify node kubelet config](#verify-node-kubelet-config)
@@ -63,11 +64,44 @@ The component has the same basic dependancies as the
6364
[generic plugin framework dependencies](../../README.md#about).
6465

6566
The SGX device plugin requires Linux Kernel SGX drivers to be available. These drivers
66-
are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git/tag/?h=v39).
67-
RFC *v39* was used to validate what is written in this document.
67+
are currently available via [RFC patches on Linux Kernel Mailing List](https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git).
68+
RFC *v41* was used to validate what is written in this document.
6869

6970
The hardware platform must support SGX Flexible Launch Control.
7071

72+
#### Backwards compatibility note
73+
74+
The SGX device nodes have changed from `/dev/sgx/[enclave|provision]`
75+
to `/dev/sgx_[enclave|provision]` in v4x RFC patches according to the
76+
LKML feedback.
77+
78+
Backwards compatibility is provided by adding `/dev/sgx` directory volume
79+
mount to containers. This assumes the cluster admin has installed the
80+
udev rules provided below to make the old device nodes as symlinks to the
81+
new device nodes.
82+
83+
**Note:** the symlinks become visible in all containers requesting SGX
84+
resources but are potentially dangling links if the device the corresponding
85+
device resource is not requested.
86+
87+
```bash
88+
$ cat /etc/udev/rules/9*.rules
89+
SUBSYSTEM=="misc",KERNEL=="enclave",MODE="0666"
90+
SUBSYSTEM=="misc",KERNEL=="sgx_enclave",MODE="0666",SYMLINK+="sgx/enclave"
91+
SUBSYSTEM=="sgx",KERNEL=="sgx/enclave",MODE="0666"
92+
SUBSYSTEM=="misc",KERNEL=="provision",MODE="0660"
93+
SUBSYSTEM=="misc",KERNEL=="sgx_provision",SYMLINK+="sgx/provision",MODE="0660"
94+
SUBSYSTEM=="sgx",KERNEL=="sgx/provision",MODE="0660"
95+
$ sudo udevadm trigger
96+
$ ls -la /dev/sgx/*
97+
lrwxrwxrwx 1 root root 14 Nov 18 01:01 /dev/sgx/enclave -> ../sgx_enclave
98+
lrwxrwxrwx 1 root root 16 Nov 18 01:01 /dev/sgx/provision -> ../sgx_provision
99+
```
100+
101+
The backwards compatibility will be removed in the next release (v0.20) and
102+
from the main development branch once the SGX SDK and DCAP releases default to
103+
the new devices.
104+
71105
### Pre-built images
72106

73107
[Pre-built images](https://hub.docker.com/u/intel/)

‎cmd/sgx_plugin/sgx_plugin.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
6868
func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
6969
devTree := dpapi.NewDeviceTree()
7070

71-
// Assume that both /dev/sgx/enclave and /dev/sgx/provision must be present.
72-
sgxEnclavePath := path.Join(dp.devfsDir, "sgx", "enclave")
73-
sgxProvisionPath := path.Join(dp.devfsDir, "sgx", "provision")
71+
// Assume that both /dev/sgx_enclave and /dev/sgx_provision must be present.
72+
sgxEnclavePath := path.Join(dp.devfsDir, "sgx_enclave")
73+
sgxProvisionPath := path.Join(dp.devfsDir, "sgx_provision")
7474
if _, err := os.Stat(sgxEnclavePath); err != nil {
7575
klog.Error("No SGX enclave file available: ", err)
7676
return devTree, nil
@@ -80,15 +80,22 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
8080
return devTree, nil
8181
}
8282

83+
deprecatedMounts := []pluginapi.Mount{
84+
{
85+
HostPath: "/dev/sgx",
86+
ContainerPath: "/dev/sgx",
87+
},
88+
}
89+
8390
for i := uint(0); i < dp.nEnclave; i++ {
8491
devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
8592
nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
86-
devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
93+
devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
8794
}
8895
for i := uint(0); i < dp.nProvision; i++ {
8996
devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
9097
nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
91-
devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil))
98+
devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, deprecatedMounts, nil))
9299
}
93100
return devTree, nil
94101
}

‎cmd/sgx_plugin/sgx_plugin_test.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -103,31 +103,31 @@ func TestScan(t *testing.T) {
103103
},
104104
{
105105
name: "only enclave file",
106-
enclaveDevice: "enclave",
106+
enclaveDevice: "sgx_enclave",
107107
requestedEnclaveDevs: 1,
108108
expectedEnclaveDevs: 0,
109109
expectedProvisionDevs: 0,
110110
},
111111
{
112112
name: "only provision file",
113-
provisionDevice: "provision",
113+
provisionDevice: "sgx_provision",
114114
requestedProvisionDevs: 1,
115115
expectedEnclaveDevs: 0,
116116
expectedProvisionDevs: 0,
117117
},
118118
{
119119
name: "one device",
120-
enclaveDevice: "enclave",
121-
provisionDevice: "provision",
120+
enclaveDevice: "sgx_enclave",
121+
provisionDevice: "sgx_provision",
122122
requestedEnclaveDevs: 1,
123123
expectedEnclaveDevs: 1,
124124
requestedProvisionDevs: 1,
125125
expectedProvisionDevs: 1,
126126
},
127127
{
128128
name: "one device",
129-
enclaveDevice: "enclave",
130-
provisionDevice: "provision",
129+
enclaveDevice: "sgx_enclave",
130+
provisionDevice: "sgx_provision",
131131
requestedEnclaveDevs: 10,
132132
expectedEnclaveDevs: 10,
133133
requestedProvisionDevs: 20,
@@ -144,20 +144,20 @@ func TestScan(t *testing.T) {
144144
defer func() { _ = os.RemoveAll(root) }()
145145

146146
devfs := path.Join(root, "dev")
147-
err = os.MkdirAll(path.Join(devfs, "sgx"), 0750)
147+
err = os.MkdirAll(devfs, 0750)
148148
if err != nil {
149149
t.Fatalf("Failed to create fake device directory: %+v", err)
150150
}
151151
if tc.enclaveDevice != "" {
152-
err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.enclaveDevice), []byte{}, 0600)
152+
err = ioutil.WriteFile(path.Join(devfs, tc.enclaveDevice), []byte{}, 0600)
153153
if err != nil {
154-
t.Fatalf("Failed to create fake vendor file: %+v", err)
154+
t.Fatalf("Failed to create fake enclave file: %+v", err)
155155
}
156156
}
157157
if tc.provisionDevice != "" {
158-
err = ioutil.WriteFile(path.Join(devfs, "sgx", tc.provisionDevice), []byte{}, 0600)
158+
err = ioutil.WriteFile(path.Join(devfs, tc.provisionDevice), []byte{}, 0600)
159159
if err != nil {
160-
t.Fatalf("Failed to create fake vendor file: %+v", err)
160+
t.Fatalf("Failed to create fake provision file: %+v", err)
161161
}
162162
}
163163

‎deployments/sgx_plugin/base/intel-sgx-plugin.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,28 @@ spec:
3434
- name: sgxdevices
3535
mountPath: /dev/sgx
3636
readOnly: true
37+
- name: sgx-enclave
38+
mountPath: /dev/sgx_enclave
39+
readOnly: true
40+
- name: sgx-provision
41+
mountPath: /dev/sgx_provision
42+
readOnly: true
3743
volumes:
3844
- name: kubeletsockets
3945
hostPath:
4046
path: /var/lib/kubelet/device-plugins
4147
- name: sgxdevices
4248
hostPath:
4349
path: /dev/sgx
50+
type: DirectoryOrCreate
51+
- name: sgx-enclave
52+
hostPath:
53+
path: /dev/sgx_enclave
54+
type: CharDevice
55+
- name: sgx-provision
56+
hostPath:
57+
path: /dev/sgx_provision
58+
type: CharDevice
4459
- name: nfd-source-hooks
4560
hostPath:
4661
path: /etc/kubernetes/node-feature-discovery/source.d/

‎pkg/controllers/sgx/controller.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
8181
}
8282

8383
yes := true
84+
charDevice := v1.HostPathCharDev
8485
directoryOrCreate := v1.HostPathDirectoryOrCreate
8586
return &apps.DaemonSet{
8687
ObjectMeta: metav1.ObjectMeta{
@@ -134,6 +135,16 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
134135
MountPath: "/dev/sgx",
135136
ReadOnly: true,
136137
},
138+
{
139+
Name: "sgx-enclave",
140+
MountPath: "/dev/sgx_enclave",
141+
ReadOnly: true,
142+
},
143+
{
144+
Name: "sgx-provision",
145+
MountPath: "/dev/sgx_provision",
146+
ReadOnly: true,
147+
},
137148
{
138149
Name: "kubeletsockets",
139150
MountPath: "/var/lib/kubelet/device-plugins",
@@ -148,6 +159,25 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
148159
VolumeSource: v1.VolumeSource{
149160
HostPath: &v1.HostPathVolumeSource{
150161
Path: "/dev/sgx",
162+
Type: &directoryOrCreate,
163+
},
164+
},
165+
},
166+
{
167+
Name: "sgx-enclave",
168+
VolumeSource: v1.VolumeSource{
169+
HostPath: &v1.HostPathVolumeSource{
170+
Path: "/dev/sgx_enclave",
171+
Type: &charDevice,
172+
},
173+
},
174+
},
175+
{
176+
Name: "sgx-provision",
177+
VolumeSource: v1.VolumeSource{
178+
HostPath: &v1.HostPathVolumeSource{
179+
Path: "/dev/sgx_provision",
180+
Type: &charDevice,
151181
},
152182
},
153183
},

0 commit comments

Comments
 (0)