Merge pull request #640 from uniemimu/froperator

gpu resource manager operator parts

Author: bart0sh

Makefile

@@ -91,6 +91,8 @@ generate:
 	$(CONTROLLER_GEN) webhook \
 		paths="./pkg/fpgacontroller/..." \
 		output:webhook:artifacts:config=deployments/fpga_admissionwebhook/webhook
+	$(CONTROLLER_GEN) rbac:roleName=gpu-manager-role paths="./cmd/gpu_plugin/..." output:dir=deployments/operator/rbac
+	cp deployments/operator/rbac/role.yaml deployments/operator/rbac/gpu_manager_role.yaml
 	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="./pkg/..." output:dir=deployments/operator/rbac
 	$(CONTROLLER_GEN) rbac:roleName=manager-role paths="./pkg/fpgacontroller/..." output:dir=deployments/fpga_admissionwebhook/rbac
 

deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml

@@ -68,6 +68,9 @@ spec:
                 description: NodeSelector provides a simple way to constrain device
                   plugin pods to nodes with particular labels.
                 type: object
+              resourceManager:
+                description: ResourceManager handles the fractional resource management for multi-GPU nodes
+                type: boolean
               sharedDevNum:
                 description: SharedDevNum is a number of containers that can share
                   the same GPU device.

deployments/operator/rbac/gpu_manager_role.yaml

@@ -0,0 +1,14 @@
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: gpu-manager-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list

deployments/operator/rbac/kustomization.yaml

@@ -3,6 +3,7 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+- gpu_manager_role.yaml
 # Comment the following 4 lines if you want to disable
 # the auth proxy (https://github.com/brancz/kube-rbac-proxy)
 # which protects your /metrics endpoint.

deployments/operator/rbac/role.yaml

@@ -14,6 +14,16 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -150,3 +160,13 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterrolebindings
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch

pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go

@@ -35,6 +35,9 @@ type GpuDevicePluginSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	SharedDevNum int `json:"sharedDevNum,omitempty"`
 
+	// ResourceManager handles the fractional resource management for multi-GPU nodes
+	ResourceManager bool `json:"resourceManager,omitempty"`
+
 	// LogLevel sets the plugin's log level.
 	// +kubebuilder:validation:Minimum=0
 	LogLevel int `json:"logLevel,omitempty"`

pkg/controllers/dsa/controller.go

@@ -57,6 +57,7 @@ func SetupReconciler(mgr ctrl.Manager, namespace string, withWebhook bool) error
 }
 
 type controller struct {
+	controllers.DefaultServiceAccountFactory
 	scheme *runtime.Scheme
 	ns     string
 }

pkg/controllers/fpga/controller.go

@@ -57,6 +57,7 @@ func SetupReconciler(mgr ctrl.Manager, namespace string, withWebhook bool) error
 }
 
 type controller struct {
+	controllers.DefaultServiceAccountFactory
 	scheme *runtime.Scheme
 	ns     string
 }

pkg/controllers/gpu/controller.go

@@ -23,6 +23,7 @@ import (
 
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/reference"
@@ -35,8 +36,9 @@ import (
 )
 
 const (
-	ownerKey = ".metadata.controller.gpu"
-	appLabel = "intel-gpu-plugin"
+	ownerKey           = ".metadata.controller.gpu"
+	appLabel           = "intel-gpu-plugin"
+	serviceAccountName = "gpu-manager-sa"
 )
 
 // +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=gpudeviceplugins,verbs=get;list;watch;create;update;patch;delete
@@ -74,6 +76,46 @@ func (c *controller) GetTotalObjectCount(ctx context.Context, clnt client.Client
 	return len(list.Items), nil
 }
 
+func (c *controller) NewServiceAccount(rawObj client.Object) *v1.ServiceAccount {
+	devicePlugin := rawObj.(*devicepluginv1.GpuDevicePlugin)
+	if devicePlugin.Spec.ResourceManager {
+		sa := v1.ServiceAccount{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "gpu-manager-sa",
+				Namespace: c.ns,
+			},
+		}
+		return &sa
+	}
+	return nil
+}
+
+func (c *controller) NewClusterRoleBinding(rawObj client.Object) *rbacv1.ClusterRoleBinding {
+	devicePlugin := rawObj.(*devicepluginv1.GpuDevicePlugin)
+	if devicePlugin.Spec.ResourceManager {
+		rb := rbacv1.ClusterRoleBinding{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "gpu-manager-rolebinding",
+				Namespace: c.ns,
+			},
+			Subjects: []rbacv1.Subject{
+				{
+					Kind:      "ServiceAccount",
+					Name:      "gpu-manager-sa",
+					Namespace: c.ns,
+				},
+			},
+			RoleRef: rbacv1.RoleRef{
+				Kind:     "ClusterRole",
+				Name:     "inteldeviceplugins-gpu-manager-role",
+				APIGroup: "rbac.authorization.k8s.io",
+			},
+		}
+		return &rb
+	}
+	return nil
+}
+
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.GpuDevicePlugin)
 
@@ -183,9 +225,46 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	if devicePlugin.Spec.InitImage != "" {
 		setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage)
 	}
+	// add service account if resource manager is enabled
+	if devicePlugin.Spec.ResourceManager {
+		daemonSet.Spec.Template.Spec.ServiceAccountName = serviceAccountName
+		addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory)
+		addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources")
+	}
 	return &daemonSet
 }
 
+func addVolumeMountIfMissing(spec *v1.PodSpec, name, mountPath string) {
+	for _, mount := range spec.Containers[0].VolumeMounts {
+		if mount.Name == name {
+			return
+		}
+	}
+
+	spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, v1.VolumeMount{
+		Name:      name,
+		MountPath: mountPath,
+	})
+}
+
+func addVolumeIfMissing(spec *v1.PodSpec, name, path string, hpType v1.HostPathType) {
+	for _, vol := range spec.Volumes {
+		if vol.Name == name {
+			return
+		}
+	}
+
+	spec.Volumes = append(spec.Volumes, v1.Volume{
+		Name: name,
+		VolumeSource: v1.VolumeSource{
+			HostPath: &v1.HostPathVolumeSource{
+				Path: path,
+				Type: &hpType,
+			},
+		},
+	})
+}
+
 func setInitContainer(spec *v1.PodSpec, imageName string) {
 	yes := true
 	spec.InitContainers = []v1.Container{
@@ -203,25 +282,7 @@ func setInitContainer(spec *v1.PodSpec, imageName string) {
 				},
 			},
 		}}
-	directoryOrCreate := v1.HostPathDirectoryOrCreate
-	missing := true
-	for _, vol := range spec.Volumes {
-		if vol.Name == "nfd-source-hooks" {
-			missing = false
-			break
-		}
-	}
-	if missing {
-		spec.Volumes = append(spec.Volumes, v1.Volume{
-			Name: "nfd-source-hooks",
-			VolumeSource: v1.VolumeSource{
-				HostPath: &v1.HostPathVolumeSource{
-					Path: "/etc/kubernetes/node-feature-discovery/source.d/",
-					Type: &directoryOrCreate,
-				},
-			},
-		})
-	}
+	addVolumeIfMissing(spec, "nfd-source-hooks", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate)
 }
 
 func removeVolume(volumes []v1.Volume, name string) []v1.Volume {
@@ -233,6 +294,15 @@ func removeVolume(volumes []v1.Volume, name string) []v1.Volume {
 	}
 	return newVolumes
 }
+func removeVolumeMount(volumeMounts []v1.VolumeMount, name string) []v1.VolumeMount {
+	newVolumeMounts := []v1.VolumeMount{}
+	for _, volume := range volumeMounts {
+		if volume.Name != name {
+			newVolumeMounts = append(newVolumeMounts, volume)
+		}
+	}
+	return newVolumeMounts
+}
 
 func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) {
 	dp := rawObj.(*devicepluginv1.GpuDevicePlugin)
@@ -269,6 +339,22 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	newServiceAccountName := "default"
+	if dp.Spec.ResourceManager {
+		newServiceAccountName = serviceAccountName
+	}
+	if ds.Spec.Template.Spec.ServiceAccountName != newServiceAccountName {
+		ds.Spec.Template.Spec.ServiceAccountName = newServiceAccountName
+		if dp.Spec.ResourceManager {
+			addVolumeIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory)
+			addVolumeMountIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources")
+		} else {
+			ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "podresources")
+			ds.Spec.Template.Spec.Containers[0].VolumeMounts = removeVolumeMount(ds.Spec.Template.Spec.Containers[0].VolumeMounts, "podresources")
+		}
+		updated = true
+	}
+
 	return updated
 }
 
@@ -313,5 +399,9 @@ func getPodArgs(gdp *devicepluginv1.GpuDevicePlugin) []string {
 		args = append(args, "-shared-dev-num", "1")
 	}
 
+	if gdp.Spec.ResourceManager {
+		args = append(args, "-resource-manager")
+	}
+
 	return args
 }

pkg/controllers/qat/controller.go

@@ -57,6 +57,7 @@ func SetupReconciler(mgr ctrl.Manager, namespace string, withWebhook bool) error
 }
 
 type controller struct {
+	controllers.DefaultServiceAccountFactory
 	scheme *runtime.Scheme
 	ns     string
 }