Merge pull request #638 from uniemimu/fractional

gpu_plugin: fractional resource management

Author: rojkov

DEVEL.md

@@ -64,6 +64,13 @@ Optionally, your device plugin may also implement the
 before they are sent to `kubelet`. To see an example, refer to the FPGA
 plugin which implements this interface to annotate its responses.
 
+In case you want to implement the whole allocation functionality in your
+device plugin, you can implement the optional `deviceplugin.Allocator`
+interface. In this case `PostAllocate()` is not called. But if you decide in your
+implementation of `deviceplugin.Allocator` that you need to resort to the default
+implementation of the allocation functionality then return an error of the type
+`deviceplugin.UseDefaultMethodError`.
+
 ### Logging
 
 The framework uses [`klog`](https://github.com/kubernetes/klog) as its logging

cmd/gpu_plugin/README.md

@@ -119,6 +119,30 @@ $ kubectl apply -k deployments/gpu_plugin/overlays/nfd_labeled_nodes
 daemonset.apps/intel-gpu-plugin created
 ```
 
+The experimental fractional-resource feature can be enabled by running:
+
+```bash
+$ kubectl apply -k deployments/gpu_plugin/overlays/fractional_resources
+serviceaccount/resource-reader-sa created
+clusterrole.rbac.authorization.k8s.io/resource-reader created
+clusterrolebinding.rbac.authorization.k8s.io/resource-reader-rb created
+daemonset.apps/intel-gpu-plugin created
+```
+
+Usage of fractional GPU resources, such as GPU memory, requires that the cluster has node
+extended resources with the name prefix `gpu.intel.com/`. Those can be created with NFD
+by running the hook installed by the plugin initcontainer. When fractional resources are
+enabled, the plugin lets a scheduler extender do card selection decisions based on resource
+availability and the amount of extended resources requested in the pod spec.
+
+The scheduler extender then needs to annotate the pod objects with unique
+increasing numeric timestamps in the annotation `gas-ts` and container card selections in
+`gas-container-cards` annotation. The latter has container separator `|` and card separator
+`,`. Example for a pod with two containers and both containers getting two cards:
+`gas-container-cards:card0,card1|card2,card3`. Enabling the fractional-resource support
+in the plugin without running such an annotation adding scheduler extender in the cluster
+will only slow down GPU-deployments, so do not enable this feature unnecessarily.
+
 > **Note**: It is also possible to run the GPU device plugin using a non-root user. To do this,
 the nodes' DAC rules must be configured to device plugin socket creation and kubelet registration.
 Furthermore, the deployments `securityContext` must be configured with appropriate `runAsUser/runAsGroup`.

cmd/gpu_plugin/gpu_plugin.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
 
@@ -51,8 +52,9 @@ const (
 )
 
 type cliOptions struct {
-	sharedDevNum     int
-	enableMonitoring bool
+	sharedDevNum       int
+	enableMonitoring   bool
+	resourceManagement bool
 }
 
 type devicePlugin struct {
@@ -66,10 +68,12 @@ type devicePlugin struct {
 
 	scanTicker *time.Ticker
 	scanDone   chan bool
+
+	resMan rm.ResourceManager
 }
 
 func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugin {
-	return &devicePlugin{
+	dp := &devicePlugin{
 		sysfsDir:         sysfsDir,
 		devfsDir:         devfsDir,
 		options:          options,
@@ -78,6 +82,17 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi
 		scanTicker:       time.NewTicker(scanPeriod),
 		scanDone:         make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
 	}
+
+	if options.resourceManagement {
+		var err error
+		dp.resMan, err = rm.NewResourceManager(monitorID, namespace+"/"+deviceType)
+		if err != nil {
+			klog.Errorf("Failed to create resource manager: %+v", err)
+			return nil
+		}
+	}
+
+	return dp
 }
 
 func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
@@ -131,6 +146,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 
 	var monitor []pluginapi.DeviceSpec
 	devTree := dpapi.NewDeviceTree()
+	rmDevInfos := rm.NewDeviceInfoMap()
 	for _, f := range files {
 		var nodes []pluginapi.DeviceSpec
 
@@ -179,6 +195,7 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 				// Currently only one device type (i915) is supported.
 				// TODO: check model ID to differentiate device models.
 				devTree.AddDevice(deviceType, devID, deviceInfo)
+				rmDevInfos[devID] = rm.NewDeviceInfo(nodes, nil, nil)
 			}
 		}
 	}
@@ -188,18 +205,36 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 		devTree.AddDevice(monitorType, monitorID, deviceInfo)
 	}
 
+	if dp.resMan != nil {
+		dp.resMan.SetDevInfos(rmDevInfos)
+	}
+
 	return devTree, nil
 }
 
+func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
+	if dp.resMan != nil {
+		return dp.resMan.ReallocateWithFractionalResources(request)
+	}
+
+	return nil, &dpapi.UseDefaultMethodError{}
+}
+
 func main() {
 	var opts cliOptions
 
 	flag.BoolVar(&opts.enableMonitoring, "enable-monitoring", false, "whether to enable 'i915_monitoring' (= all GPUs) resource")
+	flag.BoolVar(&opts.resourceManagement, "resource-manager", false, "fractional GPU resource management")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.Parse()
 
 	if opts.sharedDevNum < 1 {
-		klog.Warning("The number of containers sharing the same GPU must greater than zero")
+		klog.Error("The number of containers sharing the same GPU must greater than zero")
+		os.Exit(1)
+	}
+
+	if opts.sharedDevNum == 1 && opts.resourceManagement {
+		klog.Error("Trying to use fractional resources with shared-dev-num 1 is pointless")
 		os.Exit(1)
 	}
 

cmd/gpu_plugin/gpu_plugin_test.go

@@ -21,7 +21,9 @@ import (
 	"testing"
 
 	"github.com/pkg/errors"
+	"k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
 
@@ -43,6 +45,13 @@ func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
 	n.scanDone <- true
 }
 
+type mockResourceManager struct{}
+
+func (m *mockResourceManager) ReallocateWithFractionalResources(*v1beta1.AllocateRequest) (*v1beta1.AllocateResponse, error) {
+	return &v1beta1.AllocateResponse{}, &dpapi.UseDefaultMethodError{}
+}
+func (m *mockResourceManager) SetDevInfos(rm.DeviceInfoMap) {}
+
 func createTestFiles(root string, devfsdirs, sysfsdirs []string, sysfsfiles map[string][]byte) (string, string, error) {
 	sysfs := path.Join(root, "sys")
 	devfs := path.Join(root, "dev")
@@ -65,6 +74,30 @@ func createTestFiles(root string, devfsdirs, sysfsdirs []string, sysfsfiles map[
 	return sysfs, devfs, nil
 }
 
+func TestNewDevicePlugin(t *testing.T) {
+	if newDevicePlugin("", "", cliOptions{sharedDevNum: 2, resourceManagement: false}) == nil {
+		t.Error("Failed to create plugin")
+	}
+	if newDevicePlugin("", "", cliOptions{sharedDevNum: 2, resourceManagement: true}) != nil {
+		t.Error("Unexpectedly managed to create resource management enabled plugin inside unit tests")
+	}
+}
+
+func TestAllocate(t *testing.T) {
+	plugin := newDevicePlugin("", "", cliOptions{sharedDevNum: 2, resourceManagement: false})
+	_, err := plugin.Allocate(&v1beta1.AllocateRequest{})
+	if _, ok := err.(*dpapi.UseDefaultMethodError); !ok {
+		t.Errorf("Unexpected return value: %+v", err)
+	}
+
+	// mock the rm
+	plugin.resMan = &mockResourceManager{}
+	_, err = plugin.Allocate(&v1beta1.AllocateRequest{})
+	if _, ok := err.(*dpapi.UseDefaultMethodError); !ok {
+		t.Errorf("Unexpected return value: %+v", err)
+	}
+}
+
 func TestScan(t *testing.T) {
 	tcases := []struct {
 		name string
@@ -203,6 +236,8 @@ func TestScan(t *testing.T) {
 				scanDone: plugin.scanDone,
 			}
 
+			plugin.resMan = &mockResourceManager{}
+
 			err = plugin.Scan(notifier)
 			// Scans in GPU plugin never fail
 			if err != nil {