Merge pull request #185 from mythi/iommu

qat_plugin: kerneldrv: register VF devices when IOMMU is on

Author: Dmitry Rozhkov

cmd/qat_plugin/kerneldrv/kerneldrv.go

@@ -134,7 +134,7 @@ func newDevicePlugin(configDir string, execer utilsexec.Interface) *DevicePlugin
 	}
 }
 
-func (dp *DevicePlugin) getOnlineDevices() ([]device, error) {
+func (dp *DevicePlugin) getOnlineDevices(iommuOn bool) ([]device, error) {
 	outputBytes, err := dp.execer.Command("adf_ctl", "status").CombinedOutput()
 	if err != nil {
 		return nil, errors.Wrapf(err, "Can't get driver status")
@@ -152,6 +152,11 @@ func (dp *DevicePlugin) getOnlineDevices() ([]device, error) {
 			continue
 		}
 
+		// "Cannot use PF with IOMMU enabled"
+		if iommuOn != strings.HasSuffix(matches[1], "vf") {
+			continue
+		}
+
 		devices = append(devices, device{
 			id:      fmt.Sprintf("dev%s", matches[2]),
 			devtype: matches[1],
@@ -281,10 +286,28 @@ func (drvConfig driverConfig) update(devID string, iniSection *ini.Section) erro
 	return nil
 }
 
+func getIOMMUStatus() (bool, error) {
+	iommus, err := ioutil.ReadDir("/sys/class/iommu/")
+	if err != nil {
+		return false, errors.Wrapf(err, "Unable to read IOMMU status")
+	}
+
+	if len(iommus) > 0 {
+		return true, nil
+	}
+
+	return false, nil
+}
+
 // Scan implements Scanner interface for kernel based QAT plugin.
 func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
 	for {
-		devices, err := dp.getOnlineDevices()
+		iommuOn, err := getIOMMUStatus()
+		if err != nil {
+			return err
+		}
+
+		devices, err := dp.getOnlineDevices(iommuOn)
 		if err != nil {
 			return err
 		}

cmd/qat_plugin/kerneldrv/kerneldrv_test.go

@@ -44,6 +44,16 @@ There is 3 QAT acceleration device(s) in the system:
  qat_dev0 - type: c6xx,  inst_id: 0,  node_id: 0,  bsf: 3b:00.0,  #accel: 5 #engines: 10 state: up
  qat_dev1 - type: c6xx,  inst_id: 1,  node_id: 0,  bsf: 3d:00.0,  #accel: 5 #engines: 10 state: down
  qat_dev2 - type: c6xx,  inst_id: 2,  node_id: 3,  bsf: d8:00.0,  #accel: 5 #engines: 10 state: up
+`
+	adfCtlOutputVf = `Checking status of all devices.
+There is 7 QAT acceleration device(s) in the system:
+ qat_dev0 - type: c6xx,  inst_id: 0,  node_id: 0,  bsf: 0000:3b:00.0,  #accel: 5 #engines: 10 state: up
+ qat_dev1 - type: c6xx,  inst_id: 1,  node_id: 0,  bsf: 0000:3b:00.0,  #accel: 5 #engines: 10 state: up
+ qat_dev2 - type: c6xx,  inst_id: 2,  node_id: 3,  bsf: 0000:3b:00.0,  #accel: 5 #engines: 10 state: up
+ qat_dev3 - type: c6xxvf,  inst_id: 0,  node_id: 0,  bsf: 0000:3b:01.0,  #accel: 1 #engines: 1 state: up
+ qat_dev4 - type: c6xxvf,  inst_id: 1,  node_id: 0,  bsf: 0000:3b:01.1,  #accel: 1 #engines: 1 state: up
+ qat_dev5 - type: c6xxvf,  inst_id: 2,  node_id: 0,  bsf: 0000:3b:01.2,  #accel: 1 #engines: 1 state: up
+ qat_dev6 - type: c6xxvf,  inst_id: 3,  node_id: 0,  bsf: 0000:3b:01.3,  #accel: 1 #engines: 1 state: up
 `
 )
 
@@ -58,16 +68,25 @@ func TestGetOnlineDevices(t *testing.T) {
 		adfCtlError    error
 		expectedDevNum int
 		expectedErr    bool
+		iommuOn        bool
 	}{
 		{
 			name:           "all is good",
 			adfCtlOutput:   adfCtlOutput,
 			expectedDevNum: 3,
+			iommuOn:        false,
 		},
 		{
 			name:           "one device is down",
 			adfCtlOutput:   adfCtlOutputOneDown,
 			expectedDevNum: 2,
+			iommuOn:        false,
+		},
+		{
+			name:           "virtual functions enabled",
+			adfCtlOutput:   adfCtlOutputVf,
+			expectedDevNum: 4,
+			iommuOn:        true,
 		},
 		{
 			name:        "adf_ctl fails to run",
@@ -94,7 +113,7 @@ func TestGetOnlineDevices(t *testing.T) {
 			dp := &DevicePlugin{
 				execer: &execer,
 			}
-			devices, err := dp.getOnlineDevices()
+			devices, err := dp.getOnlineDevices(tt.iommuOn)
 			if tt.expectedErr && err == nil {
 				t.Error("Expected error hasn't been triggered")
 			}