Merge pull request #211 from kad/opae-multiple-devices

Fix OPAE library to work with multiple cards

Author: bart0sh

demo/clearlinux-demo-opae/0001-OPAE-in-containers-don-t-enumerate-missing-device.patch

@@ -0,0 +1,98 @@
+From f36acdab907e4f97f6f273ef1e81bbdc7e504dfe Mon Sep 17 00:00:00 2001
+From: Alexander Kanevskiy <kad@linux.intel.com>
+Date: Mon, 26 Aug 2019 17:21:38 +0300
+Subject: [PATCH] OPAE in containers: don't enumerate missing device
+
+In case of container usages with FPGA, sysfs entries might have
+more information than actually available to the container.
+Ignore devices during enumeration that don't have valid /dev
+nodes.
+---
+ libopae/plugins/xfpga/enum.c           | 18 ++++++++++++++++++
+ tools/extra/fpgadiag/diag_utils.cpp    | 13 ++++++++-----
+ tools/extra/fpgadiag/perf_counters.cpp |  2 ++
+ 3 files changed, 28 insertions(+), 5 deletions(-)
+
+diff --git a/libopae/plugins/xfpga/enum.c b/libopae/plugins/xfpga/enum.c
+index 9ae72629..117fdfd0 100644
+--- a/libopae/plugins/xfpga/enum.c
++++ b/libopae/plugins/xfpga/enum.c
+@@ -343,6 +343,15 @@ STATIC fpga_result enum_fme(const char *sysfspath, const char *name,
+ 
+ 	snprintf_s_s(dpath, sizeof(dpath), FPGA_DEV_PATH "/%s", name);
+ 
++	// Make device node exists
++	if (stat(dpath, &stats) != 0) {
++		FPGA_MSG("stat failed: %s", strerror(errno));
++		return FPGA_OK;
++	}
++
++	if (!S_ISCHR(stats.st_mode))
++		return FPGA_OK;
++
+ 	pdev = add_dev(sysfspath, dpath, parent);
+ 	if (!pdev) {
+ 		FPGA_MSG("Failed to allocate device");
+@@ -421,6 +430,15 @@ STATIC fpga_result enum_afu(const char *sysfspath, const char *name,
+ 
+ 	snprintf_s_s(dpath, sizeof(dpath), FPGA_DEV_PATH "/%s", name);
+ 
++	// Make device node exists
++	if (stat(dpath, &stats) != 0) {
++		FPGA_MSG("stat failed: %s", strerror(errno));
++		return FPGA_OK;
++	}
++
++	if (!S_ISCHR(stats.st_mode))
++		return FPGA_OK;
++
+ 	pdev = add_dev(sysfspath, dpath, parent);
+ 	if (!pdev) {
+ 		FPGA_ERR("Failed to allocate device");
+diff --git a/tools/extra/fpgadiag/diag_utils.cpp b/tools/extra/fpgadiag/diag_utils.cpp
+index 2c98eb66..f661adc4 100644
+--- a/tools/extra/fpgadiag/diag_utils.cpp
++++ b/tools/extra/fpgadiag/diag_utils.cpp
+@@ -68,11 +68,14 @@ properties::ptr_t get_properties(intel::utils::option_map::ptr_t opts, fpga_objt
+ token::ptr_t get_parent_token(handle::ptr_t h)
+ {
+     auto props = properties::get(h);
+-
+-    auto tokens = token::enumerate({properties::get(props->parent)});
+-    if (!tokens.empty())
+-    {
+-        return tokens[0];
++    try {
++        auto tokens = token::enumerate({properties::get(props->parent)});
++        if (!tokens.empty())
++        {
++            return tokens[0];
++        }
++    }catch(not_found &) {
++        // Ignore FPGA_NOT_FOUND if process has access only to port and FME is not visible
+     }
+     return token::ptr_t();
+ }
+diff --git a/tools/extra/fpgadiag/perf_counters.cpp b/tools/extra/fpgadiag/perf_counters.cpp
+index de918dcb..c6af30ac 100644
+--- a/tools/extra/fpgadiag/perf_counters.cpp
++++ b/tools/extra/fpgadiag/perf_counters.cpp
+@@ -48,6 +48,7 @@ fpga_cache_counters::fpga_cache_counters(token::ptr_t fme)
+ : fme_(fme)
+ , perf_feature_rev_(-1)
+ {
++    if (!fme) return;
+     auto rev = sysobject::get(fme_, "*perf/revision", FPGA_OBJECT_GLOB);
+     if (rev) {
+         perf_feature_rev_ = rev->read64();
+@@ -218,6 +219,7 @@ fpga_fabric_counters::fpga_fabric_counters(token::ptr_t fme)
+ : fme_(fme)
+ , perf_feature_rev_(-1)
+ {
++   if (!fme) return;
+     auto rev = sysobject::get(fme_, "*perf/revision", FPGA_OBJECT_GLOB);
+     if (rev) {
+         perf_feature_rev_ = rev->read64();
+-- 
+2.16.4
+

demo/clearlinux-demo-opae/Dockerfile

@@ -17,14 +17,16 @@ RUN swupd update --no-boot-update ${CLEAR_LINUX_VERSION}
 # Fetch dependencies and source code
 ARG OPAE_RElEASE=1.3.2-1
 
-RUN swupd bundle-add wget c-basic devpkg-json-c devpkg-util-linux devpkg-hwloc doxygen Sphinx && \
+RUN swupd bundle-add wget c-basic devpkg-json-c devpkg-util-linux devpkg-hwloc doxygen Sphinx patch && \
     mkdir -p /usr/src/opae && \
     cd /usr/src/opae && \
     wget https://github.com/OPAE/opae-sdk/archive/${OPAE_RElEASE}.tar.gz && \
     tar xf *.tar.gz
 
 # Build OPAE
+ADD 0001-OPAE-in-containers-don-t-enumerate-missing-device.patch /usr/src/opae/opae-sdk-${OPAE_RElEASE}
 RUN cd /usr/src/opae/opae-sdk-${OPAE_RElEASE} && \
+    patch -p1 < 0001-OPAE-in-containers-don-t-enumerate-missing-device.patch && \
     mkdir build && \
     cd build && \
     cmake .. -DBUILD_ASE=0 -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_SKIP_RPATH=true && \