Merge branch 'fix/redis' into 'master'

Fix/redis

See merge request grafolean/grafolean-collector-snmp!8

Author: grafolean

.gitignore

@@ -1,2 +1,4 @@
 .env
 .vscode
+.cache
+__pycache__

.gitlab-ci.yml

@@ -13,12 +13,15 @@ stages:
 pytest:
   stage: test
   image: python:3.6-slim-stretch
+  services:
+    - redis:5-alpine
   before_script:
     - apt-get update
     - apt-get install --no-install-recommends -q -y libsnmp-dev build-essential git
     - pip install --no-cache-dir pipenv
     - pipenv install --dev
   script:
+    - export REDIS_HOST="redis"
     - pipenv run pytest -x test_snmpcollector.py
 
 deploy to docker hub:

Pipfile

@@ -15,6 +15,7 @@ ansicolors = "*"
 easysnmp = {editable = true,git = "http://github.com/grafolean/easysnmp"}
 mathjspy = "*"
 numpy = "*"
+redis = "*"
 
 [requires]
 python_version = "3.6"

Pipfile.lock

@@ -236,6 +236,7 @@ def fetch_job_configs(self, protocol):
 
                     sensors.append({
                         "sensor_details": sensor["details"],
+                        "sensor_id": sensor_id,
                         "interval": interval,
                     })
                 # and hide all other protocols, saving just sensors for selected one: (not strictly necessary, just cleaner)

collector.py

@@ -32,3 +32,12 @@ services:
     # on the same host, '127.0.0.1' translates to container, not host - this directive changes network mode
     # so that Docker networking is bypassed.
     network_mode: "host"
+
+  redis:
+    image: redis:5-alpine
+    container_name: grafolean-collector-snmp-redis
+    ports:
+      - "127.0.0.1:6379:6379"
+    # We advise not to use `network_mode: "host"` in production, because it would expose Redis to host network
+    # (even if access is limited to 127.0.0.1).
+    network_mode: "host"

docker-compose.dev.yml

@@ -16,13 +16,16 @@ services:
     environment:
       # Backend url must be set to the address of the Grafolean backend, for example this uses Grafolean hosted service:
       # - BACKEND_URL=https://grafolean.com/api
+      # IMPORTANT: '127.0.0.1' and 'localhost' are _never_ correct addresses for Grafolean backend, because they translate
+      # to container, not host.
       - BACKEND_URL=
       # To use SNMP Collector, a bot with the protocol "snmp" must be added via user interface, then the token needs to be copied here:
       - BOT_TOKEN=
       # Interval between fetching information about jobs:
       - JOBS_REFRESH_INTERVAL=60
+      - REDIS_HOST=redis
     restart: always
-    # Grafolean backend must be accessible on BACKEND_URL *from the point of view of container*. If running
-    # on the same host, '127.0.0.1' translates to container, not host - this directive changes network mode
-    # so that Docker networking is bypassed.
-    network_mode: "host"
+
+  redis:
+    image: redis:5-alpine
+    container_name: grafolean-collector-snmp-redis

docker-compose.yml

@@ -6,6 +6,7 @@
 from pytz import utc
 from colors import color
 import requests
+import redis
 
 from easysnmp import Session, SNMPVariable
 from mathjspy import MathJS
@@ -26,18 +27,19 @@ class NoValueForOid(Exception):
     pass
 
 
-previous_counter_values = {}
+REDIS_HOST = os.environ.get('REDIS_HOST', '127.0.0.1')
+r = redis.Redis(host=REDIS_HOST)
 
 
 def _get_previous_counter_value(counter_ident):
-    prev_value = previous_counter_values.get(counter_ident)
-    if prev_value is None:
+    prev_value = r.hgetall(counter_ident)
+    if not prev_value:  # empty dict
         return None, None
-    return prev_value
+    return int(prev_value[b'v']), float(prev_value[b't'])
 
 
 def _save_current_counter_value(new_value, now, counter_ident):
-    previous_counter_values[counter_ident] = (new_value, now)
+    r.hmset(counter_ident, {b'v': new_value, b't': now})
 
 
 def _convert_counters_to_values(results, now, counter_ident_prefix):
@@ -52,7 +54,7 @@ def _convert_counters_to_values(results, now, counter_ident_prefix):
         # counter - deal with it:
         counter_ident = counter_ident_prefix + f'/{i}/{v.oid}/{v.oid_index}'
         old_value, t = _get_previous_counter_value(counter_ident)
-        new_value = float(v.value)
+        new_value = int(float(v.value))
         _save_current_counter_value(new_value, now, counter_ident)
         if old_value is None:
             new_results.append(SNMPVariable(oid=v.oid, oid_index=v.oid_index, value=None, snmp_type='COUNTER_PER_S'))
@@ -233,7 +235,7 @@ def do_snmp(*args, **job_info):
                         walk_indexes = [r.oid_index for r in result]
             log.info("Results: {}".format(list(zip(oids, methods, results))))
 
-            counter_ident_prefix = f'{job_info["entity_id"]}/{sensor["sensor_details"]["id"]}'
+            counter_ident_prefix = f'{job_info["entity_id"]}/{sensor["sensor_id"]}'
             results_no_counters = _convert_counters_to_values(results, time.time(), counter_ident_prefix)
 
             # We have SNMP results and expression - let's calculate value(s). The trick here is that