From 537d7abdb6e5e3383cd99ebfbe6076fb6781df87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Fri, 9 Mar 2018 16:35:17 +0100 Subject: [PATCH 01/39] Changed file structure, made module and updated for LoBo-port. Changed structure to be clearly arranged and to make it usable as module. Updated mqtt_as to not use some workarounds needed for ESP32 with loboris port. Updated README. --- .gitignore | 2 + README.md | 4 +- mqtt_as/README.md => README_mqtt_as.md | 19 +++-- __init__.py | 0 mqtt_as/config.py => config.py | 0 mqtt_as/mqtt_as.py => mqtt_as.py | 75 ++++++++++-------- NO_NET.md => remote_mqtt/NO_NET.md | 0 _boot.py => remote_mqtt/_boot.py | 0 .../firmware-combined.bin | Bin main.py => remote_mqtt/main.py | 0 mqtt.py => remote_mqtt/mqtt.py | 0 net_local.py => remote_mqtt/net_local.py | 0 pb_simple.py => remote_mqtt/pb_simple.py | 0 pb_status.py => remote_mqtt/pb_status.py | 0 pbmqtt.py => remote_mqtt/pbmqtt.py | 0 pbmqtt_test.py => remote_mqtt/pbmqtt_test.py | 0 pbrange.py => remote_mqtt/pbrange.py | 0 .../status_values.py | 0 syncom.py => remote_mqtt/syncom.py | 0 pubtest => remote_mqtt/tests/pubtest | 0 .../tests/pubtest_range | 0 asyn.py => tests/asyn.py | 0 {mqtt_as => tests}/clean.py | 0 {mqtt_as => tests}/main.py | 0 {mqtt_as => tests}/pubtest | 0 {mqtt_as => tests}/range.py | 0 {mqtt_as => tests}/ssl.py | 0 {mqtt_as => tests}/unclean.py | 0 28 files changed, 59 insertions(+), 41 deletions(-) rename mqtt_as/README.md => README_mqtt_as.md (95%) create mode 100644 __init__.py rename mqtt_as/config.py => config.py (100%) rename mqtt_as/mqtt_as.py => mqtt_as.py (95%) rename NO_NET.md => remote_mqtt/NO_NET.md (100%) rename _boot.py => remote_mqtt/_boot.py (100%) rename firmware-combined.bin => remote_mqtt/firmware-combined.bin (100%) rename main.py => remote_mqtt/main.py (100%) rename mqtt.py => remote_mqtt/mqtt.py (100%) rename net_local.py => remote_mqtt/net_local.py (100%) rename pb_simple.py => remote_mqtt/pb_simple.py (100%) rename pb_status.py => remote_mqtt/pb_status.py (100%) rename pbmqtt.py => remote_mqtt/pbmqtt.py (100%) rename pbmqtt_test.py => remote_mqtt/pbmqtt_test.py (100%) rename pbrange.py => remote_mqtt/pbrange.py (100%) rename status_values.py => remote_mqtt/status_values.py (100%) rename syncom.py => remote_mqtt/syncom.py (100%) rename pubtest => remote_mqtt/tests/pubtest (100%) rename pubtest_range => remote_mqtt/tests/pubtest_range (100%) rename asyn.py => tests/asyn.py (100%) rename {mqtt_as => tests}/clean.py (100%) rename {mqtt_as => tests}/main.py (100%) rename {mqtt_as => tests}/pubtest (100%) rename {mqtt_as => tests}/range.py (100%) rename {mqtt_as => tests}/ssl.py (100%) rename {mqtt_as => tests}/unclean.py (100%) diff --git a/.gitignore b/.gitignore index 7bbc71c..01e67db 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,5 @@ ENV/ # mypy .mypy_cache/ +/.project +/.pydevproject diff --git a/README.md b/README.md index 2632a4c..2b805bf 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Its main drawback is code size. Run as frozen bytecode it uses about 50% of the RAM on the ESP8266. On ESP32 it runs as a standard Python module with 64K of RAM free. -It is documented [here](./mqtt_as/README.md). +It is documented [here](./README_mqtt_as.md). ## 2. MQTT for generic MicroPython targets @@ -49,4 +49,4 @@ five GPIO pins accessible via the `machine` library should suffice. The driver is non-blocking and is designed for applications using uasyncio. -It is documented [here](./NO_NET.md). +It is documented [here](./remote_mqtt/NO_NET.md). diff --git a/mqtt_as/README.md b/README_mqtt_as.md similarity index 95% rename from mqtt_as/README.md rename to README_mqtt_as.md index d6eb900..28d2e06 100644 --- a/mqtt_as/README.md +++ b/README_mqtt_as.md @@ -10,7 +10,7 @@ but duplication can occur. Level 2 avoids duplication; it is unsuported by the official driver and by this module. Duplicates can readily be handled at the application level. -###### [Main README](../README.md) +###### [Main README](./README.md) ## 1.1 Rationale @@ -97,12 +97,15 @@ The module works without recourse to cross compilation or frozen bytecode. 1. `mqtt_as.py` The main module. 2. `config.py` Stores cross-project settings. - 3. `clean.py` Test/demo program using MQTT Clean Session. - 4. `unclean.py` Test/demo program with MQTT Clean Session `False`. - 5. `range.py` For WiFi range testing. - 6. `pubtest` Bash script illustrating publication with Mosquitto. - 7. `main.py` Example for auto-starting an application. - 8. `ssl.py` Failed attempt to run with SSL. See note above (1.3). + 3. `remote_mqtt` Folder containing all files of mqtt for platforms without WIFI + 4. `sonoff` Folder containing test files for sonoff devices + 5. `tests` Folder containing test files for mqtt_as + 5.1. `clean.py` Test/demo program using MQTT Clean Session. + 5.2. `unclean.py` Test/demo program with MQTT Clean Session `False`. + 5.3. `range.py` For WiFi range testing. + 5.4. `pubtest` Bash script illustrating publication with Mosquitto. + 5.5. `main.py` Example for auto-starting an application. + 5.6. `ssl.py` Failed attempt to run with SSL. See note above (1.3). ## 2.2 Installation @@ -322,7 +325,7 @@ connectivity has been lost if no messages have been received in that period. The module attempts to keep the connection open by issuing an MQTT ping upto four times during the keepalive interval. (It pings if the last response from the broker was over 1/4 of the keepalive period). More frequent pings may be -desirable to educe latency in subscribe-only applications. This may be achieved +desirable to reduce latency in subscribe-only applications. This may be achieved using the `ping_interval` configuration option. If the broker times out it will issue the "last will" publication (if any). diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mqtt_as/config.py b/config.py similarity index 100% rename from mqtt_as/config.py rename to config.py diff --git a/mqtt_as/mqtt_as.py b/mqtt_as.py similarity index 95% rename from mqtt_as/mqtt_as.py rename to mqtt_as.py index d7ac39a..fc85484 100644 --- a/mqtt_as/mqtt_as.py +++ b/mqtt_as.py @@ -25,14 +25,17 @@ # Legitimate errors while waiting on a socket. See uasyncio __init__.py open_connection(). BUSY_ERRORS = [EINPROGRESS, ETIMEDOUT] -ESP32 = platform == 'esp32' +ESP32 = platform == 'esp32' or platform == 'esp32_LoBo' # Set up special handling for sonoff and similar devices requiring periodic yield to RTOS SONOFF = False + + def sonoff(): global SONOFF SONOFF = True + # ESP32. It is not enough to regularly yield to RTOS with machine.idle(). There are # two cases where an explicit sleep() is required. Where data has been written to the # socket and a response is awaited, a timeout may occur without a >= 20ms sleep. @@ -41,47 +44,57 @@ def sonoff(): # https://forum.micropython.org/viewtopic.php?f=16&t=3608&p=20942#p20942 BUSY_ERRORS += [118, 119] # Add in weird ESP32 errors # 20ms seems about the minimum before we miss data read from a socket. - def esp32_pause(): # https://github.com/micropython/micropython-esp32/issues/167 - sleep_ms(20) + + if platform == 'esp32_LoBo': + esp32_pause = lambda *_: None + else: + def esp32_pause(): # https://github.com/micropython/micropython-esp32/issues/167 + sleep_ms(20) else: - esp32_pause = lambda *_ : None + esp32_pause = lambda *_: None # Default "do little" coro for optional user replacement + + async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program await asyncio.sleep_ms(_DEFAULT_MS) config = { - 'client_id' : hexlify(unique_id()), - 'server' : None, - 'port' : 0, - 'user' : '', - 'password' : '', - 'keepalive' : 60, - 'ping_interval' : 0, - 'ssl' : False, - 'ssl_params' : {}, - 'response_time' : 10, - 'clean_init' : True, - 'clean' : True, - 'max_repubs' : 4, - 'will' : None, - 'subs_cb' : lambda *_ : None, - 'wifi_coro' : eliza, - 'connect_coro' : eliza, - 'ssid' : None, - 'wifi_pw' : None, - } + 'client_id': hexlify(unique_id()), + 'server': None, + 'port': 0, + 'user': '', + 'password': '', + 'keepalive': 60, + 'ping_interval': 0, + 'ssl': False, + 'ssl_params': {}, + 'response_time': 10, + 'clean_init': True, + 'clean': True, + 'max_repubs': 4, + 'will': None, + 'subs_cb': lambda *_: None, + 'wifi_coro': eliza, + 'connect_coro': eliza, + 'ssid': None, + 'wifi_pw': None, +} + class MQTTException(Exception): pass + def newpid(pid): return pid + 1 if pid < 65535 else 1 + def qos_check(qos): if not (qos == 0 or qos == 1): raise ValueError('Only qos 0 and 1 are supported.') + class Lock(): def __init__(self): self._locked = False @@ -104,6 +117,7 @@ async def __aexit__(self, *args): class MQTT_base: REPUB_COUNT = 0 # TEST DEBUG = False + def __init__(self, config): # MQTT config self._client_id = config['client_id'] @@ -130,7 +144,7 @@ def __init__(self, config): self._cb = config['subs_cb'] self._wifi_handler = config['wifi_coro'] self._connect_handler = config['connect_coro'] - # Network + # Network self.port = config['port'] if self.port == 0: self.port = 8883 if self._ssl else 1883 @@ -146,7 +160,7 @@ def __init__(self, config): self.suback = False self.last_rx = ticks_ms() # Time of last communication from broker self.lock = Lock() - if ESP32: + if ESP32 and platform != 'esp32_LoBo': loop = asyncio.get_event_loop() loop.create_task(self._idle_task()) @@ -287,7 +301,7 @@ async def _ping(self): await self._as_write(b"\xc0\0") # Check internet connectivity by sending DNS lookup to Google's 8.8.8.8 - async def wan_ok(self, packet = b'$\x1a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x06google\x03com\x00\x00\x01\x00\x01'): + async def wan_ok(self, packet=b'$\x1a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x06google\x03com\x00\x00\x01\x00\x01'): if not self.isconnected(): # WiFi is down return False length = 32 # DNS query and response packet size @@ -296,7 +310,7 @@ async def wan_ok(self, packet = b'$\x1a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\ s.connect(('8.8.8.8', 53)) await asyncio.sleep(1) try: - await self._as_write(packet, sock = s) + await self._as_write(packet, sock=s) await asyncio.sleep(2) res = await self._as_read(length, s) if len(res) == length: @@ -359,7 +373,7 @@ async def publish(self, topic, msg, retain, qos): if count >= self._max_repubs or not self.isconnected(): raise OSError(-1) # Subclass to re-publish with new PID async with self.lock: - await self._publish(topic, msg, retain, qos, dup = 1) + await self._publish(topic, msg, retain, qos, dup=1) count += 1 self.REPUB_COUNT += 1 @@ -502,7 +516,6 @@ async def wifi_connect(self): self.dprint('Got reliable connection') # Timed out: assumed reliable - async def connect(self): if not self._has_connected: await self.wifi_connect() # On 1st call, caller handles error @@ -581,7 +594,7 @@ def _reconnect(self): # Schedule a reconnection if not underway. loop = asyncio.get_event_loop() loop.create_task(self._wifi_handler(False)) # User handler. - # Await broker connection. + # Await broker connection. async def _connection(self): while not self._isconnected: await asyncio.sleep(1) diff --git a/NO_NET.md b/remote_mqtt/NO_NET.md similarity index 100% rename from NO_NET.md rename to remote_mqtt/NO_NET.md diff --git a/_boot.py b/remote_mqtt/_boot.py similarity index 100% rename from _boot.py rename to remote_mqtt/_boot.py diff --git a/firmware-combined.bin b/remote_mqtt/firmware-combined.bin similarity index 100% rename from firmware-combined.bin rename to remote_mqtt/firmware-combined.bin diff --git a/main.py b/remote_mqtt/main.py similarity index 100% rename from main.py rename to remote_mqtt/main.py diff --git a/mqtt.py b/remote_mqtt/mqtt.py similarity index 100% rename from mqtt.py rename to remote_mqtt/mqtt.py diff --git a/net_local.py b/remote_mqtt/net_local.py similarity index 100% rename from net_local.py rename to remote_mqtt/net_local.py diff --git a/pb_simple.py b/remote_mqtt/pb_simple.py similarity index 100% rename from pb_simple.py rename to remote_mqtt/pb_simple.py diff --git a/pb_status.py b/remote_mqtt/pb_status.py similarity index 100% rename from pb_status.py rename to remote_mqtt/pb_status.py diff --git a/pbmqtt.py b/remote_mqtt/pbmqtt.py similarity index 100% rename from pbmqtt.py rename to remote_mqtt/pbmqtt.py diff --git a/pbmqtt_test.py b/remote_mqtt/pbmqtt_test.py similarity index 100% rename from pbmqtt_test.py rename to remote_mqtt/pbmqtt_test.py diff --git a/pbrange.py b/remote_mqtt/pbrange.py similarity index 100% rename from pbrange.py rename to remote_mqtt/pbrange.py diff --git a/status_values.py b/remote_mqtt/status_values.py similarity index 100% rename from status_values.py rename to remote_mqtt/status_values.py diff --git a/syncom.py b/remote_mqtt/syncom.py similarity index 100% rename from syncom.py rename to remote_mqtt/syncom.py diff --git a/pubtest b/remote_mqtt/tests/pubtest similarity index 100% rename from pubtest rename to remote_mqtt/tests/pubtest diff --git a/pubtest_range b/remote_mqtt/tests/pubtest_range similarity index 100% rename from pubtest_range rename to remote_mqtt/tests/pubtest_range diff --git a/asyn.py b/tests/asyn.py similarity index 100% rename from asyn.py rename to tests/asyn.py diff --git a/mqtt_as/clean.py b/tests/clean.py similarity index 100% rename from mqtt_as/clean.py rename to tests/clean.py diff --git a/mqtt_as/main.py b/tests/main.py similarity index 100% rename from mqtt_as/main.py rename to tests/main.py diff --git a/mqtt_as/pubtest b/tests/pubtest similarity index 100% rename from mqtt_as/pubtest rename to tests/pubtest diff --git a/mqtt_as/range.py b/tests/range.py similarity index 100% rename from mqtt_as/range.py rename to tests/range.py diff --git a/mqtt_as/ssl.py b/tests/ssl.py similarity index 100% rename from mqtt_as/ssl.py rename to tests/ssl.py diff --git a/mqtt_as/unclean.py b/tests/unclean.py similarity index 100% rename from mqtt_as/unclean.py rename to tests/unclean.py From b90de9044174e08ba464405c78ba85e6d7729adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Wed, 14 Mar 2018 16:09:20 +0100 Subject: [PATCH 02/39] removed config dictionary removed config dictionary to save ~100-200B which is especially important on esp8266. Moved config dictionary to config.py to still make it possible to configure the library using that dict. --- README_mqtt_as.md | 2 +- config.py | 23 ++++++++++++- mqtt_as.py | 88 +++++++++++++++++++++++------------------------ 3 files changed, 67 insertions(+), 46 deletions(-) diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 28d2e06..a980366 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -158,7 +158,7 @@ config['connect_coro'] = conn_han config['server'] = SERVER MQTTClient.DEBUG = True # Optional: print diagnostic messages -client = MQTTClient(config) +client = MQTTClient(**config) loop = asyncio.get_event_loop() try: loop.run_until_complete(main(client)) diff --git a/config.py b/config.py index 6d1e267..d4ccddc 100644 --- a/config.py +++ b/config.py @@ -1,8 +1,29 @@ -from mqtt_as import config from sys import platform # Include any cross-project settings. +config = { + 'client_id': hexlify(unique_id()), + 'server': None, + 'port': 0, + 'user': '', + 'password': '', + 'keepalive': 60, + 'ping_interval': 0, + 'ssl': False, + 'ssl_params': {}, + 'response_time': 10, + 'clean_init': True, + 'clean': True, + 'max_repubs': 4, + 'will': None, + 'subs_cb': lambda *_: None, + 'wifi_coro': eliza, + 'connect_coro': eliza, + 'ssid': None, + 'wifi_pw': None, +} + if platform == 'esp32': config['ssid'] = 'my SSID' # EDIT if you're using ESP32 config['wifi_pw'] = 'my WiFi password' diff --git a/mqtt_as.py b/mqtt_as.py index fc85484..5447da6 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -59,28 +59,6 @@ def esp32_pause(): # https://github.com/micropython/micropython-esp32/issues/16 async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program await asyncio.sleep_ms(_DEFAULT_MS) -config = { - 'client_id': hexlify(unique_id()), - 'server': None, - 'port': 0, - 'user': '', - 'password': '', - 'keepalive': 60, - 'ping_interval': 0, - 'ssl': False, - 'ssl_params': {}, - 'response_time': 10, - 'clean_init': True, - 'clean': True, - 'max_repubs': 4, - 'will': None, - 'subs_cb': lambda *_: None, - 'wifi_coro': eliza, - 'connect_coro': eliza, - 'ssid': None, - 'wifi_pw': None, -} - class MQTTException(Exception): pass @@ -118,37 +96,39 @@ class MQTT_base: REPUB_COUNT = 0 # TEST DEBUG = False - def __init__(self, config): + def __init__(self, client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw): # MQTT config - self._client_id = config['client_id'] - self._user = config['user'] - self._pswd = config['password'] - self._keepalive = config['keepalive'] + self.ping_interval = ping_interval + self._client_id = client_id + self._user = user + self._pswd = password + self._keepalive = keepalive if self._keepalive >= 65536: raise ValueError('invalid keepalive time') - self._response_time = config['response_time'] * 1000 # Repub if no PUBACK received (ms). - self._max_repubs = config['max_repubs'] - self._clean_init = config['clean_init'] # clean_session state on first connection - self._clean = config['clean'] # clean_session state on reconnect - will = config['will'] + self._response_time = response_time * 1000 # Repub if no PUBACK received (ms). + self._max_repubs = max_repubs + self._clean_init = clean_init # clean_session state on first connection + self._clean = clean # clean_session state on reconnect if will is None: self._lw_topic = False else: self._set_last_will(*will) # WiFi config - self._ssid = config['ssid'] # For ESP32 - self._wifi_pw = config['wifi_pw'] - self._ssl = config['ssl'] - self._ssl_params = config['ssl_params'] + self._ssid = ssid # For ESP32 + self._wifi_pw = wifi_pw + self._ssl = ssl + self._ssl_params = ssl_params # Callbacks and coros - self._cb = config['subs_cb'] - self._wifi_handler = config['wifi_coro'] - self._connect_handler = config['connect_coro'] + self._cb = subs_cb + self._wifi_handler = wifi_coro + self._connect_handler = connect_coro # Network - self.port = config['port'] + self.port = port if self.port == 0: self.port = 8883 if self._ssl else 1883 - self.server = config['server'] + self.server = server if self.server is None: raise ValueError('no server specified.') self._sock = None @@ -471,12 +451,32 @@ async def wait_msg(self): # MQTTClient class. Handles issues relating to connectivity. class MQTTClient(MQTT_base): - def __init__(self, config): - super().__init__(config) + def __init__(self, client_id=hexlify(unique_id()), + server=None, + port=0, + user='', + password='', + keepalive=60, + ping_interval=0, + ssl=False, + ssl_params={}, + response_time=10, + clean_init=True, + clean=True, + max_repubs=4, + will=None, + subs_cb=lambda *_: None, + wifi_coro=eliza, + connect_coro=eliza, + ssid=None, + wifi_pw=None): + super().__init__(client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) self._isconnected = False # Current connection state keepalive = 1000 * self._keepalive # ms self._ping_interval = keepalive // 4 if keepalive else 20000 - p_i = config['ping_interval'] * 1000 # Can specify shorter e.g. for subscribe-only + p_i = self.ping_interval * 1000 # Can specify shorter e.g. for subscribe-only if p_i and p_i < self._ping_interval: self._ping_interval = p_i self._in_connect = False From 9db5b993621baa7a47c71493ca2dd8ac29784101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 15 Mar 2018 12:40:31 +0100 Subject: [PATCH 03/39] Minimal version for esp8266 Minimal version for esp8266 saving another 150-250B by removing functions for esp32 and sonoff and by removing not needed functions. --- mqtt_as_minimal.py | 513 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 513 insertions(+) create mode 100644 mqtt_as_minimal.py diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py new file mode 100644 index 0000000..32177ba --- /dev/null +++ b/mqtt_as_minimal.py @@ -0,0 +1,513 @@ +# mqtt_as.py Asynchronous version of umqt.robust +# (C) Copyright Peter Hinch 2017. +# Released under the MIT licence. + +import gc +import usocket as socket +import ustruct as struct +gc.collect() +from ubinascii import hexlify +import uasyncio as asyncio +gc.collect() +from utime import ticks_ms, ticks_diff, sleep_ms +from uerrno import EINPROGRESS, ETIMEDOUT +gc.collect() +from micropython import const +from machine import unique_id +import network +gc.collect() + +# Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). +_DEFAULT_MS = const(20) +_SOCKET_POLL_DELAY = const(5) # 100ms added greatly to publish latency + +# Legitimate errors while waiting on a socket. See uasyncio __init__.py open_connection(). +BUSY_ERRORS = (EINPROGRESS, ETIMEDOUT,) + + +# Default "do little" coro for optional user replacement +async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program + await asyncio.sleep_ms(_DEFAULT_MS) + + +class MQTTException(Exception): + pass + + +def newpid(pid): + return pid + 1 if pid < 65535 else 1 + + +def qos_check(qos): + if not (qos == 0 or qos == 1): + raise ValueError('Only qos 0 and 1 are supported.') + + +class Lock(): + def __init__(self): + self._locked = False + + async def __aenter__(self): + while True: + if self._locked: + await asyncio.sleep_ms(_DEFAULT_MS) + else: + self._locked = True + break + + async def __aexit__(self, *args): + self._locked = False + await asyncio.sleep_ms(_DEFAULT_MS) + + +# MQTT_base class. Handles MQTT protocol on the basis of a good connection. +# Exceptions from connectivity failures are handled by MQTTClient subclass. +class MQTT_base: + REPUB_COUNT = 0 # TEST + + def __init__(self, client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw): + # MQTT config + self.ping_interval = ping_interval + self._client_id = client_id + self._user = user + self._pswd = password + self._keepalive = keepalive + if self._keepalive >= 65536: + raise ValueError('invalid keepalive time') + self._response_time = response_time * 1000 # Repub if no PUBACK received (ms). + self._max_repubs = max_repubs + self._clean_init = clean_init # clean_session state on first connection + self._clean = clean # clean_session state on reconnect + if will is None: + self._lw_topic = False + else: + self._set_last_will(*will) + # Callbacks and coros + self._cb = subs_cb + self._wifi_handler = wifi_coro + self._connect_handler = connect_coro + # Network + self.port = port + if self.port == 0: + self.port = 8883 if self._ssl else 1883 + self.server = server + if self.server is None: + raise ValueError('no server specified.') + self._sock = None + self._sta_if = network.WLAN(network.STA_IF) + self._sta_if.active(True) + + self.pid = 0 + self.rcv_pid = 0 + self.suback = False + self.last_rx = ticks_ms() # Time of last communication from broker + self.lock = Lock() + + def _set_last_will(self, topic, msg, retain=False, qos=0): + qos_check(qos) + if not topic: + raise ValueError('Empty topic.') + self._lw_topic = topic + self._lw_msg = msg + self._lw_qos = qos + self._lw_retain = retain + + def _timeout(self, t): + return ticks_diff(ticks_ms(), t) > self._response_time + + async def _as_read(self, n, sock=None): # OSError caught by superclass + if sock is None: + sock = self._sock + data = b'' + t = ticks_ms() + while len(data) < n: + if self._timeout(t) or not self.isconnected(): + raise OSError(-1) + try: + msg = sock.read(n - len(data)) + except OSError as e: # ESP32 issues weird 119 errors here + msg = None + if e.args[0] not in BUSY_ERRORS: + raise + if msg == b'': # Connection closed by host (?) + raise OSError(-1) + if msg is not None: # data received + data = b''.join((data, msg)) + t = ticks_ms() + self.last_rx = ticks_ms() + await asyncio.sleep_ms(_SOCKET_POLL_DELAY) + return data + + async def _as_write(self, bytes_wr, length=0, sock=None): + if sock is None: + sock = self._sock + if length: + bytes_wr = bytes_wr[:length] + t = ticks_ms() + while bytes_wr: + if self._timeout(t) or not self.isconnected(): + raise OSError(-1) + try: + n = sock.write(bytes_wr) + except OSError as e: # ESP32 issues weird 119 errors here + n = 0 + if e.args[0] not in BUSY_ERRORS: + raise + if n: + t = ticks_ms() + bytes_wr = bytes_wr[n:] + await asyncio.sleep_ms(_SOCKET_POLL_DELAY) + + async def _send_str(self, s): + await self._as_write(struct.pack("!H", len(s))) + await self._as_write(s) + + async def _recv_len(self): + n = 0 + sh = 0 + while 1: + res = await self._as_read(1) + b = res[0] + n |= (b & 0x7f) << sh + if not b & 0x80: + return n + sh += 7 + + async def _connect(self, clean): + self._sock = socket.socket() + self._sock.setblocking(False) + try: + self._sock.connect(self._addr) + except OSError as e: + if e.args[0] not in BUSY_ERRORS: + raise + await asyncio.sleep_ms(_DEFAULT_MS) + premsg = bytearray(b"\x10\0\0\0\0\0") + msg = bytearray(b"\x04MQTT\x04\0\0\0") + + sz = 10 + 2 + len(self._client_id) + msg[6] = clean << 1 + if self._user: + sz += 2 + len(self._user) + 2 + len(self._pswd) + msg[6] |= 0xC0 + if self._keepalive: + msg[7] |= self._keepalive >> 8 + msg[8] |= self._keepalive & 0x00FF + if self._lw_topic: + sz += 2 + len(self._lw_topic) + 2 + len(self._lw_msg) + msg[6] |= 0x4 | (self._lw_qos & 0x1) << 3 | (self._lw_qos & 0x2) << 3 + msg[6] |= self._lw_retain << 5 + + i = 1 + while sz > 0x7f: + premsg[i] = (sz & 0x7f) | 0x80 + sz >>= 7 + i += 1 + premsg[i] = sz + await self._as_write(premsg, i + 2) + await self._as_write(msg) + await self._send_str(self._client_id) + if self._lw_topic: + await self._send_str(self._lw_topic) + await self._send_str(self._lw_msg) + if self._user: + await self._send_str(self._user) + await self._send_str(self._pswd) + # Await CONNACK + # read causes ECONNABORTED if broker is out; triggers a reconnect. + resp = await self._as_read(4) + # Got CONNACK + if resp[3] != 0 or resp[0] != 0x20 or resp[1] != 0x02: + raise OSError(-1) # Bad CONNACK e.g. authentication fail. + + async def _ping(self): + async with self.lock: + await self._as_write(b"\xc0\0") + + def close(self): + if self._sock is not None: + self._sock.close() + + # qos == 1: coro blocks until wait_msg gets correct PID. + # If WiFi fails completely subclass re-publishes with new PID. + async def publish(self, topic, msg, retain, qos): + if qos: + self.pid = newpid(self.pid) + self.rcv_pid = 0 + async with self.lock: + await self._publish(topic, msg, retain, qos, 0) + if qos == 0: + return + + count = 0 + while 1: # Await PUBACK, republish on timeout + t = ticks_ms() + while self.pid != self.rcv_pid: + await asyncio.sleep_ms(200) + if self._timeout(t) or not self.isconnected(): + break # Must repub or bail out + else: + return # PID's match. All done. + # No match + if count >= self._max_repubs or not self.isconnected(): + raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock: + await self._publish(topic, msg, retain, qos, dup=1) + count += 1 + self.REPUB_COUNT += 1 + + async def _publish(self, topic, msg, retain, qos, dup): + pkt = bytearray(b"\x30\0\0\0") + pkt[0] |= qos << 1 | retain | dup << 3 + sz = 2 + len(topic) + len(msg) + if qos > 0: + sz += 2 + if sz >= 2097152: + raise MQTTException('Strings too long.') + i = 1 + while sz > 0x7f: + pkt[i] = (sz & 0x7f) | 0x80 + sz >>= 7 + i += 1 + pkt[i] = sz + await self._as_write(pkt, i + 1) + await self._send_str(topic) + if qos > 0: + struct.pack_into("!H", pkt, 0, self.pid) + await self._as_write(pkt, 2) + await self._as_write(msg) + + # Can raise OSError if WiFi fails. Subclass traps + async def subscribe(self, topic, qos): + self.suback = False + pkt = bytearray(b"\x82\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + await self._as_write(qos.to_bytes(1, "little")) + + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + raise OSError(-1) + + # Wait for a single incoming MQTT message and process it. + # Subscribed messages are delivered to a callback previously + # set by .setup() method. Other (internal) MQTT + # messages processed internally. + # Immediate return if no data available. Called from ._handle_msg(). + async def wait_msg(self): + res = self._sock.read(1) # Throws OSError on WiFi fail + if res is None: + return + if res == b'': + raise OSError(-1) + + if res == b"\xd0": # PINGRESP + await self._as_read(1) # Update .last_rx time + return + op = res[0] + + if op == 0x40: # PUBACK: save pid + sz = await self._as_read(1) + if sz != b"\x02": + raise OSError(-1) + rcv_pid = await self._as_read(2) + self.rcv_pid = rcv_pid[0] << 8 | rcv_pid[1] + + if op == 0x90: # SUBACK + resp = await self._as_read(4) + if resp[1] != self.pkt[2] or resp[2] != self.pkt[3] or resp[3] == 0x80: + raise OSError(-1) + self.suback = True + + if op & 0xf0 != 0x30: + return + sz = await self._recv_len() + topic_len = await self._as_read(2) + topic_len = (topic_len[0] << 8) | topic_len[1] + topic = await self._as_read(topic_len) + sz -= topic_len + 2 + if op & 6: + pid = await self._as_read(2) + pid = pid[0] << 8 | pid[1] + sz -= 2 + msg = await self._as_read(sz) + self._cb(topic, msg) + if op & 6 == 2: + pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK + struct.pack_into("!H", pkt, 2, pid) + await self._as_write(pkt) + elif op & 6 == 4: + raise OSError(-1) + + +# MQTTClient class. Handles issues relating to connectivity. + +class MQTTClient(MQTT_base): + def __init__(self, client_id=hexlify(unique_id()), + server=None, + port=0, + user='', + password='', + keepalive=60, + ping_interval=0, + ssl=False, + ssl_params=None, + response_time=10, + clean_init=True, + clean=True, + max_repubs=4, + will=None, + subs_cb=lambda *_: None, + wifi_coro=eliza, + connect_coro=eliza, + ssid=None, + wifi_pw=None): + super().__init__(client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) + self._isconnected = False # Current connection state + keepalive = 1000 * self._keepalive # ms + self._ping_interval = keepalive // 4 if keepalive else 20000 + p_i = self.ping_interval * 1000 # Can specify shorter e.g. for subscribe-only + if p_i and p_i < self._ping_interval: + self._ping_interval = p_i + self._in_connect = False + self._has_connected = False # Define 'Clean Session' value to use. + + async def wifi_connect(self): + s = self._sta_if + # ESP8266 + if s.isconnected(): # 1st attempt, already connected. + return + s.active(True) + s.connect() # ESP8266 remembers connection. + while s.status() == network.STAT_CONNECTING: # Break out on fail or success. Check once per sec. + await asyncio.sleep(1) # Other platforms are OK + + # Ensure connection stays up for a few secs. + t = ticks_ms() + while ticks_diff(ticks_ms(), t) < 5000: + if not s.isconnected(): + raise OSError('WiFi connection fail.') # in 1st 5 secs + await asyncio.sleep(1) + # Timed out: assumed reliable + + async def connect(self): + if not self._has_connected: + await self.wifi_connect() # On 1st call, caller handles error + # Note this blocks if DNS lookup occurs. Do it once to prevent + # blocking during later internet outage: + self._addr = socket.getaddrinfo(self.server, self.port)[0][-1] + self._in_connect = True # Disable low level ._isconnected check + clean = self._clean if self._has_connected else self._clean_init + await self._connect(clean) + # If we get here without error broker/LAN must be up. + self._isconnected = True + self._in_connect = False # Low level code can now check connectivity. + loop = asyncio.get_event_loop() + loop.create_task(self._wifi_handler(True)) # User handler. + if not self._has_connected: + self._has_connected = True # Use normal clean flag on reconnect. + loop.create_task(self._keep_connected()) # Runs forever. + + loop.create_task(self._handle_msg()) # Tasks quit on connection fail. + loop.create_task(self._keep_alive()) + loop.create_task(self._connect_handler(self)) # User handler. + + # Launched by .connect(). Runs until connectivity fails. Checks for and + # handles incoming messages. + async def _handle_msg(self): + try: + while self.isconnected(): + async with self.lock: + await self.wait_msg() # Immediate return if no message + await asyncio.sleep_ms(_DEFAULT_MS) # Let other tasks get lock + + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + + # Keep broker alive MQTT spec 3.1.2.10 Keep Alive. + # Runs until ping failure or no response in keepalive period. + async def _keep_alive(self): + while self.isconnected(): + pings_due = ticks_diff(ticks_ms(), self.last_rx) // self._ping_interval + if pings_due >= 4: + break + elif pings_due >= 1: + try: + await self._ping() + except OSError: + break + await asyncio.sleep(1) + self._reconnect() # Broker or WiFi fail. + + def isconnected(self): + if self._in_connect: # Disable low-level check during .connect() + return True + if self._isconnected and not self._sta_if.isconnected(): # It's going down. + self._reconnect() + return self._isconnected + + def _reconnect(self): # Schedule a reconnection if not underway. + if self._isconnected: + self._isconnected = False + self.close() + loop = asyncio.get_event_loop() + loop.create_task(self._wifi_handler(False)) # User handler. + + # Await broker connection. + async def _connection(self): + while not self._isconnected: + await asyncio.sleep(1) + + # Scheduled on 1st successful connection. Runs forever maintaining wifi and + # broker connection. Must handle conditions at edge of WiFi range. + async def _keep_connected(self): + while True: + if self.isconnected(): # Pause for 1 second + await asyncio.sleep(1) + gc.collect() + else: + self._sta_if.disconnect() + await asyncio.sleep(1) + try: + await self.wifi_connect() + except OSError: + continue + try: + await self.connect() + # Now has set ._isconnected and scheduled _connect_handler(). + except OSError as e: + # Can get ECONNABORTED or -1. The latter signifies no or bad CONNACK received. + self.close() # Disconnect and try again. + self._in_connect = False + self._isconnected = False + + async def subscribe(self, topic, qos=0): + qos_check(qos) + while 1: + await self._connection() + try: + return await super().subscribe(topic, qos) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + + async def publish(self, topic, msg, retain=False, qos=0): + qos_check(qos) + while 1: + await self._connection() + try: + return await super().publish(topic, msg, retain, qos) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. From f45a15d96021b714be1905e56c7c7ee65755da4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 15 Mar 2018 13:10:23 +0100 Subject: [PATCH 04/39] extended class Lock with method locked extended class Lock with method locked() returning self._locked to make it more useful outside of mqtt_as --- mqtt_as.py | 3 +++ mqtt_as_minimal.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/mqtt_as.py b/mqtt_as.py index 5447da6..0d3d545 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -89,6 +89,9 @@ async def __aexit__(self, *args): self._locked = False await asyncio.sleep_ms(_DEFAULT_MS) + def locked(self): + return self._locked + # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 32177ba..c0f3acb 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -59,6 +59,9 @@ async def __aexit__(self, *args): self._locked = False await asyncio.sleep_ms(_DEFAULT_MS) + def locked(self): + return self._locked + # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. From e0ef862cb57f6a66af5e163e15e735a0325b3700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 27 Mar 2018 11:14:33 +0200 Subject: [PATCH 05/39] Added missing documentation, bugfixes Added misssing documentation about changes made for chaning the way the constructor is called, usage in loboris fork and esp8266 minimal version. Also put all remote_mqtt tests in sepearate folder. --- README_mqtt_as.md | 67 ++++++++++++++++---------- remote_mqtt/mqtt.py | 17 +++++-- remote_mqtt/{ => tests}/pb_simple.py | 0 remote_mqtt/{ => tests}/pb_status.py | 0 remote_mqtt/{ => tests}/pbmqtt_test.py | 0 remote_mqtt/{ => tests}/pbrange.py | 0 sonoff/sonoff.py | 31 +++++++----- tests/range.py | 15 ++++-- 8 files changed, 82 insertions(+), 48 deletions(-) rename remote_mqtt/{ => tests}/pb_simple.py (100%) rename remote_mqtt/{ => tests}/pb_status.py (100%) rename remote_mqtt/{ => tests}/pbmqtt_test.py (100%) rename remote_mqtt/{ => tests}/pbrange.py (100%) diff --git a/README_mqtt_as.md b/README_mqtt_as.md index a980366..5cac4a8 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -73,19 +73,26 @@ experience. Feedback on this issue would be very welcome. The module is too large to compile on the ESP8266 and should be precompiled or preferably frozen as bytecode. +To save about 150-250 Bytes on the ESP8266 there is a minimal version `mqtt_as_minimal.py` +that works exactly like the full version but has all code not related to ESP8266 removed +as well as some functions that are not commonly used. ## 1.5 ESP32 issues -This platform has issues. During WiFi and broker connection and reconnection it -has proved necessary to issue `utime.sleep_ms(20)`. After sending data to the -socket a delay of 20ms also proved necessary to prevent a timeout occurring -while waiting for a response (without the delay, the incoming data is never -recognised). These delays block the scheduler. +When using the official port of the ESP32 this platform has issues. +During WiFi and broker connection and reconnection it has proved necessary +to issue `utime.sleep_ms(20)`. After sending data to the socket a delay +of 20ms also proved necessary to prevent a timeout occurring while waiting +for a response (without the delay, the incoming data is never recognised). +These delays block the scheduler. It recovers from outages but this hasn't been tested as thoroughly as on the ESP8266. The board must be power cycled between runs of an application. Issues have been raised. +The [loboris fork](https://github.com/loboris/MicroPython_ESP32_psRAM_LoBo) +does not suffer from these problems. + Currently (5th Sept 2017) DNS lookups don't work (`usocket.getaddrinfo()`). Hence broker addresses must be numeric IP's. @@ -96,16 +103,18 @@ The module works without recourse to cross compilation or frozen bytecode. ## 2.1 Program files 1. `mqtt_as.py` The main module. - 2. `config.py` Stores cross-project settings. - 3. `remote_mqtt` Folder containing all files of mqtt for platforms without WIFI - 4. `sonoff` Folder containing test files for sonoff devices - 5. `tests` Folder containing test files for mqtt_as - 5.1. `clean.py` Test/demo program using MQTT Clean Session. - 5.2. `unclean.py` Test/demo program with MQTT Clean Session `False`. - 5.3. `range.py` For WiFi range testing. - 5.4. `pubtest` Bash script illustrating publication with Mosquitto. - 5.5. `main.py` Example for auto-starting an application. - 5.6. `ssl.py` Failed attempt to run with SSL. See note above (1.3). + 2. `mqtt_as_minimal.py` The main module cut down for the ESP8266 to save RAM + 3. `config.py` Stores cross-project settings. + 4. `tests` Folder containing test files for mqtt_as + 4.1. `clean.py` Test/demo program using MQTT Clean Session. + 4.2. `unclean.py` Test/demo program with MQTT Clean Session `False`. + 4.3. `range.py` For WiFi range testing. + 4.4. `pubtest` Bash script illustrating publication with Mosquitto. + 4.5. `main.py` Example for auto-starting an application. + 4.6. `ssl.py` Failed attempt to run with SSL. See note above (1.3). + 5. `remote_mqtt` Folder containing all files of mqtt for platforms without WIFI + 6. `sonoff` Folder containing test files for sonoff devices + ## 2.2 Installation @@ -113,12 +122,17 @@ The only dependency is uasyncio from the [MicroPython library](https://github.co Ensure this is installed on the device. The module is too large to compile on the ESP8266. It must either be cross -compiled or (preferably) built as frozen bytecode: copy `mqtt_as.py` to -`esp8266/modules` in the source tree, build and deploy. Copy `config.py` to the -filesystem for convenience. +compiled or (preferably) built as frozen bytecode: copy the repo to +`esp8266/modules` in the source tree, build and deploy. If your firmware +gets too big, remove all unnecessary files of just copy the ones you need. +Minimal requirements: +- directory `micropython_mqtt_as` with these files in it: + - `__init__.py` to make it a package + - `mqtt_as.py` or `mqtt_as_minimal.py` + - `config.py` for convenience, optional -On the ESP32 simply copy the Python source to the filesystem (items 1 and 2 -above as a minimum). + +On the ESP32 simply copy the above listed directory structure to the filesystem. ## 2.3 Example Usage @@ -131,8 +145,8 @@ the network if it has to reconnect). The ESP32 behaves differently and requires WiFi config data. Edit `config.py` to suit. ```python -from mqtt_as import MQTTClient -from config import config +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config import uasyncio as asyncio SERVER = '192.168.0.9' # Change to suit e.g. 'iot.eclipse.org' @@ -179,9 +193,12 @@ this. ## 3.1 Constructor -This takes a dictionary as argument. The default is `mqtt_as.config`. Normally -an application imports this and modifies selected entries as required. Entries -are: +This takes all keywords found in the dictionary in `config.py` as argument. +As a convenience you can also use this dictionary by importing it and changing +the values. You then call the constructor by `MQTTClient(**config)`, this +automatically matches the contents of the dict to the keywords of the constructor. + +Entries of config dictionary are: **WiFi Parameters** diff --git a/remote_mqtt/mqtt.py b/remote_mqtt/mqtt.py index 36ddf90..2bf69c7 100644 --- a/remote_mqtt/mqtt.py +++ b/remote_mqtt/mqtt.py @@ -8,7 +8,8 @@ import gc import ubinascii -from mqtt_as import MQTTClient, config +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config from machine import Pin, unique_id, freq import uasyncio as asyncio gc.collect() @@ -21,7 +22,8 @@ from status_values import * # Numeric status values shared with user code. _WIFI_DELAY = 15 # Time (s) to wait for default network -blue = Pin(2, Pin.OUT, value = 1) +blue = Pin(2, Pin.OUT, value=1) + def loads(s): d = {} @@ -29,9 +31,12 @@ def loads(s): return d["v"] # Format an arbitrary list of positional args as a status_values.SEP separated string + + def argformat(*a): return SEP.join(['{}' for x in range(len(a))]).format(*a) + async def heartbeat(): led = Pin(0, Pin.OUT) while True: @@ -50,7 +55,7 @@ def __init__(self, channel, config): config['wifi_coro'] = self.wifi_han config['connect_coro'] = self.conn_han config['client_id'] = ubinascii.hexlify(unique_id()) - super().__init__(config) + super().__init__(**config) # Get NTP time or 0 on any error. async def get_time(self): @@ -101,13 +106,14 @@ async def conn_han(self, _): def subs_cb(self, topic, msg): self.channel.send(argformat(SUBSCRIPTION, topic.decode('UTF8'), msg.decode('UTF8'))) + class Channel(SynCom): def __init__(self): mtx = Pin(14, Pin.OUT) # Define pins mckout = Pin(15, Pin.OUT, value=0) # clocks must be initialised to 0 mrx = Pin(13, Pin.IN) mckin = Pin(12, Pin.IN) - super().__init__(True, mckin, mckout, mrx, mtx, string_mode = True) + super().__init__(True, mckin, mckout, mrx, mtx, string_mode=True) self.cstatus = False # Connection status self.client = None @@ -189,7 +195,7 @@ async def main_task(self, _): self.send(argformat(STATUS, SPECNET)) # Pause for confirmation. User may opt to reboot instead. istr = await self.await_obj(100) - ap = WLAN(AP_IF) # create access-point interface + ap = WLAN(AP_IF) # create access-point interface ap.active(False) # deactivate the interface sta_if.active(True) sta_if.connect(ssid, pw) @@ -219,6 +225,7 @@ async def main_task(self, _): gc.threshold(gc.mem_free() // 4 + gc.mem_alloc()) await asyncio.sleep(1) + loop = asyncio.get_event_loop() loop.create_task(heartbeat()) # Comms channel to Pyboard diff --git a/remote_mqtt/pb_simple.py b/remote_mqtt/tests/pb_simple.py similarity index 100% rename from remote_mqtt/pb_simple.py rename to remote_mqtt/tests/pb_simple.py diff --git a/remote_mqtt/pb_status.py b/remote_mqtt/tests/pb_status.py similarity index 100% rename from remote_mqtt/pb_status.py rename to remote_mqtt/tests/pb_status.py diff --git a/remote_mqtt/pbmqtt_test.py b/remote_mqtt/tests/pbmqtt_test.py similarity index 100% rename from remote_mqtt/pbmqtt_test.py rename to remote_mqtt/tests/pbmqtt_test.py diff --git a/remote_mqtt/pbrange.py b/remote_mqtt/tests/pbrange.py similarity index 100% rename from remote_mqtt/pbrange.py rename to remote_mqtt/tests/pbrange.py diff --git a/sonoff/sonoff.py b/sonoff/sonoff.py index ba28ff0..36ff423 100644 --- a/sonoff/sonoff.py +++ b/sonoff/sonoff.py @@ -34,7 +34,8 @@ # mosquitto_sub -h 192.168.0.9 -t sonoff_result -q 1 import gc -from mqtt_as import MQTTClient, config, sonoff +from micropython_mqtt_as.mqtt_as import MQTTClient, sonoff +from micropython_mqtt_as.config import config sonoff() # Specify special handling gc.collect() import uasyncio as asyncio @@ -56,20 +57,22 @@ # Default topic names. Caller can override. Set name to None if unused. topics = { - 'led' : b'sonoff_led', # Incoming subscriptions - 'relay' : b'sonoff_relay', - 'debug' : b'sonoff_result', # Outgoing publications - 'button' : b'sonoff_result', - 'remote' : b'sonoff_result', # Set to None if no R/C decoder fitted - 'will' : b'sonoff_result', - } + 'led': b'sonoff_led', # Incoming subscriptions + 'relay': b'sonoff_relay', + 'debug': b'sonoff_result', # Outgoing publications + 'button': b'sonoff_result', + 'remote': b'sonoff_result', # Set to None if no R/C decoder fitted + 'will': b'sonoff_result', +} + class Sonoff(MQTTClient): - led = Signal(Pin(13, Pin.OUT, value = 1), invert = True) - relay = Pin(12, Pin.OUT, value = 0) + led = Signal(Pin(13, Pin.OUT, value=1), invert=True) + relay = Pin(12, Pin.OUT, value=0) button = Pushbutton(Pin(0, Pin.IN)) # Pin 5 on serial connector is GPIO14. Pullup in case n/c - pin5 = Pin(14, Pin.IN, pull = Pin.PULL_UP) + pin5 = Pin(14, Pin.IN, pull=Pin.PULL_UP) + def __init__(self, dict_topics): self.topics = dict_topics # OVERRIDE CONFIG DEFAULTS. @@ -90,7 +93,7 @@ def __init__(self, dict_topics): # ping_interval = 5 ensures that LED starts flashing promptly on an outage. # This interval is much too fast for a public broker on the WAN. # config['ping_interval'] = 5 - super().__init__(config) + super().__init__(**config) if self.topics['button'] is not None: # CONFIGURE PUSHBUTTON self.button.press_func(self.btn_action, ('Button press',)) @@ -108,7 +111,7 @@ def __init__(self, dict_topics): def pub_msg(self, topic_name, msg): topic = self.topics[topic_name] if topic is not None and not self.outage: - loop.create_task(self.publish(topic, msg, qos = QOS)) + loop.create_task(self.publish(topic, msg, qos=QOS)) # Callback for message from IR remote. def rc_cb(self, data, addr): @@ -194,6 +197,8 @@ async def main(self): n += 1 # Topic names in dict enables multiple Sonoff units to run this code. Only main.py differs. + + def run(dict_topics=topics): MQTTClient.DEBUG = True client = Sonoff(dict_topics) diff --git a/tests/range.py b/tests/range.py index 7dd94fa..526ef8d 100644 --- a/tests/range.py +++ b/tests/range.py @@ -12,29 +12,32 @@ # blue LED pulse == message received # Publishes connection statistics. -from mqtt_as import MQTTClient, config -from config import config +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config import uasyncio as asyncio from machine import Pin SERVER = '192.168.0.9' # Change to suit # SERVER = 'iot.eclipse.org' -wifi_led = Pin(0, Pin.OUT, value = 0) # Red LED for WiFi fail/not ready yet -blue_led = Pin(2, Pin.OUT, value = 1) # Message received +wifi_led = Pin(0, Pin.OUT, value=0) # Red LED for WiFi fail/not ready yet +blue_led = Pin(2, Pin.OUT, value=1) # Message received loop = asyncio.get_event_loop() outages = 0 + async def pulse(): # This demo pulses blue LED each time a subscribed msg arrives. blue_led(False) await asyncio.sleep(1) blue_led(True) + def sub_cb(topic, msg): print((topic, msg)) loop.create_task(pulse()) + async def wifi_han(state): global outages wifi_led(state) # Off == WiFi down (LED is active low) @@ -45,9 +48,11 @@ async def wifi_han(state): print('WiFi or broker is down.') await asyncio.sleep(1) + async def conn_han(client): await client.subscribe('foo_topic', 1) + async def main(client): try: await client.connect() @@ -59,7 +64,7 @@ async def main(client): await asyncio.sleep(5) print('publish', n) # If WiFi is down the following will pause for the duration. - await client.publish('result', '{} repubs: {} outages: {}'.format(n, client.REPUB_COUNT, outages), qos = 1) + await client.publish('result', '{} repubs: {} outages: {}'.format(n, client.REPUB_COUNT, outages), qos=1) n += 1 # Define configuration From 8526ecde7381616a87aae3d8c8133b4b3bb6fdcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 27 Mar 2018 11:20:48 +0200 Subject: [PATCH 06/39] design fixes in documentation --- README_mqtt_as.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 5cac4a8..6feb8b6 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -106,13 +106,13 @@ The module works without recourse to cross compilation or frozen bytecode. 2. `mqtt_as_minimal.py` The main module cut down for the ESP8266 to save RAM 3. `config.py` Stores cross-project settings. 4. `tests` Folder containing test files for mqtt_as - 4.1. `clean.py` Test/demo program using MQTT Clean Session. - 4.2. `unclean.py` Test/demo program with MQTT Clean Session `False`. - 4.3. `range.py` For WiFi range testing. - 4.4. `pubtest` Bash script illustrating publication with Mosquitto. - 4.5. `main.py` Example for auto-starting an application. - 4.6. `ssl.py` Failed attempt to run with SSL. See note above (1.3). - 5. `remote_mqtt` Folder containing all files of mqtt for platforms without WIFI + 1. `clean.py` Test/demo program using MQTT Clean Session. + 2. `unclean.py` Test/demo program with MQTT Clean Session `False`. + 3. `range.py` For WiFi range testing. + 4. `pubtest` Bash script illustrating publication with Mosquitto. + 5. `main.py` Example for auto-starting an application. + 6. `ssl.py` Failed attempt to run with SSL. See note above (1.3). + 5. `remote_mqtt` Folder containing all files of mqtt for platforms without WIFI and tests 6. `sonoff` Folder containing test files for sonoff devices From be350c5bfda1ba213877c7e39a509864509c1e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 29 Mar 2018 09:58:56 +0200 Subject: [PATCH 07/39] Added information about diff to base repo Added information about all the changes compared to base repo of Peter Hinich --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 2b805bf..f6d6dcd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,29 @@ +# Changes to base repo of Peter Hinich + +1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time +2. Made repo a module to be used like +*from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config* +making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. +3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). +4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase +5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM +6. All other files are updated to the new changes and are usable (e.g. tests). +7. Updated documentation to reflect all changes + +Motivation for the changes: +For my project I had to adapt the library to use it on the ESP32 with loboris fork but also use it on my ESP8266 that is short on RAM all the time. +Therefore I had the following motivation for each of the above mentioned changes: +1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. +2. Like all modules this should be a directory as well, making usage easier. +3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinich made it work with loboris port as well but has the workarounds still in it to be safe) +4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. +5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. +6. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. +7. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D + + + # Introduction MQTT is an easily used networking protocol designed for IOT (internet of From 9f46ea5e16131f536c9c186bee3743bd6dc8ac70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 29 Mar 2018 09:59:56 +0200 Subject: [PATCH 08/39] Added information about diff to base repo Added information about the changes compared to base repo of Peter Hinch (sorry wrote your name wrong) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6d6dcd..3bc0479 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Changes to base repo of Peter Hinich +# Changes to base repo of Peter Hinch 1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time 2. Made repo a module to be used like @@ -16,7 +16,7 @@ For my project I had to adapt the library to use it on the ESP32 with loboris fo Therefore I had the following motivation for each of the above mentioned changes: 1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. 2. Like all modules this should be a directory as well, making usage easier. -3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinich made it work with loboris port as well but has the workarounds still in it to be safe) +3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe) 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. 5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. 6. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. From d9b52f4384d927e2e2abe5417465cda85a59a4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 25 Jun 2018 14:20:00 +0200 Subject: [PATCH 09/39] added Lock.release --- README_mqtt_as.md | 2 +- mqtt_as.py | 5 ++++- mqtt_as_minimal.py | 9 ++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 6feb8b6..59cbb41 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -75,7 +75,7 @@ The module is too large to compile on the ESP8266 and should be precompiled or preferably frozen as bytecode. To save about 150-250 Bytes on the ESP8266 there is a minimal version `mqtt_as_minimal.py` that works exactly like the full version but has all code not related to ESP8266 removed -as well as some functions that are not commonly used. +as well as some functions that are not commonly used and all debug messages. ## 1.5 ESP32 issues diff --git a/mqtt_as.py b/mqtt_as.py index 0d3d545..2a72ccb 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -73,7 +73,7 @@ def qos_check(qos): raise ValueError('Only qos 0 and 1 are supported.') -class Lock(): +class Lock: def __init__(self): self._locked = False @@ -92,6 +92,9 @@ async def __aexit__(self, *args): def locked(self): return self._locked + def release(self): # workaround until fixed https://github.com/micropython/micropython/issues/3153 + self._locked = False + # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index c0f3acb..9ae7933 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -5,16 +5,20 @@ import gc import usocket as socket import ustruct as struct + gc.collect() from ubinascii import hexlify import uasyncio as asyncio + gc.collect() from utime import ticks_ms, ticks_diff, sleep_ms from uerrno import EINPROGRESS, ETIMEDOUT + gc.collect() from micropython import const from machine import unique_id import network + gc.collect() # Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). @@ -43,7 +47,7 @@ def qos_check(qos): raise ValueError('Only qos 0 and 1 are supported.') -class Lock(): +class Lock: def __init__(self): self._locked = False @@ -62,6 +66,9 @@ async def __aexit__(self, *args): def locked(self): return self._locked + def release(self): # workaround until fixed https://github.com/micropython/micropython/issues/3153 + self._locked = False + # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. From ef1eb92e2af07637660ab9bdc1ba8b1a3c2b121a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 29 Mar 2018 09:58:56 +0200 Subject: [PATCH 10/39] Added information about diff to base repo Added information about all the changes compared to base repo of Peter Hinich --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 2b805bf..f6d6dcd 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,29 @@ +# Changes to base repo of Peter Hinich + +1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time +2. Made repo a module to be used like +*from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config* +making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. +3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). +4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase +5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM +6. All other files are updated to the new changes and are usable (e.g. tests). +7. Updated documentation to reflect all changes + +Motivation for the changes: +For my project I had to adapt the library to use it on the ESP32 with loboris fork but also use it on my ESP8266 that is short on RAM all the time. +Therefore I had the following motivation for each of the above mentioned changes: +1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. +2. Like all modules this should be a directory as well, making usage easier. +3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinich made it work with loboris port as well but has the workarounds still in it to be safe) +4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. +5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. +6. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. +7. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D + + + # Introduction MQTT is an easily used networking protocol designed for IOT (internet of From 5bb0091999485d4f93673b1303cce81ae9f4b1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 29 Mar 2018 09:59:56 +0200 Subject: [PATCH 11/39] Added information about diff to base repo Added information about the changes compared to base repo of Peter Hinch (sorry wrote your name wrong) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6d6dcd..3bc0479 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Changes to base repo of Peter Hinich +# Changes to base repo of Peter Hinch 1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time 2. Made repo a module to be used like @@ -16,7 +16,7 @@ For my project I had to adapt the library to use it on the ESP32 with loboris fo Therefore I had the following motivation for each of the above mentioned changes: 1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. 2. Like all modules this should be a directory as well, making usage easier. -3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinich made it work with loboris port as well but has the workarounds still in it to be safe) +3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe) 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. 5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. 6. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. From 7120bd0375cab0b886c43647259e21a86b3f117f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 27 Sep 2018 12:26:16 +0200 Subject: [PATCH 12/39] Add support for unsubscribe, recognize retained messages --- README.md | 12 ++++++++---- README_mqtt_as.md | 21 ++++++++++++++++----- mqtt_as.py | 41 +++++++++++++++++++++++++++++++++++++---- mqtt_as_minimal.py | 39 ++++++++++++++++++++++++++++++++++++--- remote_mqtt/mqtt.py | 16 ++++++++-------- sonoff/sonoff.py | 8 ++++++-- 6 files changed, 111 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3bc0479..0a0b550 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ making it possible to just clone the repo and copy it to `espXXXX/modules` also 3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase 5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM -6. All other files are updated to the new changes and are usable (e.g. tests). -7. Updated documentation to reflect all changes +6. Added support for "unsubscribe" +7. Added support for recognizing retained publications (makes change in "subs_cb" neccessary as it now has to take 3 args) +8. All other files are updated to the new changes and are usable (e.g. tests). +9. Updated documentation to reflect all changes Motivation for the changes: For my project I had to adapt the library to use it on the ESP32 with loboris fork but also use it on my ESP8266 that is short on RAM all the time. @@ -19,8 +21,10 @@ Therefore I had the following motivation for each of the above mentioned changes 3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe) 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. 5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. -6. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. -7. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D +6. At first I did not need that but later it became important to me so I added it +7. I made a huge workaround in a subclass to recognize retained messages instead of just supporting it directly +8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. +9. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 59cbb41..0b097a2 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -91,7 +91,7 @@ ESP8266. The board must be power cycled between runs of an application. Issues have been raised. The [loboris fork](https://github.com/loboris/MicroPython_ESP32_psRAM_LoBo) -does not suffer from these problems. +does not suffer from these problems as far as I can tell by short tests (more needed). Currently (5th Sept 2017) DNS lookups don't work (`usocket.getaddrinfo()`). Hence broker addresses must be numeric IP's. @@ -151,8 +151,8 @@ import uasyncio as asyncio SERVER = '192.168.0.9' # Change to suit e.g. 'iot.eclipse.org' -def callback(topic, msg): - print((topic, msg)) +def callback(topic, msg, retained): + print((topic, msg, retained)) async def conn_han(client): await client.subscribe('foo_topic', 1) @@ -230,8 +230,9 @@ below. **Callbacks and coros** 'subs_cb' [a null lambda function] Subscription callback. Runs when a message -is received whose topic matches a subscription. The callback must take two -args, `topic` and `message`. These will be `bytes` instances. +is received whose topic matches a subscription. The callback must take three +args, `topic`,`message` and `retained`. These will be `bytes` instances, +except `retained` which will be `bool` instance. 'wifi_coro' [a null coro] A coroutine. Defines a task to run when the network state changes. The coro receives a single `bool` arg being the network state. 'connect_coro' [a null coro] A coroutine. Defines a task to run when a @@ -327,6 +328,16 @@ Returns `True` if internet connectivity is available, else `False`. It first checks current WiFi and broker connectivity. If present, it sends a DNS query to '8.8.8.8' and checks for a valid response. +### 3.2.9 unsubscribe (async) + +Unsubscribes a topic, so no messages will be received anymore. + +The coro will pause until a `UNSUBACK` has been received from the broker, if +necessary reconnecting to a failed network. + +Args: + 1. `topic` + ## 3.3 Class Attributes 1. `DEBUG` If `True` causes diagnostic messages to be printed. diff --git a/mqtt_as.py b/mqtt_as.py index 2a72ccb..43abeed 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -402,6 +402,23 @@ async def subscribe(self, topic, qos): if self._timeout(t): raise OSError(-1) + # Can raise OSError if WiFi fails. Subclass traps + async def unsubscribe(self, topic): + self.suback = False + pkt = bytearray(b"\xa2\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + raise OSError(-1) + # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously # set by .setup() method. Other (internal) MQTT @@ -433,6 +450,12 @@ async def wait_msg(self): raise OSError(-1) self.suback = True + if op == 0xB0: # UNSUBACK + resp = await self._as_read(3) + if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: + raise OSError(-1) + self.suback = True + if op & 0xf0 != 0x30: return sz = await self._recv_len() @@ -445,12 +468,13 @@ async def wait_msg(self): pid = pid[0] << 8 | pid[1] sz -= 2 msg = await self._as_read(sz) - self._cb(topic, msg) - if op & 6 == 2: + retained = op & 0x01 + self._cb(topic, msg, bool(retained)) + if op & 6 == 2: # qos 1 pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK struct.pack_into("!H", pkt, 2, pid) await self._as_write(pkt) - elif op & 6 == 4: + elif op & 6 == 4: # qos 2 not supported raise OSError(-1) @@ -496,7 +520,7 @@ async def wifi_connect(self): await asyncio.sleep(1) s.connect(self._ssid, self._wifi_pw) while not s.isconnected(): # ESP32 does not yet support STAT_CONNECTING - esp32_pause() # https://github.com/micropython/micropython-esp32/issues/167 still seems necessary + esp32_pause() # https://github.com/micropython/micropython-esp32/issues/167 still seems necessary await asyncio.sleep(1) else: # ESP8266 @@ -646,6 +670,15 @@ async def subscribe(self, topic, qos=0): pass self._reconnect() # Broker or WiFi fail. + async def unsubscribe(self, topic): + while 1: + await self._connection() + try: + return await super().unsubscribe(topic) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + async def publish(self, topic, msg, retain=False, qos=0): qos_check(qos) while 1: diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 9ae7933..6552cb0 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -307,6 +307,23 @@ async def subscribe(self, topic, qos): if self._timeout(t): raise OSError(-1) + # Can raise OSError if WiFi fails. Subclass traps + async def unsubscribe(self, topic): + self.suback = False + pkt = bytearray(b"\xa2\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + raise OSError(-1) + # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously # set by .setup() method. Other (internal) MQTT @@ -337,6 +354,12 @@ async def wait_msg(self): raise OSError(-1) self.suback = True + if op == 0xB0: # UNSUBACK + resp = await self._as_read(3) + if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: + raise OSError(-1) + self.suback = True + if op & 0xf0 != 0x30: return sz = await self._recv_len() @@ -349,12 +372,13 @@ async def wait_msg(self): pid = pid[0] << 8 | pid[1] sz -= 2 msg = await self._as_read(sz) - self._cb(topic, msg) - if op & 6 == 2: + retained = op & 0x01 + self._cb(topic, msg, bool(retained)) + if op & 6 == 2: # qos 1 pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK struct.pack_into("!H", pkt, 2, pid) await self._as_write(pkt) - elif op & 6 == 4: + elif op & 6 == 4: # qos 2 not supported raise OSError(-1) @@ -512,6 +536,15 @@ async def subscribe(self, topic, qos=0): pass self._reconnect() # Broker or WiFi fail. + async def unsubscribe(self, topic): + while 1: + await self._connection() + try: + return await super().unsubscribe(topic) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + async def publish(self, topic, msg, retain=False, qos=0): qos_check(qos) while 1: diff --git a/remote_mqtt/mqtt.py b/remote_mqtt/mqtt.py index 2bf69c7..c39a6d1 100644 --- a/remote_mqtt/mqtt.py +++ b/remote_mqtt/mqtt.py @@ -103,13 +103,13 @@ async def conn_han(self, _): for topic, qos in self.subscriptions.items(): await self.subscribe(topic, qos) - def subs_cb(self, topic, msg): + def subs_cb(self, topic, msg, retained): self.channel.send(argformat(SUBSCRIPTION, topic.decode('UTF8'), msg.decode('UTF8'))) class Channel(SynCom): def __init__(self): - mtx = Pin(14, Pin.OUT) # Define pins + mtx = Pin(14, Pin.OUT) # Define pins mckout = Pin(15, Pin.OUT, value=0) # clocks must be initialised to 0 mrx = Pin(13, Pin.IN) mckin = Pin(12, Pin.IN) @@ -117,8 +117,8 @@ def __init__(self): self.cstatus = False # Connection status self.client = None -# Task runs continuously. Process incoming Pyboard messages. -# Started by main_task() after client instantiated. + # Task runs continuously. Process incoming Pyboard messages. + # Started by main_task() after client instantiated. async def from_pyboard(self): client = self.client while True: @@ -142,9 +142,9 @@ async def from_pyboard(self): else: self.send(argformat(STATUS, UNKNOWN, 'Unknown command:', istr)) -# Runs when channel has synchronised. No return: Pyboard resets ESP on fail. -# Get parameters from Pyboard. Process them. Connect. Instantiate client. Start -# from_pyboard() task. Wait forever, updating connected status. + # Runs when channel has synchronised. No return: Pyboard resets ESP on fail. + # Get parameters from Pyboard. Process them. Connect. Instantiate client. Start + # from_pyboard() task. Wait forever, updating connected status. async def main_task(self, _): got_params = False # Await connection parameters (init record) @@ -196,7 +196,7 @@ async def main_task(self, _): # Pause for confirmation. User may opt to reboot instead. istr = await self.await_obj(100) ap = WLAN(AP_IF) # create access-point interface - ap.active(False) # deactivate the interface + ap.active(False) # deactivate the interface sta_if.active(True) sta_if.connect(ssid, pw) while not sta_if.isconnected(): diff --git a/sonoff/sonoff.py b/sonoff/sonoff.py index 36ff423..175ba71 100644 --- a/sonoff/sonoff.py +++ b/sonoff/sonoff.py @@ -36,15 +36,18 @@ import gc from micropython_mqtt_as.mqtt_as import MQTTClient, sonoff from micropython_mqtt_as.config import config + sonoff() # Specify special handling gc.collect() import uasyncio as asyncio from machine import Pin, Signal from aswitch import Pushbutton + gc.collect() from aremote import * from network import WLAN, STA_IF from utime import ticks_ms, ticks_diff + gc.collect() SERVER = '192.168.0.9' # Change to suit e.g. 'iot.eclipse.org' @@ -92,7 +95,7 @@ def __init__(self, dict_topics): config['clean_init'] = True # ping_interval = 5 ensures that LED starts flashing promptly on an outage. # This interval is much too fast for a public broker on the WAN. -# config['ping_interval'] = 5 + # config['ping_interval'] = 5 super().__init__(**config) if self.topics['button'] is not None: # CONFIGURE PUSHBUTTON @@ -128,7 +131,7 @@ def btn_action(self, msg): self.pub_msg('button', msg) # Callback for subscribed messages - def sub_cb(self, topic, msg): + def sub_cb(self, topic, msg, retained): if topic == self.topics['relay']: if msg == M_ON or msg == M_OFF: self.relay(int(msg == M_ON)) @@ -196,6 +199,7 @@ async def main(self): self.pub_msg('debug', msg) n += 1 + # Topic names in dict enables multiple Sonoff units to run this code. Only main.py differs. From 490d9e0ca0536d5928e33ceb00f3d7e1e7b36785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 27 Sep 2018 14:01:53 +0200 Subject: [PATCH 13/39] Updated README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a0b550..a7ae7ca 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# WARNING: +The latest commits made this repository NOT USEABLE AS A DROP-IN REPLACEMENT for the original module of Peter Hinch. +This is due to the changes in point 7 explained below. + # Changes to base repo of Peter Hinch 1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time @@ -9,7 +13,7 @@ making it possible to just clone the repo and copy it to `espXXXX/modules` also 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase 5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM 6. Added support for "unsubscribe" -7. Added support for recognizing retained publications (makes change in "subs_cb" neccessary as it now has to take 3 args) +7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained]) 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes From 3bb844db60d3d8fa7e77d3b52b078b0deaa3a785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Fri, 28 Sep 2018 12:04:40 +0200 Subject: [PATCH 14/39] Reliability update When having many subscribe/unsubscribes in a short time, a reliability problem became apparent: The next subscription had been send before the last one received its confirmation from the broker, resulting in an endless reconnect as the received pid did not match the last pid sent to the broker. Solution: Added an additional Lock for subscribe/unsubscribe to wait until the confirmation is received or try again. --- mqtt_as.py | 64 +++++++++++++++++++++++++++------------------- mqtt_as_minimal.py | 59 +++++++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/mqtt_as.py b/mqtt_as.py index 43abeed..a579ffa 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -5,16 +5,20 @@ import gc import usocket as socket import ustruct as struct + gc.collect() from ubinascii import hexlify import uasyncio as asyncio + gc.collect() from utime import ticks_ms, ticks_diff, sleep_ms from uerrno import EINPROGRESS, ETIMEDOUT + gc.collect() from micropython import const from machine import unique_id, idle import network + gc.collect() from sys import platform @@ -53,6 +57,7 @@ def esp32_pause(): # https://github.com/micropython/micropython-esp32/issues/16 else: esp32_pause = lambda *_: None + # Default "do little" coro for optional user replacement @@ -146,6 +151,7 @@ def __init__(self, client_id, server, port, user, password, keepalive, ping_inte self.suback = False self.last_rx = ticks_ms() # Time of last communication from broker self.lock = Lock() + self.lock_operation = Lock() if ESP32 and platform != 'esp32_LoBo': loop = asyncio.get_event_loop() loop.create_task(self._idle_task()) @@ -386,38 +392,42 @@ async def _publish(self, topic, msg, retain, qos, dup): # Can raise OSError if WiFi fails. Subclass traps async def subscribe(self, topic, qos): - self.suback = False - pkt = bytearray(b"\x82\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - await self._as_write(qos.to_bytes(1, "little")) + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\x82\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + await self._as_write(qos.to_bytes(1, "little")) - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Can raise OSError if WiFi fails. Subclass traps async def unsubscribe(self, topic): - self.suback = False - pkt = bytearray(b"\xa2\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\xa2\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 6552cb0..161d1ff 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -114,6 +114,7 @@ def __init__(self, client_id, server, port, user, password, keepalive, ping_inte self.suback = False self.last_rx = ticks_ms() # Time of last communication from broker self.lock = Lock() + self.lock_operation = Lock() def _set_last_will(self, topic, msg, retain=False, qos=0): qos_check(qos) @@ -291,38 +292,42 @@ async def _publish(self, topic, msg, retain, qos, dup): # Can raise OSError if WiFi fails. Subclass traps async def subscribe(self, topic, qos): - self.suback = False - pkt = bytearray(b"\x82\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - await self._as_write(qos.to_bytes(1, "little")) + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\x82\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + await self._as_write(qos.to_bytes(1, "little")) - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Can raise OSError if WiFi fails. Subclass traps async def unsubscribe(self, topic): - self.suback = False - pkt = bytearray(b"\xa2\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\xa2\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously From 7fe9b258f7d35f18ff1866290d378fac5ba3512b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Fri, 28 Sep 2018 12:14:51 +0200 Subject: [PATCH 15/39] Update README to reliability fix --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a7ae7ca..2b9c713 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ making it possible to just clone the repo and copy it to `espXXXX/modules` also 7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained]) 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes +10. Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation) Motivation for the changes: For my project I had to adapt the library to use it on the ESP32 with loboris fork but also use it on my ESP8266 that is short on RAM all the time. @@ -29,7 +30,7 @@ Therefore I had the following motivation for each of the above mentioned changes 7. I made a huge workaround in a subclass to recognize retained messages instead of just supporting it directly 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. 9. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D - +10. was simply needed. Sadly makes the module a little bigger # Introduction From 61158d7b052865ed410daab2c680befe59b54283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Sat, 29 Sep 2018 09:20:40 +0200 Subject: [PATCH 16/39] Even more reliability Increased reliability by putting qos>0 publications under a lock too, so self.pid can't get overwritten by another qos>0 publication. This is probably really improbable but this makes the reliability update complete. --- mqtt_as.py | 45 +++++++++++++++++++++++---------------------- mqtt_as_minimal.py | 45 +++++++++++++++++++++++---------------------- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/mqtt_as.py b/mqtt_as.py index a579ffa..b8a2a6e 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -345,29 +345,30 @@ def close(self): # If WiFi fails completely subclass re-publishes with new PID. async def publish(self, topic, msg, retain, qos): if qos: - self.pid = newpid(self.pid) - self.rcv_pid = 0 - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - if qos == 0: - return - - count = 0 - while 1: # Await PUBACK, republish on timeout - t = ticks_ms() - while self.pid != self.rcv_pid: - await asyncio.sleep_ms(200) - if self._timeout(t) or not self.isconnected(): - break # Must repub or bail out - else: - return # PID's match. All done. - # No match - if count >= self._max_repubs or not self.isconnected(): - raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock_operation: + self.pid = newpid(self.pid) + self.rcv_pid = 0 + count = 0 + while 1: # Await PUBACK, republish on timeout + t = ticks_ms() + while self.pid != self.rcv_pid: + await asyncio.sleep_ms(200) + if self._timeout(t) or not self.isconnected(): + break # Must repub or bail out + else: + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + return # PID's match. All done. + # No match + if count >= self._max_repubs or not self.isconnected(): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock: + await self._publish(topic, msg, retain, qos, dup=1) + count += 1 + self.REPUB_COUNT += 1 + else: async with self.lock: - await self._publish(topic, msg, retain, qos, dup=1) - count += 1 - self.REPUB_COUNT += 1 + await self._publish(topic, msg, retain, qos, 0) async def _publish(self, topic, msg, retain, qos, dup): pkt = bytearray(b"\x30\0\0\0") diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 161d1ff..4af8ec6 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -245,29 +245,30 @@ def close(self): # If WiFi fails completely subclass re-publishes with new PID. async def publish(self, topic, msg, retain, qos): if qos: - self.pid = newpid(self.pid) - self.rcv_pid = 0 - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - if qos == 0: - return - - count = 0 - while 1: # Await PUBACK, republish on timeout - t = ticks_ms() - while self.pid != self.rcv_pid: - await asyncio.sleep_ms(200) - if self._timeout(t) or not self.isconnected(): - break # Must repub or bail out - else: - return # PID's match. All done. - # No match - if count >= self._max_repubs or not self.isconnected(): - raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock_operation: + self.pid = newpid(self.pid) + self.rcv_pid = 0 + count = 0 + while 1: # Await PUBACK, republish on timeout + t = ticks_ms() + while self.pid != self.rcv_pid: + await asyncio.sleep_ms(200) + if self._timeout(t) or not self.isconnected(): + break # Must repub or bail out + else: + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + return # PID's match. All done. + # No match + if count >= self._max_repubs or not self.isconnected(): + self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 + raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock: + await self._publish(topic, msg, retain, qos, dup=1) + count += 1 + self.REPUB_COUNT += 1 + else: async with self.lock: - await self._publish(topic, msg, retain, qos, dup=1) - count += 1 - self.REPUB_COUNT += 1 + await self._publish(topic, msg, retain, qos, 0) async def _publish(self, topic, msg, retain, qos, dup): pkt = bytearray(b"\x30\0\0\0") From dde685f254125dd2451b1ca935789016f5a46e18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Sat, 29 Sep 2018 09:56:19 +0200 Subject: [PATCH 17/39] Bugfix introduced in last commit --- mqtt_as.py | 2 ++ mqtt_as_minimal.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/mqtt_as.py b/mqtt_as.py index b8a2a6e..7db3035 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -349,6 +349,8 @@ async def publish(self, topic, msg, retain, qos): self.pid = newpid(self.pid) self.rcv_pid = 0 count = 0 + async with self.lock: + await self._publish(topic, msg, retain, qos, 0) while 1: # Await PUBACK, republish on timeout t = ticks_ms() while self.pid != self.rcv_pid: diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 4af8ec6..972458e 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -249,6 +249,8 @@ async def publish(self, topic, msg, retain, qos): self.pid = newpid(self.pid) self.rcv_pid = 0 count = 0 + async with self.lock: + await self._publish(topic, msg, retain, qos, 0) while 1: # Await PUBACK, republish on timeout t = ticks_ms() while self.pid != self.rcv_pid: From 7b5197cc99be0f72d6e4dbaf82d9de37d24d78d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 29 Jan 2019 18:28:59 +0100 Subject: [PATCH 18/39] Remove workarounds needed before micropython 1.10. https://github.com/micropython/micropython/issues/3153 --- mqtt_as.py | 9 +- mqtt_as_minimal.py | 1124 ++++++++++++++++++++++---------------------- 2 files changed, 563 insertions(+), 570 deletions(-) diff --git a/mqtt_as.py b/mqtt_as.py index 7db3035..4dc33f5 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -97,7 +97,7 @@ async def __aexit__(self, *args): def locked(self): return self._locked - def release(self): # workaround until fixed https://github.com/micropython/micropython/issues/3153 + def release(self): self._locked = False @@ -293,7 +293,8 @@ async def _ping(self): await self._as_write(b"\xc0\0") # Check internet connectivity by sending DNS lookup to Google's 8.8.8.8 - async def wan_ok(self, packet=b'$\x1a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x06google\x03com\x00\x00\x01\x00\x01'): + async def wan_ok(self, + packet=b'$\x1a\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x06google\x03com\x00\x00\x01\x00\x01'): if not self.isconnected(): # WiFi is down return False length = 32 # DNS query and response packet size @@ -358,11 +359,9 @@ async def publish(self, topic, msg, retain, qos): if self._timeout(t) or not self.isconnected(): break # Must repub or bail out else: - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 return # PID's match. All done. # No match if count >= self._max_repubs or not self.isconnected(): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 raise OSError(-1) # Subclass to re-publish with new PID async with self.lock: await self._publish(topic, msg, retain, qos, dup=1) @@ -410,7 +409,6 @@ async def subscribe(self, topic, qos): while not self.suback: await asyncio.sleep_ms(200) if self._timeout(t): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 raise OSError(-1) # Can raise OSError if WiFi fails. Subclass traps @@ -429,7 +427,6 @@ async def unsubscribe(self, topic): while not self.suback: await asyncio.sleep_ms(200) if self._timeout(t): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 raise OSError(-1) # Wait for a single incoming MQTT message and process it. diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py index 972458e..c076909 100644 --- a/mqtt_as_minimal.py +++ b/mqtt_as_minimal.py @@ -1,564 +1,560 @@ -# mqtt_as.py Asynchronous version of umqt.robust -# (C) Copyright Peter Hinch 2017. -# Released under the MIT licence. - -import gc -import usocket as socket -import ustruct as struct - -gc.collect() -from ubinascii import hexlify -import uasyncio as asyncio - -gc.collect() -from utime import ticks_ms, ticks_diff, sleep_ms -from uerrno import EINPROGRESS, ETIMEDOUT - -gc.collect() -from micropython import const -from machine import unique_id -import network - -gc.collect() - -# Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). -_DEFAULT_MS = const(20) -_SOCKET_POLL_DELAY = const(5) # 100ms added greatly to publish latency - -# Legitimate errors while waiting on a socket. See uasyncio __init__.py open_connection(). -BUSY_ERRORS = (EINPROGRESS, ETIMEDOUT,) - - -# Default "do little" coro for optional user replacement -async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program - await asyncio.sleep_ms(_DEFAULT_MS) - - -class MQTTException(Exception): - pass - - -def newpid(pid): - return pid + 1 if pid < 65535 else 1 - - -def qos_check(qos): - if not (qos == 0 or qos == 1): - raise ValueError('Only qos 0 and 1 are supported.') - - -class Lock: - def __init__(self): - self._locked = False - - async def __aenter__(self): - while True: - if self._locked: - await asyncio.sleep_ms(_DEFAULT_MS) - else: - self._locked = True - break - - async def __aexit__(self, *args): - self._locked = False - await asyncio.sleep_ms(_DEFAULT_MS) - - def locked(self): - return self._locked - - def release(self): # workaround until fixed https://github.com/micropython/micropython/issues/3153 - self._locked = False - - -# MQTT_base class. Handles MQTT protocol on the basis of a good connection. -# Exceptions from connectivity failures are handled by MQTTClient subclass. -class MQTT_base: - REPUB_COUNT = 0 # TEST - - def __init__(self, client_id, server, port, user, password, keepalive, ping_interval, - ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, - subs_cb, wifi_coro, connect_coro, ssid, wifi_pw): - # MQTT config - self.ping_interval = ping_interval - self._client_id = client_id - self._user = user - self._pswd = password - self._keepalive = keepalive - if self._keepalive >= 65536: - raise ValueError('invalid keepalive time') - self._response_time = response_time * 1000 # Repub if no PUBACK received (ms). - self._max_repubs = max_repubs - self._clean_init = clean_init # clean_session state on first connection - self._clean = clean # clean_session state on reconnect - if will is None: - self._lw_topic = False - else: - self._set_last_will(*will) - # Callbacks and coros - self._cb = subs_cb - self._wifi_handler = wifi_coro - self._connect_handler = connect_coro - # Network - self.port = port - if self.port == 0: - self.port = 8883 if self._ssl else 1883 - self.server = server - if self.server is None: - raise ValueError('no server specified.') - self._sock = None - self._sta_if = network.WLAN(network.STA_IF) - self._sta_if.active(True) - - self.pid = 0 - self.rcv_pid = 0 - self.suback = False - self.last_rx = ticks_ms() # Time of last communication from broker - self.lock = Lock() - self.lock_operation = Lock() - - def _set_last_will(self, topic, msg, retain=False, qos=0): - qos_check(qos) - if not topic: - raise ValueError('Empty topic.') - self._lw_topic = topic - self._lw_msg = msg - self._lw_qos = qos - self._lw_retain = retain - - def _timeout(self, t): - return ticks_diff(ticks_ms(), t) > self._response_time - - async def _as_read(self, n, sock=None): # OSError caught by superclass - if sock is None: - sock = self._sock - data = b'' - t = ticks_ms() - while len(data) < n: - if self._timeout(t) or not self.isconnected(): - raise OSError(-1) - try: - msg = sock.read(n - len(data)) - except OSError as e: # ESP32 issues weird 119 errors here - msg = None - if e.args[0] not in BUSY_ERRORS: - raise - if msg == b'': # Connection closed by host (?) - raise OSError(-1) - if msg is not None: # data received - data = b''.join((data, msg)) - t = ticks_ms() - self.last_rx = ticks_ms() - await asyncio.sleep_ms(_SOCKET_POLL_DELAY) - return data - - async def _as_write(self, bytes_wr, length=0, sock=None): - if sock is None: - sock = self._sock - if length: - bytes_wr = bytes_wr[:length] - t = ticks_ms() - while bytes_wr: - if self._timeout(t) or not self.isconnected(): - raise OSError(-1) - try: - n = sock.write(bytes_wr) - except OSError as e: # ESP32 issues weird 119 errors here - n = 0 - if e.args[0] not in BUSY_ERRORS: - raise - if n: - t = ticks_ms() - bytes_wr = bytes_wr[n:] - await asyncio.sleep_ms(_SOCKET_POLL_DELAY) - - async def _send_str(self, s): - await self._as_write(struct.pack("!H", len(s))) - await self._as_write(s) - - async def _recv_len(self): - n = 0 - sh = 0 - while 1: - res = await self._as_read(1) - b = res[0] - n |= (b & 0x7f) << sh - if not b & 0x80: - return n - sh += 7 - - async def _connect(self, clean): - self._sock = socket.socket() - self._sock.setblocking(False) - try: - self._sock.connect(self._addr) - except OSError as e: - if e.args[0] not in BUSY_ERRORS: - raise - await asyncio.sleep_ms(_DEFAULT_MS) - premsg = bytearray(b"\x10\0\0\0\0\0") - msg = bytearray(b"\x04MQTT\x04\0\0\0") - - sz = 10 + 2 + len(self._client_id) - msg[6] = clean << 1 - if self._user: - sz += 2 + len(self._user) + 2 + len(self._pswd) - msg[6] |= 0xC0 - if self._keepalive: - msg[7] |= self._keepalive >> 8 - msg[8] |= self._keepalive & 0x00FF - if self._lw_topic: - sz += 2 + len(self._lw_topic) + 2 + len(self._lw_msg) - msg[6] |= 0x4 | (self._lw_qos & 0x1) << 3 | (self._lw_qos & 0x2) << 3 - msg[6] |= self._lw_retain << 5 - - i = 1 - while sz > 0x7f: - premsg[i] = (sz & 0x7f) | 0x80 - sz >>= 7 - i += 1 - premsg[i] = sz - await self._as_write(premsg, i + 2) - await self._as_write(msg) - await self._send_str(self._client_id) - if self._lw_topic: - await self._send_str(self._lw_topic) - await self._send_str(self._lw_msg) - if self._user: - await self._send_str(self._user) - await self._send_str(self._pswd) - # Await CONNACK - # read causes ECONNABORTED if broker is out; triggers a reconnect. - resp = await self._as_read(4) - # Got CONNACK - if resp[3] != 0 or resp[0] != 0x20 or resp[1] != 0x02: - raise OSError(-1) # Bad CONNACK e.g. authentication fail. - - async def _ping(self): - async with self.lock: - await self._as_write(b"\xc0\0") - - def close(self): - if self._sock is not None: - self._sock.close() - - # qos == 1: coro blocks until wait_msg gets correct PID. - # If WiFi fails completely subclass re-publishes with new PID. - async def publish(self, topic, msg, retain, qos): - if qos: - async with self.lock_operation: - self.pid = newpid(self.pid) - self.rcv_pid = 0 - count = 0 - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - while 1: # Await PUBACK, republish on timeout - t = ticks_ms() - while self.pid != self.rcv_pid: - await asyncio.sleep_ms(200) - if self._timeout(t) or not self.isconnected(): - break # Must repub or bail out - else: - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 - return # PID's match. All done. - # No match - if count >= self._max_repubs or not self.isconnected(): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 - raise OSError(-1) # Subclass to re-publish with new PID - async with self.lock: - await self._publish(topic, msg, retain, qos, dup=1) - count += 1 - self.REPUB_COUNT += 1 - else: - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - - async def _publish(self, topic, msg, retain, qos, dup): - pkt = bytearray(b"\x30\0\0\0") - pkt[0] |= qos << 1 | retain | dup << 3 - sz = 2 + len(topic) + len(msg) - if qos > 0: - sz += 2 - if sz >= 2097152: - raise MQTTException('Strings too long.') - i = 1 - while sz > 0x7f: - pkt[i] = (sz & 0x7f) | 0x80 - sz >>= 7 - i += 1 - pkt[i] = sz - await self._as_write(pkt, i + 1) - await self._send_str(topic) - if qos > 0: - struct.pack_into("!H", pkt, 0, self.pid) - await self._as_write(pkt, 2) - await self._as_write(msg) - - # Can raise OSError if WiFi fails. Subclass traps - async def subscribe(self, topic, qos): - async with self.lock_operation: - self.suback = False - pkt = bytearray(b"\x82\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - await self._as_write(qos.to_bytes(1, "little")) - - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 - raise OSError(-1) - - # Can raise OSError if WiFi fails. Subclass traps - async def unsubscribe(self, topic): - async with self.lock_operation: - self.suback = False - pkt = bytearray(b"\xa2\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - self.lock_operation.release() # needed until bug fixed and released: https://github.com/micropython/micropython/issues/3153 - raise OSError(-1) - - # Wait for a single incoming MQTT message and process it. - # Subscribed messages are delivered to a callback previously - # set by .setup() method. Other (internal) MQTT - # messages processed internally. - # Immediate return if no data available. Called from ._handle_msg(). - async def wait_msg(self): - res = self._sock.read(1) # Throws OSError on WiFi fail - if res is None: - return - if res == b'': - raise OSError(-1) - - if res == b"\xd0": # PINGRESP - await self._as_read(1) # Update .last_rx time - return - op = res[0] - - if op == 0x40: # PUBACK: save pid - sz = await self._as_read(1) - if sz != b"\x02": - raise OSError(-1) - rcv_pid = await self._as_read(2) - self.rcv_pid = rcv_pid[0] << 8 | rcv_pid[1] - - if op == 0x90: # SUBACK - resp = await self._as_read(4) - if resp[1] != self.pkt[2] or resp[2] != self.pkt[3] or resp[3] == 0x80: - raise OSError(-1) - self.suback = True - - if op == 0xB0: # UNSUBACK - resp = await self._as_read(3) - if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: - raise OSError(-1) - self.suback = True - - if op & 0xf0 != 0x30: - return - sz = await self._recv_len() - topic_len = await self._as_read(2) - topic_len = (topic_len[0] << 8) | topic_len[1] - topic = await self._as_read(topic_len) - sz -= topic_len + 2 - if op & 6: - pid = await self._as_read(2) - pid = pid[0] << 8 | pid[1] - sz -= 2 - msg = await self._as_read(sz) - retained = op & 0x01 - self._cb(topic, msg, bool(retained)) - if op & 6 == 2: # qos 1 - pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK - struct.pack_into("!H", pkt, 2, pid) - await self._as_write(pkt) - elif op & 6 == 4: # qos 2 not supported - raise OSError(-1) - - -# MQTTClient class. Handles issues relating to connectivity. - -class MQTTClient(MQTT_base): - def __init__(self, client_id=hexlify(unique_id()), - server=None, - port=0, - user='', - password='', - keepalive=60, - ping_interval=0, - ssl=False, - ssl_params=None, - response_time=10, - clean_init=True, - clean=True, - max_repubs=4, - will=None, - subs_cb=lambda *_: None, - wifi_coro=eliza, - connect_coro=eliza, - ssid=None, - wifi_pw=None): - super().__init__(client_id, server, port, user, password, keepalive, ping_interval, - ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, - subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) - self._isconnected = False # Current connection state - keepalive = 1000 * self._keepalive # ms - self._ping_interval = keepalive // 4 if keepalive else 20000 - p_i = self.ping_interval * 1000 # Can specify shorter e.g. for subscribe-only - if p_i and p_i < self._ping_interval: - self._ping_interval = p_i - self._in_connect = False - self._has_connected = False # Define 'Clean Session' value to use. - - async def wifi_connect(self): - s = self._sta_if - # ESP8266 - if s.isconnected(): # 1st attempt, already connected. - return - s.active(True) - s.connect() # ESP8266 remembers connection. - while s.status() == network.STAT_CONNECTING: # Break out on fail or success. Check once per sec. - await asyncio.sleep(1) # Other platforms are OK - - # Ensure connection stays up for a few secs. - t = ticks_ms() - while ticks_diff(ticks_ms(), t) < 5000: - if not s.isconnected(): - raise OSError('WiFi connection fail.') # in 1st 5 secs - await asyncio.sleep(1) - # Timed out: assumed reliable - - async def connect(self): - if not self._has_connected: - await self.wifi_connect() # On 1st call, caller handles error - # Note this blocks if DNS lookup occurs. Do it once to prevent - # blocking during later internet outage: - self._addr = socket.getaddrinfo(self.server, self.port)[0][-1] - self._in_connect = True # Disable low level ._isconnected check - clean = self._clean if self._has_connected else self._clean_init - await self._connect(clean) - # If we get here without error broker/LAN must be up. - self._isconnected = True - self._in_connect = False # Low level code can now check connectivity. - loop = asyncio.get_event_loop() - loop.create_task(self._wifi_handler(True)) # User handler. - if not self._has_connected: - self._has_connected = True # Use normal clean flag on reconnect. - loop.create_task(self._keep_connected()) # Runs forever. - - loop.create_task(self._handle_msg()) # Tasks quit on connection fail. - loop.create_task(self._keep_alive()) - loop.create_task(self._connect_handler(self)) # User handler. - - # Launched by .connect(). Runs until connectivity fails. Checks for and - # handles incoming messages. - async def _handle_msg(self): - try: - while self.isconnected(): - async with self.lock: - await self.wait_msg() # Immediate return if no message - await asyncio.sleep_ms(_DEFAULT_MS) # Let other tasks get lock - - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - # Keep broker alive MQTT spec 3.1.2.10 Keep Alive. - # Runs until ping failure or no response in keepalive period. - async def _keep_alive(self): - while self.isconnected(): - pings_due = ticks_diff(ticks_ms(), self.last_rx) // self._ping_interval - if pings_due >= 4: - break - elif pings_due >= 1: - try: - await self._ping() - except OSError: - break - await asyncio.sleep(1) - self._reconnect() # Broker or WiFi fail. - - def isconnected(self): - if self._in_connect: # Disable low-level check during .connect() - return True - if self._isconnected and not self._sta_if.isconnected(): # It's going down. - self._reconnect() - return self._isconnected - - def _reconnect(self): # Schedule a reconnection if not underway. - if self._isconnected: - self._isconnected = False - self.close() - loop = asyncio.get_event_loop() - loop.create_task(self._wifi_handler(False)) # User handler. - - # Await broker connection. - async def _connection(self): - while not self._isconnected: - await asyncio.sleep(1) - - # Scheduled on 1st successful connection. Runs forever maintaining wifi and - # broker connection. Must handle conditions at edge of WiFi range. - async def _keep_connected(self): - while True: - if self.isconnected(): # Pause for 1 second - await asyncio.sleep(1) - gc.collect() - else: - self._sta_if.disconnect() - await asyncio.sleep(1) - try: - await self.wifi_connect() - except OSError: - continue - try: - await self.connect() - # Now has set ._isconnected and scheduled _connect_handler(). - except OSError as e: - # Can get ECONNABORTED or -1. The latter signifies no or bad CONNACK received. - self.close() # Disconnect and try again. - self._in_connect = False - self._isconnected = False - - async def subscribe(self, topic, qos=0): - qos_check(qos) - while 1: - await self._connection() - try: - return await super().subscribe(topic, qos) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - async def unsubscribe(self, topic): - while 1: - await self._connection() - try: - return await super().unsubscribe(topic) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - async def publish(self, topic, msg, retain=False, qos=0): - qos_check(qos) - while 1: - await self._connection() - try: - return await super().publish(topic, msg, retain, qos) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. +# mqtt_as.py Asynchronous version of umqt.robust +# (C) Copyright Peter Hinch 2017. +# Released under the MIT licence. + +import gc +import usocket as socket +import ustruct as struct + +gc.collect() +from ubinascii import hexlify +import uasyncio as asyncio + +gc.collect() +from utime import ticks_ms, ticks_diff, sleep_ms +from uerrno import EINPROGRESS, ETIMEDOUT + +gc.collect() +from micropython import const +from machine import unique_id +import network + +gc.collect() + +# Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). +_DEFAULT_MS = const(20) +_SOCKET_POLL_DELAY = const(5) # 100ms added greatly to publish latency + +# Legitimate errors while waiting on a socket. See uasyncio __init__.py open_connection(). +BUSY_ERRORS = (EINPROGRESS, ETIMEDOUT,) + + +# Default "do little" coro for optional user replacement +async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program + await asyncio.sleep_ms(_DEFAULT_MS) + + +class MQTTException(Exception): + pass + + +def newpid(pid): + return pid + 1 if pid < 65535 else 1 + + +def qos_check(qos): + if not (qos == 0 or qos == 1): + raise ValueError('Only qos 0 and 1 are supported.') + + +class Lock: + def __init__(self): + self._locked = False + + async def __aenter__(self): + while True: + if self._locked: + await asyncio.sleep_ms(_DEFAULT_MS) + else: + self._locked = True + break + + async def __aexit__(self, *args): + self._locked = False + await asyncio.sleep_ms(_DEFAULT_MS) + + def locked(self): + return self._locked + + def release(self): + self._locked = False + + +# MQTT_base class. Handles MQTT protocol on the basis of a good connection. +# Exceptions from connectivity failures are handled by MQTTClient subclass. +class MQTT_base: + REPUB_COUNT = 0 # TEST + + def __init__(self, client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw): + # MQTT config + self.ping_interval = ping_interval + self._client_id = client_id + self._user = user + self._pswd = password + self._keepalive = keepalive + if self._keepalive >= 65536: + raise ValueError('invalid keepalive time') + self._response_time = response_time * 1000 # Repub if no PUBACK received (ms). + self._max_repubs = max_repubs + self._clean_init = clean_init # clean_session state on first connection + self._clean = clean # clean_session state on reconnect + if will is None: + self._lw_topic = False + else: + self._set_last_will(*will) + # Callbacks and coros + self._cb = subs_cb + self._wifi_handler = wifi_coro + self._connect_handler = connect_coro + # Network + self.port = port + if self.port == 0: + self.port = 8883 if self._ssl else 1883 + self.server = server + if self.server is None: + raise ValueError('no server specified.') + self._sock = None + self._sta_if = network.WLAN(network.STA_IF) + self._sta_if.active(True) + + self.pid = 0 + self.rcv_pid = 0 + self.suback = False + self.last_rx = ticks_ms() # Time of last communication from broker + self.lock = Lock() + self.lock_operation = Lock() + + def _set_last_will(self, topic, msg, retain=False, qos=0): + qos_check(qos) + if not topic: + raise ValueError('Empty topic.') + self._lw_topic = topic + self._lw_msg = msg + self._lw_qos = qos + self._lw_retain = retain + + def _timeout(self, t): + return ticks_diff(ticks_ms(), t) > self._response_time + + async def _as_read(self, n, sock=None): # OSError caught by superclass + if sock is None: + sock = self._sock + data = b'' + t = ticks_ms() + while len(data) < n: + if self._timeout(t) or not self.isconnected(): + raise OSError(-1) + try: + msg = sock.read(n - len(data)) + except OSError as e: # ESP32 issues weird 119 errors here + msg = None + if e.args[0] not in BUSY_ERRORS: + raise + if msg == b'': # Connection closed by host (?) + raise OSError(-1) + if msg is not None: # data received + data = b''.join((data, msg)) + t = ticks_ms() + self.last_rx = ticks_ms() + await asyncio.sleep_ms(_SOCKET_POLL_DELAY) + return data + + async def _as_write(self, bytes_wr, length=0, sock=None): + if sock is None: + sock = self._sock + if length: + bytes_wr = bytes_wr[:length] + t = ticks_ms() + while bytes_wr: + if self._timeout(t) or not self.isconnected(): + raise OSError(-1) + try: + n = sock.write(bytes_wr) + except OSError as e: # ESP32 issues weird 119 errors here + n = 0 + if e.args[0] not in BUSY_ERRORS: + raise + if n: + t = ticks_ms() + bytes_wr = bytes_wr[n:] + await asyncio.sleep_ms(_SOCKET_POLL_DELAY) + + async def _send_str(self, s): + await self._as_write(struct.pack("!H", len(s))) + await self._as_write(s) + + async def _recv_len(self): + n = 0 + sh = 0 + while 1: + res = await self._as_read(1) + b = res[0] + n |= (b & 0x7f) << sh + if not b & 0x80: + return n + sh += 7 + + async def _connect(self, clean): + self._sock = socket.socket() + self._sock.setblocking(False) + try: + self._sock.connect(self._addr) + except OSError as e: + if e.args[0] not in BUSY_ERRORS: + raise + await asyncio.sleep_ms(_DEFAULT_MS) + premsg = bytearray(b"\x10\0\0\0\0\0") + msg = bytearray(b"\x04MQTT\x04\0\0\0") + + sz = 10 + 2 + len(self._client_id) + msg[6] = clean << 1 + if self._user: + sz += 2 + len(self._user) + 2 + len(self._pswd) + msg[6] |= 0xC0 + if self._keepalive: + msg[7] |= self._keepalive >> 8 + msg[8] |= self._keepalive & 0x00FF + if self._lw_topic: + sz += 2 + len(self._lw_topic) + 2 + len(self._lw_msg) + msg[6] |= 0x4 | (self._lw_qos & 0x1) << 3 | (self._lw_qos & 0x2) << 3 + msg[6] |= self._lw_retain << 5 + + i = 1 + while sz > 0x7f: + premsg[i] = (sz & 0x7f) | 0x80 + sz >>= 7 + i += 1 + premsg[i] = sz + await self._as_write(premsg, i + 2) + await self._as_write(msg) + await self._send_str(self._client_id) + if self._lw_topic: + await self._send_str(self._lw_topic) + await self._send_str(self._lw_msg) + if self._user: + await self._send_str(self._user) + await self._send_str(self._pswd) + # Await CONNACK + # read causes ECONNABORTED if broker is out; triggers a reconnect. + resp = await self._as_read(4) + # Got CONNACK + if resp[3] != 0 or resp[0] != 0x20 or resp[1] != 0x02: + raise OSError(-1) # Bad CONNACK e.g. authentication fail. + + async def _ping(self): + async with self.lock: + await self._as_write(b"\xc0\0") + + def close(self): + if self._sock is not None: + self._sock.close() + + # qos == 1: coro blocks until wait_msg gets correct PID. + # If WiFi fails completely subclass re-publishes with new PID. + async def publish(self, topic, msg, retain, qos): + if qos: + async with self.lock_operation: + self.pid = newpid(self.pid) + self.rcv_pid = 0 + count = 0 + async with self.lock: + await self._publish(topic, msg, retain, qos, 0) + while 1: # Await PUBACK, republish on timeout + t = ticks_ms() + while self.pid != self.rcv_pid: + await asyncio.sleep_ms(200) + if self._timeout(t) or not self.isconnected(): + break # Must repub or bail out + else: + return # PID's match. All done. + # No match + if count >= self._max_repubs or not self.isconnected(): + raise OSError(-1) # Subclass to re-publish with new PID + async with self.lock: + await self._publish(topic, msg, retain, qos, dup=1) + count += 1 + self.REPUB_COUNT += 1 + else: + async with self.lock: + await self._publish(topic, msg, retain, qos, 0) + + async def _publish(self, topic, msg, retain, qos, dup): + pkt = bytearray(b"\x30\0\0\0") + pkt[0] |= qos << 1 | retain | dup << 3 + sz = 2 + len(topic) + len(msg) + if qos > 0: + sz += 2 + if sz >= 2097152: + raise MQTTException('Strings too long.') + i = 1 + while sz > 0x7f: + pkt[i] = (sz & 0x7f) | 0x80 + sz >>= 7 + i += 1 + pkt[i] = sz + await self._as_write(pkt, i + 1) + await self._send_str(topic) + if qos > 0: + struct.pack_into("!H", pkt, 0, self.pid) + await self._as_write(pkt, 2) + await self._as_write(msg) + + # Can raise OSError if WiFi fails. Subclass traps + async def subscribe(self, topic, qos): + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\x82\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + await self._as_write(qos.to_bytes(1, "little")) + + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + raise OSError(-1) + + # Can raise OSError if WiFi fails. Subclass traps + async def unsubscribe(self, topic): + async with self.lock_operation: + self.suback = False + pkt = bytearray(b"\xa2\0\0\0") + self.pid = newpid(self.pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) + self.pkt = pkt + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) + + t = ticks_ms() + while not self.suback: + await asyncio.sleep_ms(200) + if self._timeout(t): + raise OSError(-1) + + # Wait for a single incoming MQTT message and process it. + # Subscribed messages are delivered to a callback previously + # set by .setup() method. Other (internal) MQTT + # messages processed internally. + # Immediate return if no data available. Called from ._handle_msg(). + async def wait_msg(self): + res = self._sock.read(1) # Throws OSError on WiFi fail + if res is None: + return + if res == b'': + raise OSError(-1) + + if res == b"\xd0": # PINGRESP + await self._as_read(1) # Update .last_rx time + return + op = res[0] + + if op == 0x40: # PUBACK: save pid + sz = await self._as_read(1) + if sz != b"\x02": + raise OSError(-1) + rcv_pid = await self._as_read(2) + self.rcv_pid = rcv_pid[0] << 8 | rcv_pid[1] + + if op == 0x90: # SUBACK + resp = await self._as_read(4) + if resp[1] != self.pkt[2] or resp[2] != self.pkt[3] or resp[3] == 0x80: + raise OSError(-1) + self.suback = True + + if op == 0xB0: # UNSUBACK + resp = await self._as_read(3) + if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: + raise OSError(-1) + self.suback = True + + if op & 0xf0 != 0x30: + return + sz = await self._recv_len() + topic_len = await self._as_read(2) + topic_len = (topic_len[0] << 8) | topic_len[1] + topic = await self._as_read(topic_len) + sz -= topic_len + 2 + if op & 6: + pid = await self._as_read(2) + pid = pid[0] << 8 | pid[1] + sz -= 2 + msg = await self._as_read(sz) + retained = op & 0x01 + self._cb(topic, msg, bool(retained)) + if op & 6 == 2: # qos 1 + pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK + struct.pack_into("!H", pkt, 2, pid) + await self._as_write(pkt) + elif op & 6 == 4: # qos 2 not supported + raise OSError(-1) + + +# MQTTClient class. Handles issues relating to connectivity. + +class MQTTClient(MQTT_base): + def __init__(self, client_id=hexlify(unique_id()), + server=None, + port=0, + user='', + password='', + keepalive=60, + ping_interval=0, + ssl=False, + ssl_params=None, + response_time=10, + clean_init=True, + clean=True, + max_repubs=4, + will=None, + subs_cb=lambda *_: None, + wifi_coro=eliza, + connect_coro=eliza, + ssid=None, + wifi_pw=None): + super().__init__(client_id, server, port, user, password, keepalive, ping_interval, + ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, + subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) + self._isconnected = False # Current connection state + keepalive = 1000 * self._keepalive # ms + self._ping_interval = keepalive // 4 if keepalive else 20000 + p_i = self.ping_interval * 1000 # Can specify shorter e.g. for subscribe-only + if p_i and p_i < self._ping_interval: + self._ping_interval = p_i + self._in_connect = False + self._has_connected = False # Define 'Clean Session' value to use. + + async def wifi_connect(self): + s = self._sta_if + # ESP8266 + if s.isconnected(): # 1st attempt, already connected. + return + s.active(True) + s.connect() # ESP8266 remembers connection. + while s.status() == network.STAT_CONNECTING: # Break out on fail or success. Check once per sec. + await asyncio.sleep(1) # Other platforms are OK + + # Ensure connection stays up for a few secs. + t = ticks_ms() + while ticks_diff(ticks_ms(), t) < 5000: + if not s.isconnected(): + raise OSError('WiFi connection fail.') # in 1st 5 secs + await asyncio.sleep(1) + # Timed out: assumed reliable + + async def connect(self): + if not self._has_connected: + await self.wifi_connect() # On 1st call, caller handles error + # Note this blocks if DNS lookup occurs. Do it once to prevent + # blocking during later internet outage: + self._addr = socket.getaddrinfo(self.server, self.port)[0][-1] + self._in_connect = True # Disable low level ._isconnected check + clean = self._clean if self._has_connected else self._clean_init + await self._connect(clean) + # If we get here without error broker/LAN must be up. + self._isconnected = True + self._in_connect = False # Low level code can now check connectivity. + loop = asyncio.get_event_loop() + loop.create_task(self._wifi_handler(True)) # User handler. + if not self._has_connected: + self._has_connected = True # Use normal clean flag on reconnect. + loop.create_task(self._keep_connected()) # Runs forever. + + loop.create_task(self._handle_msg()) # Tasks quit on connection fail. + loop.create_task(self._keep_alive()) + loop.create_task(self._connect_handler(self)) # User handler. + + # Launched by .connect(). Runs until connectivity fails. Checks for and + # handles incoming messages. + async def _handle_msg(self): + try: + while self.isconnected(): + async with self.lock: + await self.wait_msg() # Immediate return if no message + await asyncio.sleep_ms(_DEFAULT_MS) # Let other tasks get lock + + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + + # Keep broker alive MQTT spec 3.1.2.10 Keep Alive. + # Runs until ping failure or no response in keepalive period. + async def _keep_alive(self): + while self.isconnected(): + pings_due = ticks_diff(ticks_ms(), self.last_rx) // self._ping_interval + if pings_due >= 4: + break + elif pings_due >= 1: + try: + await self._ping() + except OSError: + break + await asyncio.sleep(1) + self._reconnect() # Broker or WiFi fail. + + def isconnected(self): + if self._in_connect: # Disable low-level check during .connect() + return True + if self._isconnected and not self._sta_if.isconnected(): # It's going down. + self._reconnect() + return self._isconnected + + def _reconnect(self): # Schedule a reconnection if not underway. + if self._isconnected: + self._isconnected = False + self.close() + loop = asyncio.get_event_loop() + loop.create_task(self._wifi_handler(False)) # User handler. + + # Await broker connection. + async def _connection(self): + while not self._isconnected: + await asyncio.sleep(1) + + # Scheduled on 1st successful connection. Runs forever maintaining wifi and + # broker connection. Must handle conditions at edge of WiFi range. + async def _keep_connected(self): + while True: + if self.isconnected(): # Pause for 1 second + await asyncio.sleep(1) + gc.collect() + else: + self._sta_if.disconnect() + await asyncio.sleep(1) + try: + await self.wifi_connect() + except OSError: + continue + try: + await self.connect() + # Now has set ._isconnected and scheduled _connect_handler(). + except OSError as e: + # Can get ECONNABORTED or -1. The latter signifies no or bad CONNACK received. + self.close() # Disconnect and try again. + self._in_connect = False + self._isconnected = False + + async def subscribe(self, topic, qos=0): + qos_check(qos) + while 1: + await self._connection() + try: + return await super().subscribe(topic, qos) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + + async def unsubscribe(self, topic): + while 1: + await self._connection() + try: + return await super().unsubscribe(topic) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. + + async def publish(self, topic, msg, retain=False, qos=0): + qos_check(qos) + while 1: + await self._connection() + try: + return await super().publish(topic, msg, retain, qos) + except OSError: + pass + self._reconnect() # Broker or WiFi fail. From 842fae3d0e5ea8e00bc23dbfb87f4abb13781dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 2 Jul 2019 12:21:48 +0200 Subject: [PATCH 19/39] Added support for unix port of Micropython, small bugfixes --- README.md | 7 ++++--- README_mqtt_as.md | 21 ++++++++++++++------- config.py | 9 ++++++--- mqtt_as.py | 42 ++++++++++++++++++++++++++++++++---------- remote_mqtt/mqtt.py | 4 ++-- tests/range.py | 4 ++-- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 2d87cfb..360fedd 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ This is due to the changes in point 7 explained below. 1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time 2. Made repo a module to be used like -*from micropython_mqtt_as.mqtt_as import MQTTClient -from micropython_mqtt_as.config import config* +*from micropython_mqtt.mqtt_as import MQTTClient +from micropython_mqtt.config import config* making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. 3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase @@ -17,6 +17,7 @@ making it possible to just clone the repo and copy it to `espXXXX/modules` also 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes 10. Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation) +11. Added support for the unix port of Micropython Motivation for the changes: For my project I had to adapt the library to use it on the ESP32 with loboris fork but also use it on my ESP8266 that is short on RAM all the time. @@ -31,7 +32,7 @@ Therefore I had the following motivation for each of the above mentioned changes 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. 9. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D 10. was simply needed. Sadly makes the module a little bigger - +11. Needed that to run my projects on a Pi # Introduction diff --git a/README_mqtt_as.md b/README_mqtt_as.md index c3311b9..117bdea 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -22,7 +22,7 @@ The official "robust" MQTT client has the following limitations. arrives; this can occur on a WiFi network if an outage occurs at this point in the sequence. - This blocking behaviour implies limited compatibilty with asynchronous + This blocking behaviour implies limited compatibility with asynchronous applications since pending coroutines will not be scheduled for the duration. 2. It is unable reliably to resume operation after a temporary WiFi outage. @@ -62,15 +62,19 @@ modified for resilience and for asynchronous operation. ## 1.3 Project Status The API has changed. Configuration is now via a dictionary, with a default -supplied in the main module. Cross-project settings (e.g. WiFi credentials for -ESP32) may be provided in `config.py` with per-project settings in the -application itself. +supplied in the main module. 1st April 2019 In the light of improved ESP32 firmware and the availability of the Pyboard D the code has had minor changes to support these platforms. The API is unchanged. +2nd July 2019 +Added support for the unix port of Micropython. The unique_id must be set manually +as the unix port doesn't have the function *unique_id()* to read a chip's id. +The library assumes that the device is correctly connected to the network as the OS +will take care of the network connection. + My attempts to test with SSL/TLS have failed. I gather TLS on nonblocking sockets is work in progress. Feedback on this issue would be very welcome. @@ -138,7 +142,7 @@ compiled or (preferably) built as frozen bytecode: copy the repo to `esp8266/modules` in the source tree, build and deploy. If your firmware gets too big, remove all unnecessary files or just copy the ones you need. Minimal requirements: -- directory `micropython_mqtt_as` with these files in it: +- directory `micropython_mqtt` with these files in it: - `__init__.py` to make it a package - `mqtt_as.py` or `mqtt_as_minimal.py` - `config.py` for convenience, optional @@ -152,9 +156,10 @@ with the topic `foo_topic` the topic and message are printed. The code periodically publishes an incrementing count under the topic `result`. ```python -from micropython_mqtt_as.mqtt_as import MQTTClient -from micropython_mqtt_as.config import config +from micropython_mqtt.mqtt_as import MQTTClient +from micropython_mqtt.config import config import uasyncio as asyncio +from sys import platform SERVER = '192.168.0.9' # Change to suit e.g. 'iot.eclipse.org' @@ -177,6 +182,8 @@ async def main(client): config['subs_cb'] = callback config['connect_coro'] = conn_han config['server'] = SERVER +if platform == "linux": + config["client_id"]="linux" MQTTClient.DEBUG = True # Optional: print diagnostic messages client = MQTTClient(**config) diff --git a/config.py b/config.py index 4df5c2b..a626ec7 100644 --- a/config.py +++ b/config.py @@ -3,7 +3,7 @@ # Include any cross-project settings. config = { - 'client_id': hexlify(unique_id()), + 'client_id': None, # will default to hexlify(unique_id()) 'server': None, 'port': 0, 'user': '', @@ -18,8 +18,8 @@ 'max_repubs': 4, 'will': None, 'subs_cb': lambda *_: None, - 'wifi_coro': eliza, - 'connect_coro': eliza, + 'wifi_coro': None, + 'connect_coro': None, 'ssid': None, 'wifi_pw': None, } @@ -37,6 +37,9 @@ config['ssid'] = 'my_SSID' config['wifi_pw'] = 'my_WiFi_password' +if platform == "linux": + config["client_id"] = "linux" # change this to whatever your client_id should be + # For demos ensure the same calling convention for LED's on all platforms. # ESP8266 Feather Huzzah reference board has active low LED's on pins 0 and 2. # ESP32 is assumed to have user supplied active low LED's on same pins. diff --git a/mqtt_as.py b/mqtt_as.py index 0e0ecc9..41c0c25 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -20,8 +20,6 @@ gc.collect() from micropython import const -from machine import unique_id -import network gc.collect() from sys import platform @@ -41,6 +39,14 @@ ESP32 = platform == 'esp32' PYBOARD = platform == 'pyboard' LOBO = platform == 'esp32_LoBo' +LINUX = platform == "linux" + +if LINUX is False: + import network + from machine import unique_id +else: + def unique_id(): + raise NotImplementedError("Linux doesn't have a unique id. Provide the argument client_id") # Default "do little" coro for optional user replacement @@ -126,8 +132,11 @@ def __init__(self, client_id, server, port, user, password, keepalive, ping_inte if self.server is None: raise ValueError('no server specified.') self._sock = None - self._sta_if = network.WLAN(network.STA_IF) - self._sta_if.active(True) + if LINUX is True: + self._sta_isconnected = True + else: + self._sta_if = network.WLAN(network.STA_IF) + self._sta_if.active(True) self.pid = 0 self.rcv_pid = 0 @@ -463,7 +472,7 @@ async def wait_msg(self): # MQTTClient class. Handles issues relating to connectivity. class MQTTClient(MQTT_base): - def __init__(self, client_id=hexlify(unique_id()), + def __init__(self, client_id=None, server=None, port=0, user='', @@ -478,10 +487,13 @@ def __init__(self, client_id=hexlify(unique_id()), max_repubs=4, will=None, subs_cb=lambda *_: None, - wifi_coro=eliza, - connect_coro=eliza, + wifi_coro=None, + connect_coro=None, ssid=None, wifi_pw=None): + client_id = client_id or hexlify(unique_id()) + wifi_coro = wifi_coro or eliza + connect_coro = connect_coro or eliza super().__init__(client_id, server, port, user, password, keepalive, ping_interval, ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) @@ -495,6 +507,9 @@ def __init__(self, client_id=hexlify(unique_id()), self._has_connected = False # Define 'Clean Session' value to use. async def wifi_connect(self): + if LINUX is True: # no network control, assume connected as OS takes care of that + self._sta_isconnected = True + return s = self._sta_if if ESP8266: if s.isconnected(): # 1st attempt, already connected. @@ -597,8 +612,12 @@ async def _memory(self): def isconnected(self): if self._in_connect: # Disable low-level check during .connect() return True - if self._isconnected and not self._sta_if.isconnected(): # It's going down. - self._reconnect() + if LINUX is True: + if self._isconnected and self._sta_isconnected is False: + self._reconnect() + else: + if self._isconnected and not self._sta_if.isconnected(): # It's going down. + self._reconnect() return self._isconnected def _reconnect(self): # Schedule a reconnection if not underway. @@ -621,7 +640,10 @@ async def _keep_connected(self): await asyncio.sleep(1) gc.collect() else: - self._sta_if.disconnect() + if LINUX is True: + self._sta_isconnected = False + else: + self._sta_if.disconnect() # if PYBOARD: # self._sta_if.deinit() await asyncio.sleep(1) diff --git a/remote_mqtt/mqtt.py b/remote_mqtt/mqtt.py index c39a6d1..9aa2f82 100644 --- a/remote_mqtt/mqtt.py +++ b/remote_mqtt/mqtt.py @@ -8,8 +8,8 @@ import gc import ubinascii -from micropython_mqtt_as.mqtt_as import MQTTClient -from micropython_mqtt_as.config import config +from micropython_mqtt.mqtt_as import MQTTClient +from micropython_mqtt.config import config from machine import Pin, unique_id, freq import uasyncio as asyncio gc.collect() diff --git a/tests/range.py b/tests/range.py index 8229686..adf323d 100644 --- a/tests/range.py +++ b/tests/range.py @@ -12,8 +12,8 @@ # blue LED pulse == message received # Publishes connection statistics. -from micropython_mqtt_as.mqtt_as import MQTTClient -from micropython_mqtt_as.config import config, wifi_led, blue_led +from micropython_mqtt.mqtt_as import MQTTClient +from micropython_mqtt.config import config, wifi_led, blue_led import uasyncio as asyncio loop = asyncio.get_event_loop() From 66df3dac66168616b2c3632342f85b158abe94bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 2 Jul 2019 14:29:11 +0200 Subject: [PATCH 20/39] Rollback renaming the package --- README.md | 4 ++-- README_mqtt_as.md | 8 ++++---- mqtt_as.py | 2 +- remote_mqtt/mqtt.py | 4 ++-- tests/clean.py | 2 +- tests/range.py | 4 ++-- tests/unclean.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 360fedd..cdec697 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ This is due to the changes in point 7 explained below. 1. Sorted files and made a structure so you know which file belongs where without reading the documentation every time 2. Made repo a module to be used like -*from micropython_mqtt.mqtt_as import MQTTClient -from micropython_mqtt.config import config* +*from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config* making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. 3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 117bdea..dc7e225 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -122,7 +122,7 @@ the LAN prior to running there is no need explicitly to specify these. On other platforms `config.py` should be edited to provide them. A sample cross-platform file: ```python -from micropython_mqtt.mqtt_as import config +from micropython_mqtt_as.mqtt_as import config config['server'] = '192.168.0.33' # Change to suit e.g. 'iot.eclipse.org' @@ -142,7 +142,7 @@ compiled or (preferably) built as frozen bytecode: copy the repo to `esp8266/modules` in the source tree, build and deploy. If your firmware gets too big, remove all unnecessary files or just copy the ones you need. Minimal requirements: -- directory `micropython_mqtt` with these files in it: +- directory `micropython_mqtt_as` with these files in it: - `__init__.py` to make it a package - `mqtt_as.py` or `mqtt_as_minimal.py` - `config.py` for convenience, optional @@ -156,8 +156,8 @@ with the topic `foo_topic` the topic and message are printed. The code periodically publishes an incrementing count under the topic `result`. ```python -from micropython_mqtt.mqtt_as import MQTTClient -from micropython_mqtt.config import config +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config import uasyncio as asyncio from sys import platform diff --git a/mqtt_as.py b/mqtt_as.py index 41c0c25..a533702 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -15,7 +15,7 @@ import uasyncio as asyncio gc.collect() -from utime import ticks_ms, ticks_diff, sleep_ms +from utime import ticks_ms, ticks_diff from uerrno import EINPROGRESS, ETIMEDOUT gc.collect() diff --git a/remote_mqtt/mqtt.py b/remote_mqtt/mqtt.py index 9aa2f82..c39a6d1 100644 --- a/remote_mqtt/mqtt.py +++ b/remote_mqtt/mqtt.py @@ -8,8 +8,8 @@ import gc import ubinascii -from micropython_mqtt.mqtt_as import MQTTClient -from micropython_mqtt.config import config +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config from machine import Pin, unique_id, freq import uasyncio as asyncio gc.collect() diff --git a/tests/clean.py b/tests/clean.py index fd81a24..ee17c7b 100644 --- a/tests/clean.py +++ b/tests/clean.py @@ -12,7 +12,7 @@ # red LED: ON == WiFi fail # blue LED heartbeat: demonstrates scheduler is running. -from micropython_mqtt.mqtt_as import MQTTClient, config +from micropython_mqtt_as.mqtt_as import MQTTClient, config from config import wifi_led, blue_led # Local definitions import uasyncio as asyncio diff --git a/tests/range.py b/tests/range.py index adf323d..8229686 100644 --- a/tests/range.py +++ b/tests/range.py @@ -12,8 +12,8 @@ # blue LED pulse == message received # Publishes connection statistics. -from micropython_mqtt.mqtt_as import MQTTClient -from micropython_mqtt.config import config, wifi_led, blue_led +from micropython_mqtt_as.mqtt_as import MQTTClient +from micropython_mqtt_as.config import config, wifi_led, blue_led import uasyncio as asyncio loop = asyncio.get_event_loop() diff --git a/tests/unclean.py b/tests/unclean.py index 85209e2..7cc9b78 100644 --- a/tests/unclean.py +++ b/tests/unclean.py @@ -13,7 +13,7 @@ # blue LED heartbeat: demonstrates scheduler is running. # Publishes connection statistics. -from micropython_mqtt.mqtt_as import MQTTClient, config +from micropython_mqtt_as.mqtt_as import MQTTClient, config from config import wifi_led, blue_led import uasyncio as asyncio From f0a90f2b433f77782d47b5027b5a2ae718946a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 12 Sep 2019 09:17:27 +0200 Subject: [PATCH 21/39] Removed minimal version as RAM increase is also minimal by now --- README.md | 4 +- mqtt_as_minimal.py | 565 --------------------------------------------- 2 files changed, 2 insertions(+), 567 deletions(-) delete mode 100644 mqtt_as_minimal.py diff --git a/README.md b/README.md index cdec697..61f2bf6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ from micropython_mqtt_as.config import config* making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. 3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase -5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM +~~5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM~~ Removed again as the removing of workarounds in the main version only got ~150B less RAM usage which is not worth the effort. 6. Added support for "unsubscribe" 7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained]) 8. All other files are updated to the new changes and are usable (e.g. tests). @@ -26,7 +26,7 @@ Therefore I had the following motivation for each of the above mentioned changes 2. Like all modules this should be a directory as well, making usage easier. 3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe) 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. -5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM. +~~5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM.~~ 6. At first I did not need that but later it became important to me so I added it 7. I made a huge workaround in a subclass to recognize retained messages instead of just supporting it directly 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. diff --git a/mqtt_as_minimal.py b/mqtt_as_minimal.py deleted file mode 100644 index d99d79d..0000000 --- a/mqtt_as_minimal.py +++ /dev/null @@ -1,565 +0,0 @@ -# mqtt_as.py Asynchronous version of umqtt.robust -# (C) Copyright Peter Hinch 2017-2019. -# (C) Copyright Kevin Köck 2018-2019. -# Released under the MIT licence. -# Support for Sonoff removed. -# ESP32 hacks removed to reflect improvements to firmware. -# Pyboard D support added - -import gc -import usocket as socket -import ustruct as struct - -gc.collect() -from ubinascii import hexlify -import uasyncio as asyncio - -gc.collect() -from utime import ticks_ms, ticks_diff -from uerrno import EINPROGRESS, ETIMEDOUT - -gc.collect() -from micropython import const -from machine import unique_id -import network - -gc.collect() - -# Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). -_DEFAULT_MS = const(20) -_SOCKET_POLL_DELAY = const(5) # 100ms added greatly to publish latency - -# Legitimate errors while waiting on a socket. See uasyncio __init__.py open_connection(). -BUSY_ERRORS = (EINPROGRESS, ETIMEDOUT,) - - -# Default "do little" coro for optional user replacement -async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program - await asyncio.sleep_ms(_DEFAULT_MS) - - -class MQTTException(Exception): - pass - - -def newpid(pid): - return pid + 1 if pid < 65535 else 1 - - -def qos_check(qos): - if not (qos == 0 or qos == 1): - raise ValueError('Only qos 0 and 1 are supported.') - - -class Lock: - def __init__(self): - self._locked = False - - async def __aenter__(self): - while True: - if self._locked: - await asyncio.sleep_ms(_DEFAULT_MS) - else: - self._locked = True - break - - async def __aexit__(self, *args): - self._locked = False - await asyncio.sleep_ms(_DEFAULT_MS) - - def locked(self): - return self._locked - - def release(self): - self._locked = False - - -# MQTT_base class. Handles MQTT protocol on the basis of a good connection. -# Exceptions from connectivity failures are handled by MQTTClient subclass. -class MQTT_base: - REPUB_COUNT = 0 # TEST - - def __init__(self, client_id, server, port, user, password, keepalive, ping_interval, - ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, - subs_cb, wifi_coro, connect_coro, ssid, wifi_pw): - # MQTT config - self.ping_interval = ping_interval - self._client_id = client_id - self._user = user - self._pswd = password - self._keepalive = keepalive - if self._keepalive >= 65536: - raise ValueError('invalid keepalive time') - self._response_time = response_time * 1000 # Repub if no PUBACK received (ms). - self._max_repubs = max_repubs - self._clean_init = clean_init # clean_session state on first connection - self._clean = clean # clean_session state on reconnect - if will is None: - self._lw_topic = False - else: - self._set_last_will(*will) - # Callbacks and coros - self._cb = subs_cb - self._wifi_handler = wifi_coro - self._connect_handler = connect_coro - # Network - self.port = port - if self.port == 0: - self.port = 8883 if self._ssl else 1883 - self.server = server - if self.server is None: - raise ValueError('no server specified.') - self._sock = None - self._sta_if = network.WLAN(network.STA_IF) - self._sta_if.active(True) - - self.pid = 0 - self.rcv_pid = 0 - self.suback = False - self.last_rx = ticks_ms() # Time of last communication from broker - self.lock = Lock() - self.lock_operation = Lock() - - def _set_last_will(self, topic, msg, retain=False, qos=0): - qos_check(qos) - if not topic: - raise ValueError('Empty topic.') - self._lw_topic = topic - self._lw_msg = msg - self._lw_qos = qos - self._lw_retain = retain - - def _timeout(self, t): - return ticks_diff(ticks_ms(), t) > self._response_time - - async def _as_read(self, n, sock=None): # OSError caught by superclass - if sock is None: - sock = self._sock - data = b'' - t = ticks_ms() - while len(data) < n: - if self._timeout(t) or not self.isconnected(): - raise OSError(-1) - try: - msg = sock.read(n - len(data)) - except OSError as e: # ESP32 issues weird 119 errors here - msg = None - if e.args[0] not in BUSY_ERRORS: - raise - if msg == b'': # Connection closed by host (?) - raise OSError(-1) - if msg is not None: # data received - data = b''.join((data, msg)) - t = ticks_ms() - self.last_rx = ticks_ms() - await asyncio.sleep_ms(_SOCKET_POLL_DELAY) - return data - - async def _as_write(self, bytes_wr, length=0, sock=None): - if sock is None: - sock = self._sock - if length: - bytes_wr = bytes_wr[:length] - t = ticks_ms() - while bytes_wr: - if self._timeout(t) or not self.isconnected(): - raise OSError(-1) - try: - n = sock.write(bytes_wr) - except OSError as e: # ESP32 issues weird 119 errors here - n = 0 - if e.args[0] not in BUSY_ERRORS: - raise - if n: - t = ticks_ms() - bytes_wr = bytes_wr[n:] - await asyncio.sleep_ms(_SOCKET_POLL_DELAY) - - async def _send_str(self, s): - await self._as_write(struct.pack("!H", len(s))) - await self._as_write(s) - - async def _recv_len(self): - n = 0 - sh = 0 - while 1: - res = await self._as_read(1) - b = res[0] - n |= (b & 0x7f) << sh - if not b & 0x80: - return n - sh += 7 - - async def _connect(self, clean): - self._sock = socket.socket() - self._sock.setblocking(False) - try: - self._sock.connect(self._addr) - except OSError as e: - if e.args[0] not in BUSY_ERRORS: - raise - await asyncio.sleep_ms(_DEFAULT_MS) - premsg = bytearray(b"\x10\0\0\0\0\0") - msg = bytearray(b"\x04MQTT\x04\0\0\0") - - sz = 10 + 2 + len(self._client_id) - msg[6] = clean << 1 - if self._user: - sz += 2 + len(self._user) + 2 + len(self._pswd) - msg[6] |= 0xC0 - if self._keepalive: - msg[7] |= self._keepalive >> 8 - msg[8] |= self._keepalive & 0x00FF - if self._lw_topic: - sz += 2 + len(self._lw_topic) + 2 + len(self._lw_msg) - msg[6] |= 0x4 | (self._lw_qos & 0x1) << 3 | (self._lw_qos & 0x2) << 3 - msg[6] |= self._lw_retain << 5 - - i = 1 - while sz > 0x7f: - premsg[i] = (sz & 0x7f) | 0x80 - sz >>= 7 - i += 1 - premsg[i] = sz - await self._as_write(premsg, i + 2) - await self._as_write(msg) - await self._send_str(self._client_id) - if self._lw_topic: - await self._send_str(self._lw_topic) - await self._send_str(self._lw_msg) - if self._user: - await self._send_str(self._user) - await self._send_str(self._pswd) - # Await CONNACK - # read causes ECONNABORTED if broker is out; triggers a reconnect. - resp = await self._as_read(4) - # Got CONNACK - if resp[3] != 0 or resp[0] != 0x20 or resp[1] != 0x02: - raise OSError(-1) # Bad CONNACK e.g. authentication fail. - - async def _ping(self): - async with self.lock: - await self._as_write(b"\xc0\0") - - def close(self): - if self._sock is not None: - self._sock.close() - - # qos == 1: coro blocks until wait_msg gets correct PID. - # If WiFi fails completely subclass re-publishes with new PID. - async def publish(self, topic, msg, retain, qos): - if qos: - async with self.lock_operation: - self.pid = newpid(self.pid) - self.rcv_pid = 0 - count = 0 - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - while 1: # Await PUBACK, republish on timeout - t = ticks_ms() - while self.pid != self.rcv_pid: - await asyncio.sleep_ms(200) - if self._timeout(t) or not self.isconnected(): - break # Must repub or bail out - else: - return # PID's match. All done. - # No match - if count >= self._max_repubs or not self.isconnected(): - raise OSError(-1) # Subclass to re-publish with new PID - async with self.lock: - await self._publish(topic, msg, retain, qos, dup=1) - count += 1 - self.REPUB_COUNT += 1 - else: - async with self.lock: - await self._publish(topic, msg, retain, qos, 0) - - async def _publish(self, topic, msg, retain, qos, dup): - pkt = bytearray(b"\x30\0\0\0") - pkt[0] |= qos << 1 | retain | dup << 3 - sz = 2 + len(topic) + len(msg) - if qos > 0: - sz += 2 - if sz >= 2097152: - raise MQTTException('Strings too long.') - i = 1 - while sz > 0x7f: - pkt[i] = (sz & 0x7f) | 0x80 - sz >>= 7 - i += 1 - pkt[i] = sz - await self._as_write(pkt, i + 1) - await self._send_str(topic) - if qos > 0: - struct.pack_into("!H", pkt, 0, self.pid) - await self._as_write(pkt, 2) - await self._as_write(msg) - - # Can raise OSError if WiFi fails. Subclass traps - async def subscribe(self, topic, qos): - async with self.lock_operation: - self.suback = False - pkt = bytearray(b"\x82\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic) + 1, self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - await self._as_write(qos.to_bytes(1, "little")) - - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) - - # Can raise OSError if WiFi fails. Subclass traps - async def unsubscribe(self, topic): - async with self.lock_operation: - self.suback = False - pkt = bytearray(b"\xa2\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) - - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) - - # Wait for a single incoming MQTT message and process it. - # Subscribed messages are delivered to a callback previously - # set by .setup() method. Other (internal) MQTT - # messages processed internally. - # Immediate return if no data available. Called from ._handle_msg(). - async def wait_msg(self): - res = self._sock.read(1) # Throws OSError on WiFi fail - if res is None: - return - if res == b'': - raise OSError(-1) - - if res == b"\xd0": # PINGRESP - await self._as_read(1) # Update .last_rx time - return - op = res[0] - - if op == 0x40: # PUBACK: save pid - sz = await self._as_read(1) - if sz != b"\x02": - raise OSError(-1) - rcv_pid = await self._as_read(2) - self.rcv_pid = rcv_pid[0] << 8 | rcv_pid[1] - - if op == 0x90: # SUBACK - resp = await self._as_read(4) - if resp[1] != self.pkt[2] or resp[2] != self.pkt[3] or resp[3] == 0x80: - raise OSError(-1) - self.suback = True - - if op == 0xB0: # UNSUBACK - resp = await self._as_read(3) - if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: - raise OSError(-1) - self.suback = True - - if op & 0xf0 != 0x30: - return - sz = await self._recv_len() - topic_len = await self._as_read(2) - topic_len = (topic_len[0] << 8) | topic_len[1] - topic = await self._as_read(topic_len) - sz -= topic_len + 2 - if op & 6: - pid = await self._as_read(2) - pid = pid[0] << 8 | pid[1] - sz -= 2 - msg = await self._as_read(sz) - retained = op & 0x01 - self._cb(topic, msg, bool(retained)) - if op & 6 == 2: # qos 1 - pkt = bytearray(b"\x40\x02\0\0") # Send PUBACK - struct.pack_into("!H", pkt, 2, pid) - await self._as_write(pkt) - elif op & 6 == 4: # qos 2 not supported - raise OSError(-1) - - -# MQTTClient class. Handles issues relating to connectivity. - -class MQTTClient(MQTT_base): - def __init__(self, client_id=hexlify(unique_id()), - server=None, - port=0, - user='', - password='', - keepalive=60, - ping_interval=0, - ssl=False, - ssl_params=None, - response_time=10, - clean_init=True, - clean=True, - max_repubs=4, - will=None, - subs_cb=lambda *_: None, - wifi_coro=eliza, - connect_coro=eliza, - ssid=None, - wifi_pw=None): - super().__init__(client_id, server, port, user, password, keepalive, ping_interval, - ssl, ssl_params, response_time, clean_init, clean, max_repubs, will, - subs_cb, wifi_coro, connect_coro, ssid, wifi_pw) - self._isconnected = False # Current connection state - keepalive = 1000 * self._keepalive # ms - self._ping_interval = keepalive // 4 if keepalive else 20000 - p_i = self.ping_interval * 1000 # Can specify shorter e.g. for subscribe-only - if p_i and p_i < self._ping_interval: - self._ping_interval = p_i - self._in_connect = False - self._has_connected = False # Define 'Clean Session' value to use. - - async def wifi_connect(self): - s = self._sta_if - # ESP8266 - if s.isconnected(): # 1st attempt, already connected. - return - s.active(True) - s.connect() # ESP8266 remembers connection. - while s.status() == network.STAT_CONNECTING: # Break out on fail or success. Check once per sec. - await asyncio.sleep(1) # Other platforms are OK - - if not s.isconnected(): - raise OSError - # Ensure connection stays up for a few secs. - for _ in range(5): - if not s.isconnected(): - raise OSError # in 1st 5 secs - await asyncio.sleep(1) - # Timed out: assumed reliable - - async def connect(self): - if not self._has_connected: - await self.wifi_connect() # On 1st call, caller handles error - # Note this blocks if DNS lookup occurs. Do it once to prevent - # blocking during later internet outage: - self._addr = socket.getaddrinfo(self.server, self.port)[0][-1] - self._in_connect = True # Disable low level ._isconnected check - clean = self._clean if self._has_connected else self._clean_init - await self._connect(clean) - # If we get here without error broker/LAN must be up. - self._isconnected = True - self._in_connect = False # Low level code can now check connectivity. - loop = asyncio.get_event_loop() - loop.create_task(self._wifi_handler(True)) # User handler. - if not self._has_connected: - self._has_connected = True # Use normal clean flag on reconnect. - loop.create_task(self._keep_connected()) # Runs forever. - - loop.create_task(self._handle_msg()) # Tasks quit on connection fail. - loop.create_task(self._keep_alive()) - loop.create_task(self._connect_handler(self)) # User handler. - - # Launched by .connect(). Runs until connectivity fails. Checks for and - # handles incoming messages. - async def _handle_msg(self): - try: - while self.isconnected(): - async with self.lock: - await self.wait_msg() # Immediate return if no message - await asyncio.sleep_ms(_DEFAULT_MS) # Let other tasks get lock - - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - # Keep broker alive MQTT spec 3.1.2.10 Keep Alive. - # Runs until ping failure or no response in keepalive period. - async def _keep_alive(self): - while self.isconnected(): - pings_due = ticks_diff(ticks_ms(), self.last_rx) // self._ping_interval - if pings_due >= 4: - break - elif pings_due >= 1: - try: - await self._ping() - except OSError: - break - await asyncio.sleep(1) - self._reconnect() # Broker or WiFi fail. - - def isconnected(self): - if self._in_connect: # Disable low-level check during .connect() - return True - if self._isconnected and not self._sta_if.isconnected(): # It's going down. - self._reconnect() - return self._isconnected - - def _reconnect(self): # Schedule a reconnection if not underway. - if self._isconnected: - self._isconnected = False - self.close() - loop = asyncio.get_event_loop() - loop.create_task(self._wifi_handler(False)) # User handler. - - # Await broker connection. - async def _connection(self): - while not self._isconnected: - await asyncio.sleep(1) - - # Scheduled on 1st successful connection. Runs forever maintaining wifi and - # broker connection. Must handle conditions at edge of WiFi range. - async def _keep_connected(self): - while True: - if self.isconnected(): # Pause for 1 second - await asyncio.sleep(1) - gc.collect() - else: - self._sta_if.disconnect() - await asyncio.sleep(1) - try: - await self.wifi_connect() - except OSError: - continue - try: - await self.connect() - # Now has set ._isconnected and scheduled _connect_handler(). - except OSError as e: - # Can get ECONNABORTED or -1. The latter signifies no or bad CONNACK received. - self.close() # Disconnect and try again. - self._in_connect = False - self._isconnected = False - - async def subscribe(self, topic, qos=0): - qos_check(qos) - while 1: - await self._connection() - try: - return await super().subscribe(topic, qos) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - async def unsubscribe(self, topic): - while 1: - await self._connection() - try: - return await super().unsubscribe(topic) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. - - async def publish(self, topic, msg, retain=False, qos=0): - qos_check(qos) - while 1: - await self._connection() - try: - return await super().publish(topic, msg, retain, qos) - except OSError: - pass - self._reconnect() # Broker or WiFi fail. From 27615df21be893ebb753aa5e1bf8a6a3792538b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Sat, 19 Oct 2019 16:36:23 +0200 Subject: [PATCH 22/39] Possibility to use timeouts with publish coroutine. Can be implemented similarly for unsubscribe and subscribe. --- tests/mqtt_as_cancel.py => mqtt_as_cancel.py | 3 +++ 1 file changed, 3 insertions(+) rename tests/mqtt_as_cancel.py => mqtt_as_cancel.py (96%) diff --git a/tests/mqtt_as_cancel.py b/mqtt_as_cancel.py similarity index 96% rename from tests/mqtt_as_cancel.py rename to mqtt_as_cancel.py index a54dbcf..4281fd1 100644 --- a/tests/mqtt_as_cancel.py +++ b/mqtt_as_cancel.py @@ -1,4 +1,7 @@ from mqtt_as import MQTTClient as _MQTTClient +import time +import uasyncio as asyncio + class MQTTClient(_MQTTClient): _pub_coro = None From 339e0f5a4dda3e649828b5e60ac6018aa511083a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Sat, 19 Oct 2019 16:42:15 +0200 Subject: [PATCH 23/39] Updated Readme --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 61f2bf6..b2ff172 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ -# WARNING: -The latest commits made this repository NOT USEABLE AS A DROP-IN REPLACEMENT for the original module of Peter Hinch. -This is due to the changes in point 7 explained below. +# NOTE: +With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Looks [here](./README_mqtt_as.md) at 2.3 for an example, it is simple. # Changes to base repo of Peter Hinch @@ -9,11 +8,11 @@ This is due to the changes in point 7 explained below. *from micropython_mqtt_as.mqtt_as import MQTTClient from micropython_mqtt_as.config import config* making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. -3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues). +~~3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues).~~ implemented upstream 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase ~~5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM~~ Removed again as the removing of workarounds in the main version only got ~150B less RAM usage which is not worth the effort. 6. Added support for "unsubscribe" -7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained]) +~~7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained])~~ implemented upstream 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes 10. Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation) @@ -24,7 +23,7 @@ For my project I had to adapt the library to use it on the ESP32 with loboris fo Therefore I had the following motivation for each of the above mentioned changes: 1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. 2. Like all modules this should be a directory as well, making usage easier. -3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe) +~~3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe)~~ 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. ~~5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM.~~ 6. At first I did not need that but later it became important to me so I added it @@ -32,7 +31,7 @@ Therefore I had the following motivation for each of the above mentioned changes 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. 9. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D 10. was simply needed. Sadly makes the module a little bigger -11. Needed that to run my projects on a Pi +11. Needed that to run my projects on a Pi and it's great for testing code # Introduction From a6c35b46dfc3fbbd75b4f2863f92138deac5d166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Sat, 19 Oct 2019 16:43:38 +0200 Subject: [PATCH 24/39] Updated Readme --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b2ff172..f6366da 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # NOTE: -With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Looks [here](./README_mqtt_as.md) at 2.3 for an example, it is simple. +With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Look [here](./README_mqtt_as.md) at 2.3 for an example, it is simple. # Changes to base repo of Peter Hinch @@ -8,11 +8,11 @@ With the latest commits in the original repository by Peter Hinch, this module i *from micropython_mqtt_as.mqtt_as import MQTTClient from micropython_mqtt_as.config import config* making it possible to just clone the repo and copy it to `espXXXX/modules` also reducing file clutter in this directory. -~~3. Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues).~~ implemented upstream +3. ~~Removed unnecessary workarounds of official ESP32 port for ESP32 loboris fork (Feel free to report issues).~~ implemented upstream 4. Changed MQTTClient constructor initialization from using a dictionary to using keywords with default parameters. It's still possible to use the dictionary for initialization with almost no changes to existing codebase -~~5. Made a minimal version of mqtt_as for the ESP8266 to save some RAM~~ Removed again as the removing of workarounds in the main version only got ~150B less RAM usage which is not worth the effort. +5. ~~Made a minimal version of mqtt_as for the ESP8266 to save some RAM~~ Removed again as the removing of workarounds in the main version only got ~150B less RAM usage which is not worth the effort. 6. Added support for "unsubscribe" -~~7. Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained])~~ implemented upstream +7. ~~Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained])~~ implemented upstream 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes 10. Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation) @@ -23,9 +23,9 @@ For my project I had to adapt the library to use it on the ESP32 with loboris fo Therefore I had the following motivation for each of the above mentioned changes: 1. I don't like to walk through a mess of files not knowing which one is important or where it belongs to and I don't want to read all the documentation just to know which files belong where. 2. Like all modules this should be a directory as well, making usage easier. -~~3. Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe)~~ +3. ~~Made it work with loboris fork but did not want to use workarounds that are not needed on this fork. (Peter Hinch made it work with loboris port as well but has the workarounds still in it to be safe)~~ 4. I felt that this kind of initialization is the more pythonic way of doing things but apart from that it has an important advantage on the ESP8266, removing the config dict completely uses 100-200 Bytes less, which is important on ESP8266. -~~5. This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM.~~ +5. ~~This version for the ESP8266 has all non related code (workarounds for ESP32) and also some not commonly functions removed, saving another 150-250 Bytes so that after all changes I get 250-450 Bytes more RAM which is about 2% of the available RAM.~~ 6. At first I did not need that but later it became important to me so I added it 7. I made a huge workaround in a subclass to recognize retained messages instead of just supporting it directly 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. From 7fc254268541cd4bd270d2863f929bdbf5dd4683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 31 Oct 2019 12:31:06 +0100 Subject: [PATCH 25/39] Pulled upstream patches, adapted unsubscribe and removed locks added as workarounds. Concurrent publish/(un)subscribe operations will now work correctly. The locks workaround making them execute after each other is not needed anymore. --- mqtt_as.py | 39 ++++++++++++++++++++------------------- tests/asyn.py | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/mqtt_as.py b/mqtt_as.py index e5933cc..75e87b5 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -2,7 +2,7 @@ # (C) Copyright Peter Hinch 2017-2019. # (C) Copyright Kevin Köck 2018-2019. # Released under the MIT licence. -# Support for Sonoff removed. +# Support for Sonoff removed. Newer Sonoff devices work reliable without any workarounds. # ESP32 hacks removed to reflect improvements to firmware. # Pyboard D support added # Patch for retained message support supplied by Kevin Köck. @@ -21,8 +21,6 @@ gc.collect() from micropython import const -from machine import unique_id -import network gc.collect() from sys import platform @@ -402,21 +400,18 @@ async def subscribe(self, topic, qos): # Can raise OSError if WiFi fails. Subclass traps async def unsubscribe(self, topic): - async with self.lock_operation: - self.suback = False - pkt = bytearray(b"\xa2\0\0\0") - self.pid = newpid(self.pid) - struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), self.pid) - self.pkt = pkt - async with self.lock: - await self._as_write(pkt) - await self._send_str(topic) + pkt = bytearray(b"\xa2\0\0\0") + pid = newpid(self.pid) + self.pid = pid # will otherwise result in multiple operations having the same pid + self.rcv_pids.add(pid) + self.dprint("unsubscribe", topic, "pid", pid) + struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), pid) + async with self.lock: + await self._as_write(pkt) + await self._send_str(topic) - t = ticks_ms() - while not self.suback: - await asyncio.sleep_ms(200) - if self._timeout(t): - raise OSError(-1) + if not await self._await_pid(pid): + raise OSError(-1) # Wait for a single incoming MQTT message and process it. # Subscribed messages are delivered to a callback previously @@ -473,10 +468,16 @@ async def wait_msg(self): # self.dprint("rcv_pids after pruning:", self.rcv_pids) if op == 0xB0: # UNSUBACK + self.dprint("received unsuback") resp = await self._as_read(3) - if resp[1] != self.pkt[2] or resp[2] != self.pkt[3]: + pid = resp[2] | (resp[1] << 8) + self.dprint("got unsuback pid", pid) + if pid in self.rcv_pids: + self.rcv_pids.discard(pid) + else: + self.dprint("UNSUBACK unknown pid", pid) raise OSError(-1) - self.suback = True + self.dprint("rcv_pids:", self.rcv_pids) if op & 0xf0 != 0x30: return diff --git a/tests/asyn.py b/tests/asyn.py index b49256a..8ae4d06 100644 --- a/tests/asyn.py +++ b/tests/asyn.py @@ -336,7 +336,7 @@ def new_gen(*args, **kwargs): args = (args[0],) + args[2:] g = f(*args, **kwargs) try: - res = await g + res = yield g return res finally: NamedTask._stopped(task_id) From 1e55dd2d9ca096f0438bd5d1e6464973b6f91aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 31 Oct 2019 12:47:57 +0100 Subject: [PATCH 26/39] Broke the Lock by removing locked() and released() which other components might depend on. --- mqtt_as.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mqtt_as.py b/mqtt_as.py index 75e87b5..0deae00 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -84,6 +84,12 @@ async def __aexit__(self, *args): self._locked = False await asyncio.sleep_ms(_DEFAULT_MS) + def locked(self): + return self._locked + + def release(self): + self._locked = False + # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. From 2e393325a037a10087d5be90914fa2ed01eaf2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 31 Oct 2019 13:43:33 +0100 Subject: [PATCH 27/39] Updated README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f6366da..0a01d96 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ making it possible to just clone the repo and copy it to `espXXXX/modules` also 7. ~~Added support for recognizing retained publications (makes change in "subs_cb" necessary as it now has to take 3 args [topic,msg,retained])~~ implemented upstream 8. All other files are updated to the new changes and are usable (e.g. tests). 9. Updated documentation to reflect all changes -10. Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation) +10. ~~Fixes a reliability problem when having many subscribe/unsubscribe in a short time, resulting endless reconnects (see commit for changes and explanation)~~ fixed upstream, real concurrent operations possible now 11. Added support for the unix port of Micropython Motivation for the changes: @@ -30,7 +30,7 @@ Therefore I had the following motivation for each of the above mentioned changes 7. I made a huge workaround in a subclass to recognize retained messages instead of just supporting it directly 8. Although I do not need any other file I felt that it is important to finish the work I started and not leave half the repo unusable. 9. Wouldn't want issues because of wrong documentation or frustrated users. Have fun with it :D -10. was simply needed. Sadly makes the module a little bigger +10. ~~was simply needed. Sadly makes the module a little bigger~~ 11. Needed that to run my projects on a Pi and it's great for testing code # Introduction From 75f8997a8d0e9c95e54c3d91506ec49b4e9dd0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 31 Oct 2019 13:45:03 +0100 Subject: [PATCH 28/39] Updated README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a01d96..5332e86 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # NOTE: -With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Look [here](./README_mqtt_as.md) at 2.3 for an example, it is simple. +With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Look [here](./README_mqtt_as.md#23-example-usage) for an example, it is simple, the pythonic way. # Changes to base repo of Peter Hinch From 42ae4eb250f6276e9b0971763d08fefb74c72a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 31 Oct 2019 13:46:56 +0100 Subject: [PATCH 29/39] Updated README --- README_mqtt_as.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README_mqtt_as.md b/README_mqtt_as.md index c07b64e..44e9c6b 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -205,7 +205,9 @@ if platform == "linux": config["client_id"]="linux" MQTTClient.DEBUG = True # Optional: print diagnostic messages -client = MQTTClient(**config) +client = MQTTClient(**config) # Using dict to stay compatible to upstream. +# Alternatively initialize MQTTClient the pythonic way using arguments like: +# client = MQTTClient(server=SERVER, port=1883, ...) loop = asyncio.get_event_loop() try: loop.run_until_complete(main(client)) From bd6b80e67beb4d30a1146b28d04d82eab0546402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 4 Nov 2019 16:27:04 +0100 Subject: [PATCH 30/39] Merged upstream patches --- README_mqtt_as.md | 11 ++- mqtt_as.py | 17 ++-- mqtt_as_OOM_protection.py | 10 ++- .../mqtt_as_timeout.py => mqtt_as_timeout.py | 3 +- ...y~1eae5d0a182646f65988add0b030c61361c23639 | 85 ------------------- ...y~1eae5d0a182646f65988add0b030c61361c23639 | 63 -------------- 6 files changed, 20 insertions(+), 169 deletions(-) rename tests/mqtt_as_timeout.py => mqtt_as_timeout.py (97%) delete mode 100644 tests/tls.py~1eae5d0a182646f65988add0b030c61361c23639 delete mode 100644 tests/tls32.py~1eae5d0a182646f65988add0b030c61361c23639 diff --git a/README_mqtt_as.md b/README_mqtt_as.md index 918f356..81da247 100644 --- a/README_mqtt_as.md +++ b/README_mqtt_as.md @@ -117,16 +117,16 @@ the retained message flag. On ESP8266 the code disables automatic sleep: this reduces reconnects at cost of increased power consumption. -1st April 2019 -In the light of improved ESP32 firmware and the availability of the Pyboard D -the code has minor changes to support these platforms. - 2nd July 2019 Added support for the unix port of Micropython. The unique_id must be set manually as the unix port doesn't have the function *unique_id()* to read a chip's id. The library assumes that the device is correctly connected to the network as the OS will take care of the network connection. +1st April 2019 +In the light of improved ESP32 firmware and the availability of the Pyboard D +the code has minor changes to support these platforms. + My attempts to test with SSL/TLS have failed. I gather TLS on nonblocking sockets is work in progress. Feedback on this issue would be very welcome. @@ -462,7 +462,6 @@ to '8.8.8.8' and checks for a valid response. There is a single arg `packet` which is a bytes object being the DNS query. The default object queries the Google DNS server. -## 3.3 Class Variables ### 3.2.9 unsubscribe (async) Unsubscribes a topic, so no messages will be received anymore. @@ -473,7 +472,7 @@ necessary reconnecting to a failed network. Args: 1. `topic` -## 3.3 Class Attributes +## 3.3 Class Variables 1. `DEBUG` If `True` causes diagnostic messages to be printed. 2. `REPUB_COUNT` For debug purposes. Logs the total number of republications diff --git a/mqtt_as.py b/mqtt_as.py index d1f945f..0b341a0 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -59,12 +59,14 @@ async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program class MQTTException(Exception): pass + def pid_gen(): pid = 0 while True: pid = pid + 1 if pid < 65535 else 1 yield pid + def qos_check(qos): if not (qos == 0 or qos == 1): raise ValueError('Only qos 0 and 1 are supported.') @@ -119,7 +121,7 @@ def __init__(self, client_id, server, port, user, password, keepalive, ping_inte else: self._set_last_will(*will) # WiFi config - self._ssid = ssid # For ESP32 / Pyboard D + self._ssid = ssid # Required ESP32 / Pyboard D self._wifi_pw = wifi_pw self._ssl = ssl self._ssl_params = ssl_params @@ -397,10 +399,8 @@ async def subscribe(self, topic, qos): # Can raise OSError if WiFi fails. Subclass traps async def unsubscribe(self, topic): pkt = bytearray(b"\xa2\0\0\0") - pid = newpid(self.pid) - self.pid = pid # will otherwise result in multiple operations having the same pid + pid = next(self.newpid) self.rcv_pids.add(pid) - self.dprint("unsubscribe", topic, "pid", pid) struct.pack_into("!BH", pkt, 1, 2 + 2 + len(topic), pid) async with self.lock: await self._as_write(pkt) @@ -448,16 +448,12 @@ async def wait_msg(self): raise OSError(-1) if op == 0xB0: # UNSUBACK - self.dprint("received unsuback") resp = await self._as_read(3) pid = resp[2] | (resp[1] << 8) - self.dprint("got unsuback pid", pid) if pid in self.rcv_pids: self.rcv_pids.discard(pid) else: - self.dprint("UNSUBACK unknown pid", pid) raise OSError(-1) - self.dprint("rcv_pids:", self.rcv_pids) if op & 0xf0 != 0x30: return @@ -590,7 +586,8 @@ async def connect(self): loop.create_task(self._wifi_handler(True)) # User handler. if not self._has_connected: self._has_connected = True # Use normal clean flag on reconnect. - loop.create_task(self._keep_connected()) # Runs forever unless user issues .disconnect() + loop.create_task( + self._keep_connected()) # Runs forever unless user issues .disconnect() loop.create_task(self._handle_msg()) # Tasks quit on connection fail. loop.create_task(self._keep_alive()) @@ -673,8 +670,6 @@ async def _keep_connected(self): self._sta_isconnected = False else: self._sta_if.disconnect() - # if PYBOARD: - # self._sta_if.deinit() await asyncio.sleep(1) try: await self.wifi_connect() diff --git a/mqtt_as_OOM_protection.py b/mqtt_as_OOM_protection.py index 4d2fd6d..72a70de 100644 --- a/mqtt_as_OOM_protection.py +++ b/mqtt_as_OOM_protection.py @@ -1,6 +1,12 @@ # Author: Kevin Köck # Copyright Kevin Köck 2019 Released under the MIT license -# Created on 2019-10-28 +# Created on 2019-10-28 + +# This is basically just a paranoid change protecting the device from crashing +# with a memory allocation error if it receives a message not fitting in RAM. +# It will still receive and process the message but the arguments "topic" or "msg" +# of the callback might be None depending on which values couldn't be received +# because of the memory allocation error. __updated__ = "2019-10-28" __version__ = "0.1" @@ -33,7 +39,7 @@ async def _as_read(self, n, sock=None): # OSError caught by superclass # received later and lead to buffer overflows and keepalive timeout anyway # so best to terminate the connection raise OSError - if msg == b'': # Connection closed by host (?) + if msg == b'': # Connection closed by host raise OSError(-1) if msg is not None and data is not None: # data received try: diff --git a/tests/mqtt_as_timeout.py b/mqtt_as_timeout.py similarity index 97% rename from tests/mqtt_as_timeout.py rename to mqtt_as_timeout.py index a3b9bd5..b2d97e0 100644 --- a/tests/mqtt_as_timeout.py +++ b/mqtt_as_timeout.py @@ -23,6 +23,7 @@ import time import uasyncio as asyncio + class MQTTClient(_MQTTClient): _pub_coro = None @@ -34,8 +35,6 @@ async def _connection(self): async def _publishTimeout(self, topic, msg, retain, qos): try: await super().publish(topic, msg, retain, qos) - except asyncio.CancelledError: - pass finally: self._pub_coro = None diff --git a/tests/tls.py~1eae5d0a182646f65988add0b030c61361c23639 b/tests/tls.py~1eae5d0a182646f65988add0b030c61361c23639 deleted file mode 100644 index d144ead..0000000 --- a/tests/tls.py~1eae5d0a182646f65988add0b030c61361c23639 +++ /dev/null @@ -1,85 +0,0 @@ -# tls.py Test of asynchronous mqtt client with SSL for Pyboard D. Tested OK. - -# (C) Copyright Peter Hinch 2017-2019. -# Released under the MIT licence. - -# This demo publishes to topic "result" and also subscribes to that topic. -# This demonstrates bidirectional TLS communication. -# You can also run the following on a PC to verify: -# mosquitto_sub -h test.mosquitto.org -t result -# I haven't yet figured out how to get mosquitto_sub to use a secure connection. - -# Public brokers https://github.com/mqtt/mqtt.github.io/wiki/public_brokers - -# red LED: ON == WiFi fail -# green LED heartbeat: demonstrates scheduler is running. - -from mqtt_as import MQTTClient -from config import config -import uasyncio as asyncio -from pyb import LED - -SERVER = 'test.mosquitto.org' - -loop = asyncio.get_event_loop() - -# Subscription callback -async def flash(): - LED(3).on() - await asyncio.sleep_ms(500) - LED(3).off() - -sub_led = LED(3) # Blue -def sub_cb(topic, msg, retained): - c, r = [int(x) for x in msg.decode().split(' ')] - print('Topic = {} Count = {} Retransmissions = {} Retained = {}'.format(topic.decode(), c, r, retained)) - loop.create_task(flash()) - -# Demonstrate scheduler is operational and TLS is nonblocking. -async def heartbeat(): - led = LED(2) # Green - while True: - await asyncio.sleep_ms(500) - led.toggle() - -wifi_led = LED(1) # LED on for WiFi fail/not ready yet - -async def wifi_han(state): - if state: - wifi_led.off() - else: - wifi_led.on() - print('Wifi is ', 'up' if state else 'down') - await asyncio.sleep(1) - -# If you connect with clean_session True, must re-subscribe (MQTT spec 3.1.2.4) -async def conn_han(client): - await client.subscribe('result', 1) - -async def main(client): - await client.connect() - n = 0 - await asyncio.sleep(2) # Give broker time - while True: - print('publish', n) - # If WiFi is down the following will pause for the duration. - await client.publish('result', '{} {}'.format(n, client.REPUB_COUNT), qos = 1) - n += 1 - await asyncio.sleep(20) # Broker is slow - -# Define configuration -config['subs_cb'] = sub_cb -config['server'] = SERVER -config['connect_coro'] = conn_han -config['wifi_coro'] = wifi_han -config['ssl'] = True - -# Set up client -MQTTClient.DEBUG = True # Optional -client = MQTTClient(config) - -loop.create_task(heartbeat()) -try: - loop.run_until_complete(main(client)) -finally: - client.close() diff --git a/tests/tls32.py~1eae5d0a182646f65988add0b030c61361c23639 b/tests/tls32.py~1eae5d0a182646f65988add0b030c61361c23639 deleted file mode 100644 index 5d286d4..0000000 --- a/tests/tls32.py~1eae5d0a182646f65988add0b030c61361c23639 +++ /dev/null @@ -1,63 +0,0 @@ -# tls32.py Test of asynchronous mqtt client with SSL for ESP32. Fails with -# mbedtls_ssl_handshake error: -77 -# Please help me fix it. - -# (C) Copyright Peter Hinch 2017-2019. -# Released under the MIT licence. - -# This demo publishes to topic "result" and also subscribes to that topic. -# This demonstrates bidirectional TLS communication. -# You can also run the following on a PC to verify: -# mosquitto_sub -h test.mosquitto.org -t result -# I haven't yet figured out how to get mosquitto_sub to use a secure connection. - -# Public brokers https://github.com/mqtt/mqtt.github.io/wiki/public_brokers - -# red LED: ON == WiFi fail -# green LED heartbeat: demonstrates scheduler is running. - -from mqtt_as import MQTTClient -from config import config -import uasyncio as asyncio - -SERVER = 'test.mosquitto.org' - -loop = asyncio.get_event_loop() - -def sub_cb(topic, msg, retained): - c, r = [int(x) for x in msg.decode().split(' ')] - print('Topic = {} Count = {} Retransmissions = {} Retained = {}'.format(topic.decode(), c, r, retained)) - -async def wifi_han(state): - print('Wifi is ', 'up' if state else 'down') - await asyncio.sleep(1) - -# If you connect with clean_session True, must re-subscribe (MQTT spec 3.1.2.4) -async def conn_han(client): - await client.subscribe('result', 1) - -async def main(client): - await client.connect() - n = 0 - await asyncio.sleep(2) # Give broker time - while True: - print('publish', n) - # If WiFi is down the following will pause for the duration. - await client.publish('result', '{} {}'.format(n, client.REPUB_COUNT), qos = 1) - n += 1 - await asyncio.sleep(20) # Broker is slow - -# Define configuration -config['subs_cb'] = sub_cb -config['server'] = SERVER -config['connect_coro'] = conn_han -config['wifi_coro'] = wifi_han -config['ssl'] = True - -# Set up client -MQTTClient.DEBUG = True # Optional -client = MQTTClient(config) -try: - loop.run_until_complete(main(client)) -finally: - client.close() From e0b0646f4d3b2f4d9faf91ee325c362116fa6e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 4 Nov 2019 16:34:38 +0100 Subject: [PATCH 31/39] delete .idea --- .idea/.gitignore | 2 -- .idea/encodings.xml | 6 ------ .idea/inspectionProfiles/profiles_settings.xml | 5 ----- .idea/libraries/MicroPython.xml | 10 ---------- .idea/micropython-mqtt.iml | 16 ---------------- .idea/misc.xml | 7 ------- .idea/modules.xml | 8 -------- .idea/vagrant.xml | 7 ------- .idea/vcs.xml | 6 ------ 9 files changed, 67 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/libraries/MicroPython.xml delete mode 100644 .idea/micropython-mqtt.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vagrant.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 5c98b42..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Default ignored files -/workspace.xml \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 97626ba..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 0eefe32..0000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/libraries/MicroPython.xml b/.idea/libraries/MicroPython.xml deleted file mode 100644 index ef729b0..0000000 --- a/.idea/libraries/MicroPython.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/micropython-mqtt.iml b/.idea/micropython-mqtt.iml deleted file mode 100644 index 47e8b45..0000000 --- a/.idea/micropython-mqtt.iml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 8656114..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 5074206..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vagrant.xml b/.idea/vagrant.xml deleted file mode 100644 index a5aa786..0000000 --- a/.idea/vagrant.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From edf1ea035a12a77dfb1c341ef586c88d84334120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 4 Nov 2019 16:36:03 +0100 Subject: [PATCH 32/39] remove gitignore --- .gitignore | 103 ----------------------------------------------------- 1 file changed, 103 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 01e67db..0000000 --- a/.gitignore +++ /dev/null @@ -1,103 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -/.project -/.pydevproject From eb66b972af8f926ed529be673e8da689ddcc7a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 4 Nov 2019 22:04:02 +0100 Subject: [PATCH 33/39] concurrent timeout subclass for all operations --- mqtt_as_timeout_concurrent.py | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 mqtt_as_timeout_concurrent.py diff --git a/mqtt_as_timeout_concurrent.py b/mqtt_as_timeout_concurrent.py new file mode 100644 index 0000000..51914a2 --- /dev/null +++ b/mqtt_as_timeout_concurrent.py @@ -0,0 +1,75 @@ +# Author: Kevin Köck +# Copyright Kevin Köck 2019 Released under the MIT license +# Created on 2019-11-04 + +__updated__ = "2019-11-04" +__version__ = "0.1" + +from .mqtt_as import MQTTClient as _MQTTClient +import uasyncio as asyncio +import time + + +class MQTTClient(_MQTTClient): + _ops_coros = [None] * 10 + + async def publish(self, topic, msg, retain=False, qos=0, timeout=None, await_connection=True): + return await self._preprocessor(super().publish, topic, msg, retain, qos, + timeout=timeout, await_connection=await_connection) + + async def subscribe(self, topic, qos=0, timeout=None, await_connection=True): + await self._preprocessor(super().subscribe, topic, qos, timeout=timeout, + await_connection=await_connection) + + async def unsubscribe(self, topic, timeout=None, await_connection=False): + # with clean sessions a connection loss is basically a successful unsubscribe + return await self._preprocessor(super().unsubscribe, topic, timeout=timeout, + await_connection=False) + + # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms + async def _connection(self): + while not self._isconnected: + await asyncio.sleep_ms(50) + + async def _operationTimeout(self, coro, *args, slot): + try: + await coro(*args) + finally: + self._ops_coros[slot] = None + + async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=True): + start = time.ticks_ms() + slot = None + coro = None + try: + while timeout is None or time.ticks_diff(time.ticks_ms(), start) < timeout * 1000: + if not await_connection and not self._isconnected: + return False + if slot is None: + # wait for slot in queue + for i, c in enumerate(self._ops_coros): + if c is None: + slot = i + break + elif slot and self._ops_coros[slot] is coro is None: + # create task + coro = self._operationTimeout(coroutine, *args, slot=slot) + asyncio.get_event_loop().create_task(coro) + self._ops_coros[slot] = coro + elif self._ops_coros[slot] != coro: + # Slot either None or already new coro assigned. + # Means the operation finished successfully. + return True + await asyncio.sleep_ms(20) + self.dprint("timeout on", args) + except asyncio.CancelledError: + raise # the caller of this coro should be cancelled too + finally: + if coro and self._ops_coros[slot] == coro: + # coro still active, cancel it + async with self.lock: + asyncio.cancel(coro) + # self._ops_coros[slot] = None is done by finally in _operationTimeout + return False + # else: returns return value during process + return False From 5ea7b829dacfc4f7c8bd5bf76e03252e9d3a85e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Mon, 4 Nov 2019 22:31:20 +0100 Subject: [PATCH 34/39] concurrent timeout subclass for all operations with no queue limitation --- mqtt_as_timeout_concurrent.py | 47 ++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/mqtt_as_timeout_concurrent.py b/mqtt_as_timeout_concurrent.py index 51914a2..c00ae0e 100644 --- a/mqtt_as_timeout_concurrent.py +++ b/mqtt_as_timeout_concurrent.py @@ -3,7 +3,7 @@ # Created on 2019-11-04 __updated__ = "2019-11-04" -__version__ = "0.1" +__version__ = "0.2" from .mqtt_as import MQTTClient as _MQTTClient import uasyncio as asyncio @@ -11,7 +11,7 @@ class MQTTClient(_MQTTClient): - _ops_coros = [None] * 10 + _ops_coros = set() async def publish(self, topic, msg, retain=False, qos=0, timeout=None, await_connection=True): return await self._preprocessor(super().publish, topic, msg, retain, qos, @@ -35,41 +35,48 @@ async def _operationTimeout(self, coro, *args, slot): try: await coro(*args) finally: - self._ops_coros[slot] = None + for obj in self._ops_coros: + if obj[0] == slot: + self._ops_coros.discard(obj) + return async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=True): start = time.ticks_ms() - slot = None coro = None try: while timeout is None or time.ticks_diff(time.ticks_ms(), start) < timeout * 1000: if not await_connection and not self._isconnected: return False - if slot is None: - # wait for slot in queue - for i, c in enumerate(self._ops_coros): - if c is None: - slot = i + elif coro is None: + # search for unused identifier + found = False + identifier = None + for i in range(1024): + for obj in self._ops_coros: + if obj[0] == i: + found = True + break + if not found: # id unique + identifier = i break - elif slot and self._ops_coros[slot] is coro is None: # create task - coro = self._operationTimeout(coroutine, *args, slot=slot) - asyncio.get_event_loop().create_task(coro) - self._ops_coros[slot] = coro - elif self._ops_coros[slot] != coro: - # Slot either None or already new coro assigned. - # Means the operation finished successfully. + task = self._operationTimeout(coroutine, *args, slot=identifier) + asyncio.get_event_loop().create_task(task) + coro = (identifier, task) + self._ops_coros.add(coro) + elif coro not in self._ops_coros: + # coro removed, so operation was successful return True await asyncio.sleep_ms(20) self.dprint("timeout on", args) except asyncio.CancelledError: raise # the caller of this coro should be cancelled too finally: - if coro and self._ops_coros[slot] == coro: + if coro and coro in self._ops_coros: # coro still active, cancel it async with self.lock: - asyncio.cancel(coro) - # self._ops_coros[slot] = None is done by finally in _operationTimeout + asyncio.cancel(coro[1]) + # self._ops_coros.discard(coro) is done by finally in _operationTimeout return False - # else: returns return value during process + # else: returns return value during process, which is True in case it was successful return False From a5d14047d4d92e23f4cb39874055694dbd219412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 5 Nov 2019 08:17:52 +0100 Subject: [PATCH 35/39] bugfix, testcase --- mqtt_as_timeout_concurrent.py | 5 +- tests/timeout_concurrent.py | 111 ++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 tests/timeout_concurrent.py diff --git a/mqtt_as_timeout_concurrent.py b/mqtt_as_timeout_concurrent.py index c00ae0e..c4a4020 100644 --- a/mqtt_as_timeout_concurrent.py +++ b/mqtt_as_timeout_concurrent.py @@ -5,7 +5,7 @@ __updated__ = "2019-11-04" __version__ = "0.2" -from .mqtt_as import MQTTClient as _MQTTClient +from micropython_mqtt_as.mqtt_as import MQTTClient as _MQTTClient import uasyncio as asyncio import time @@ -49,9 +49,9 @@ async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=T return False elif coro is None: # search for unused identifier - found = False identifier = None for i in range(1024): + found = False for obj in self._ops_coros: if obj[0] == i: found = True @@ -66,6 +66,7 @@ async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=T self._ops_coros.add(coro) elif coro not in self._ops_coros: # coro removed, so operation was successful + self.dprint("Success on", args) return True await asyncio.sleep_ms(20) self.dprint("timeout on", args) diff --git a/tests/timeout_concurrent.py b/tests/timeout_concurrent.py new file mode 100644 index 0000000..498e70d --- /dev/null +++ b/tests/timeout_concurrent.py @@ -0,0 +1,111 @@ +# Author: Kevin Köck +# Copyright Kevin Köck 2019 Released under the MIT license +# Created on 2019-11-05 + +__updated__ = "2019-11-05" +__version__ = "0.1" + +from ..mqtt_as_timeout_concurrent import MQTTClient + +import uasyncio as asyncio + +loop = asyncio.get_event_loop(waitq_len=60, runq_len=60) + + +async def publish(val, t): + print("Publishing", val, "timeout", t) + await client.publish("foo_topic", val, qos=1, timeout=t) + + +def callback(topic, msg, retained): + print((topic, msg, retained)) + + +first = True + + +async def conn_han(client): + global first + if first: + # await client.subscribe('foo_topic', 1) + loop = asyncio.get_event_loop() + loop.create_task(publish("payload {!s}".format(1), 1)) + loop.create_task(publish("payload {!s}".format(2), 2)) + loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) + loop.create_task(publish("payload {!s}".format(4), 4)) + loop.create_task(publish("payload {!s}".format(5), 5)) + loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) + first = False + await asyncio.sleep(1) + print("Closing connection") + await client.disconnect() + await asyncio.sleep(5) + print("Publishing disconnected") + loop.create_task(publish("payload {!s}".format(1), 1)) + loop.create_task(publish("payload {!s}".format(2), 2)) + loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) + loop.create_task(publish("payload {!s}".format(4), 4)) + loop.create_task(publish("payload {!s}".format(5), 5)) + loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) + await asyncio.sleep(10) + print("Reconnecting after all timeouts") + await client.connect() + loop.create_task(publish("payload {!s}".format(8), 8)) + await asyncio.sleep(5) + print("Test done") + await client.disconnect() + + +import config +from ubinascii import hexlify +from machine import unique_id + + +async def wifi(state): + print("WIFI state", state) + + +async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program + await asyncio.sleep_ms(20) + + +config_dict = { + 'client_id': hexlify(unique_id()), + 'server': config.MQTT_HOST, + 'port': config.MQTT_PORT, + 'user': config.MQTT_USER, + 'password': config.MQTT_PASSWORD, + 'keepalive': 60, + 'ping_interval': 0, + 'ssl': False, + 'ssl_params': {}, + 'response_time': 10, + 'clean_init': True, + 'clean': True, + 'max_repubs': 4, + 'will': None, + 'subs_cb': lambda *_: None, + 'wifi_coro': wifi, + 'connect_coro': eliza, + 'ssid': None, + 'wifi_pw': None, +} +config_dict['connect_coro'] = conn_han +config_dict['subs_cb'] = callback + +client = MQTTClient(**config_dict) +client.DEBUG = True + + +async def main(client): + await client.connect() + n = 0 + while True: + await asyncio.sleep(5) + + +def test(): + try: + loop.run_until_complete(main(client)) + finally: + client.close() # Prevent LmacRxBlk:1 errors From e840a6d48d69ff8f74bec9b986e115eff0e95af6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Wed, 6 Nov 2019 09:01:53 +0100 Subject: [PATCH 36/39] timeout option with wait_for uasyncio extension from https://github.com/kevinkk525/micropython_uasyncio_extension --- mqtt_as_timeout_concurrent.py | 6 +- mqtt_as_timeout_concurrent2.py | 129 +++++++++++++++++++++++++++++++++ tests/timeout_concurrent2.py | 115 +++++++++++++++++++++++++++++ 3 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 mqtt_as_timeout_concurrent2.py create mode 100644 tests/timeout_concurrent2.py diff --git a/mqtt_as_timeout_concurrent.py b/mqtt_as_timeout_concurrent.py index c4a4020..c418b40 100644 --- a/mqtt_as_timeout_concurrent.py +++ b/mqtt_as_timeout_concurrent.py @@ -24,7 +24,7 @@ async def subscribe(self, topic, qos=0, timeout=None, await_connection=True): async def unsubscribe(self, topic, timeout=None, await_connection=False): # with clean sessions a connection loss is basically a successful unsubscribe return await self._preprocessor(super().unsubscribe, topic, timeout=timeout, - await_connection=False) + await_connection=await_connection) # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms async def _connection(self): @@ -59,7 +59,9 @@ async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=T if not found: # id unique identifier = i break - # create task + # create task (if task was created outside loop, a return would cancel the + # not-started generator and prevent it from removing itself from + # self._ops_coros) task = self._operationTimeout(coroutine, *args, slot=identifier) asyncio.get_event_loop().create_task(task) coro = (identifier, task) diff --git a/mqtt_as_timeout_concurrent2.py b/mqtt_as_timeout_concurrent2.py new file mode 100644 index 0000000..232a011 --- /dev/null +++ b/mqtt_as_timeout_concurrent2.py @@ -0,0 +1,129 @@ +# Author: Kevin Köck +# Copyright Kevin Köck 2019 Released under the MIT license +# Created on 2019-11-05 + +__updated__ = "2019-11-06" +__version__ = "0.1" + +try: + from micropython_mqtt_as.mqtt_as import MQTTClient as _MQTTClient +except ImportError: + from .mqtt_as import MQTTClient as _MQTTClient +import uasyncio as asyncio +import time + + +### +# wait_for for official uasyncio from https://github.com/kevinkk525/micropython_uasyncio_extension +### +class TimeoutError(Exception): + pass + + +asyncio.TimeoutError = TimeoutError + + +async def wait_for(coro, timeout): + return await wait_for_ms(coro, timeout * 1000) + + +async def wait_for_ms(coro, timeout): + canned = False + t = time.ticks_ms() + + async def killer(): + nonlocal canned + while time.ticks_diff(time.ticks_ms(), t) < timeout: + await asyncio.sleep(0) # keeps killer in runq, leaves more slots in waitq + asyncio.cancel(coro) + canned = True # used to identify if killer cancelled coro or got cancelled + # because the CancelledError will also be raised in line 33. + + kill = killer() + asyncio.ensure_future(kill) + try: + res = await coro + except asyncio.CancelledError: + if canned: # coro got canceled, not wait_for + if asyncio.DEBUG and __debug__: + asyncio.log.debug("Coro %s cancelled in wait_for", coro) + raise TimeoutError + if asyncio.DEBUG and __debug__: + asyncio.log.debug("Wait_for got cancelled awaiting %s", coro) + raise + except Exception: + raise + else: + if canned: + if asyncio.DEBUG and __debug__: + asyncio.log.debug("Coro %s cancelled in wait_for but caught Exception", coro) + raise TimeoutError + finally: + asyncio.cancel(kill) + return res + + +asyncio.wait_for = wait_for +asyncio.wait_for_ms = wait_for_ms + + +### + +class MQTTClient(_MQTTClient): + + # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms + async def _connection(self): + while not self._isconnected: + await asyncio.sleep_ms(50) + + async def _killer(self, coro): + done = False + res = None # since mqtt_as doesn't return on success, None is success + + async def op(): + try: + res = await coro + finally: + nonlocal done + done = True + + task = op() + asyncio.ensure_future(task) + try: + while not done: + await asyncio.sleep(0) # keep on runq + except asyncio.CancelledError: + async with self.lock: + # print("canceled with lock") + asyncio.cancel(task) + return res + + async def publish(self, topic, msg, retain=False, qos=0, timeout=None): + if timeout: + try: + return await asyncio.wait_for( + self._killer(super().publish(topic, msg, retain, qos)), timeout) + except asyncio.TimeoutError: + return False + else: + return await super().publish(topic, msg, retain, qos) + + async def subscribe(self, topic, qos=0, timeout=None): + if timeout: + try: + return await asyncio.wait_for( + self._killer(super().subscribe(topic, qos)), timeout) + except asyncio.TimeoutError: + return False + else: + return await super().subscribe(topic, qos) + + async def unsubscribe(self, topic, timeout=None): + if timeout: + try: + return await asyncio.wait_for( + self._killer(super().unsubscribe(topic)), timeout) + except asyncio.TimeoutError: + return False + else: + return await super().unsubscribe(topic) diff --git a/tests/timeout_concurrent2.py b/tests/timeout_concurrent2.py new file mode 100644 index 0000000..0e31934 --- /dev/null +++ b/tests/timeout_concurrent2.py @@ -0,0 +1,115 @@ +# Author: Kevin Köck +# Copyright Kevin Köck 2019 Released under the MIT license +# Created on 2019-11-05 + +__updated__ = "2019-11-05" +__version__ = "0.1" + +try: + from mqtt_as_timeout_concurrent2 import MQTTClient +except ImportError: + from ..mqtt_as_timeout_concurrent2 import MQTTClient + +import uasyncio as asyncio + +loop = asyncio.get_event_loop(waitq_len=60, runq_len=60) + + +async def publish(val, t): + print("Publishing", val, "timeout", t) + res = await client.publish("foo_topic", val, qos=1, timeout=t) + print("publish result for", val, ":", res) + + +def callback(topic, msg, retained): + print((topic, msg, retained)) + + +first = True + + +async def conn_han(client): + global first + if first: + # await client.subscribe('foo_topic', 1) + loop = asyncio.get_event_loop() + loop.create_task(publish("payload {!s}".format(1), 1)) + loop.create_task(publish("payload {!s}".format(2), 2)) + loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) + loop.create_task(publish("payload {!s}".format(4), 4)) + loop.create_task(publish("payload {!s}".format(5), 5)) + loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) + first = False + await asyncio.sleep(1) + print("Closing connection") + await client.disconnect() + await asyncio.sleep(5) + print("Publishing disconnected") + loop.create_task(publish("payload {!s}".format(1), 1)) + loop.create_task(publish("payload {!s}".format(2), 2)) + loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) + loop.create_task(publish("payload {!s}".format(4), 4)) + loop.create_task(publish("payload {!s}".format(5), 5)) + loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) + await asyncio.sleep(10) + print("Reconnecting after all timeouts") + await client.connect() + loop.create_task(publish("payload {!s}".format(8), 8)) + await asyncio.sleep(5) + print("Test done") + await client.disconnect() + + +import config +from ubinascii import hexlify +from machine import unique_id + + +async def wifi(state): + print("WIFI state", state) + + +async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program + await asyncio.sleep_ms(20) + + +config_dict = { + 'client_id': hexlify(unique_id()), + 'server': config.MQTT_HOST, + 'port': config.MQTT_PORT, + 'user': config.MQTT_USER, + 'password': config.MQTT_PASSWORD, + 'keepalive': 60, + 'ping_interval': 0, + 'ssl': False, + 'ssl_params': {}, + 'response_time': 10, + 'clean_init': True, + 'clean': True, + 'max_repubs': 4, + 'will': None, + 'subs_cb': lambda *_: None, + 'wifi_coro': wifi, + 'connect_coro': eliza, + 'ssid': None, + 'wifi_pw': None, +} +config_dict['connect_coro'] = conn_han +config_dict['subs_cb'] = callback + +client = MQTTClient(**config_dict) +client.DEBUG = True + + +async def main(client): + await client.connect() + n = 0 + while True: + await asyncio.sleep(5) + + +def test(): + try: + loop.run_until_complete(main(client)) + finally: + client.close() # Prevent LmacRxBlk:1 errors From 21459720051ed33da1358dad9ddfec1a43fa2482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Wed, 6 Nov 2019 09:02:27 +0100 Subject: [PATCH 37/39] timeout option with wait_for() uasyncio extension from https://github.com/kevinkk525/micropython_uasyncio_extension --- mqtt_as_timeout_concurrent2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mqtt_as_timeout_concurrent2.py b/mqtt_as_timeout_concurrent2.py index 232a011..1a8fd32 100644 --- a/mqtt_as_timeout_concurrent2.py +++ b/mqtt_as_timeout_concurrent2.py @@ -81,6 +81,7 @@ async def _killer(self, coro): res = None # since mqtt_as doesn't return on success, None is success async def op(): + nonlocal res try: res = await coro finally: From c5b5829c2f3ddb402f20b30a9f5671766f2a5a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Thu, 26 Mar 2020 22:16:25 +0100 Subject: [PATCH 38/39] support new uasyncio. the other timeout files and tests still need to be updated. --- README.md | 2 ++ mqtt_as.py | 27 ++------------------------- mqtt_as_timeout.py | 32 +++++++++++++++----------------- 3 files changed, 19 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 5332e86..af57c11 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +# Update: This library can now only be used with the new uasyncio version that got merged into the main micropython repository! + # NOTE: With the latest commits in the original repository by Peter Hinch, this module is now USEABLE AS A DROP-IN REPLACEMENT! Only change needed is how the MQTTClient is created. Look [here](./README_mqtt_as.md#23-example-usage) for an example, it is simple, the pythonic way. diff --git a/mqtt_as.py b/mqtt_as.py index 89ce441..6d8192e 100644 --- a/mqtt_as.py +++ b/mqtt_as.py @@ -24,7 +24,7 @@ gc.collect() from sys import platform -VERSION = (0, 5, 0) +VERSION = (0, 6, 0) # Default short delay for good SynCom throughput (avoid sleep(0) with SynCom). _DEFAULT_MS = const(20) @@ -72,29 +72,6 @@ def qos_check(qos): raise ValueError('Only qos 0 and 1 are supported.') -class Lock: - def __init__(self): - self._locked = False - - async def __aenter__(self): - while True: - if self._locked: - await asyncio.sleep_ms(_DEFAULT_MS) - else: - self._locked = True - break - - async def __aexit__(self, *args): - self._locked = False - await asyncio.sleep_ms(_DEFAULT_MS) - - def locked(self): - return self._locked - - def release(self): - self._locked = False - - # MQTT_base class. Handles MQTT protocol on the basis of a good connection. # Exceptions from connectivity failures are handled by MQTTClient subclass. class MQTT_base: @@ -146,7 +123,7 @@ def __init__(self, client_id, server, port, user, password, keepalive, ping_inte self.newpid = pid_gen() self.rcv_pids = set() # PUBACK and SUBACK pids awaiting ACK response self.last_rx = ticks_ms() # Time of last communication from broker - self.lock = Lock() + self.lock = asyncio.Lock() def _set_last_will(self, topic, msg, retain=False, qos=0): qos_check(qos) diff --git a/mqtt_as_timeout.py b/mqtt_as_timeout.py index b2d97e0..8927ace 100644 --- a/mqtt_as_timeout.py +++ b/mqtt_as_timeout.py @@ -7,12 +7,8 @@ # connectivity and cancels it if the delay exceeds a timeout. # Note that it blocks other attempts at publication while waiting for a PUBACK, -# counter to the normal operation of the module. A solution capable of handling -# concurrent qos == 1 publications would require a set instance containing coros. - -# It incorporates a workround for the bug in uasyncio V2 whereby cancellation -# is deferred if a task is waiting on a sleep command. -# For these reasons it was not included in the mqtt_as module. +# counter to the normal operation of the module but uses less RAM than the +# implementation with concurrent operations. # The occurrence of a timeout does not guarantee non-reception of the message: # connectivity loss may occur between reception by the broker and reception of @@ -25,7 +21,7 @@ class MQTTClient(_MQTTClient): - _pub_coro = None + _pub_task = None # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms async def _connection(self): @@ -36,21 +32,23 @@ async def _publishTimeout(self, topic, msg, retain, qos): try: await super().publish(topic, msg, retain, qos) finally: - self._pub_coro = None + self._pub_task = None async def publish(self, topic, msg, retain=False, qos=0, timeout=None): - coro = None + task = None start = time.ticks_ms() while timeout is None or time.ticks_diff(time.ticks_ms(), start) < timeout: - if self._pub_coro is None and coro is None: - coro = self._publishTimeout(topic, msg, retain, qos) - asyncio.get_event_loop().create_task(coro) - self._pub_coro = coro - elif coro is not None: - if self._pub_coro != coro: + # Can't use wait_for because cancelling a wait_for would cancel _publishTimeout + # Also a timeout in wait_for would cancel _publishTimeout without waiting for + # the socket lock to be available, breaking mqtt protocol. + if self._pub_task is None and task is None: + task = asyncio.create_task(self._publishTimeout(topic, msg, retain, qos)) + self._pub_task = task + elif task is not None: + if self._pub_task != task: return # published await asyncio.sleep_ms(20) - if coro is not None: + if task is not None: async with self.lock: - asyncio.cancel(coro) + task.cancel() return From 6e25ac7568e55245a9d51f728203c1fb11e765b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kevin=20K=C3=B6ck?= Date: Tue, 7 Apr 2020 22:56:21 +0200 Subject: [PATCH 39/39] new implementation for concurrent timeout operations --- mqtt_as_timeout_concurrent.py | 118 ++++++++++++++---------------- mqtt_as_timeout_concurrent2.py | 130 --------------------------------- tests/timeout_concurrent2.py | 115 ----------------------------- 3 files changed, 53 insertions(+), 310 deletions(-) delete mode 100644 mqtt_as_timeout_concurrent2.py delete mode 100644 tests/timeout_concurrent2.py diff --git a/mqtt_as_timeout_concurrent.py b/mqtt_as_timeout_concurrent.py index c418b40..9eb9b7f 100644 --- a/mqtt_as_timeout_concurrent.py +++ b/mqtt_as_timeout_concurrent.py @@ -2,84 +2,72 @@ # Copyright Kevin Köck 2019 Released under the MIT license # Created on 2019-11-04 -__updated__ = "2019-11-04" -__version__ = "0.2" +__updated__ = "2020-04-01" +__version__ = "0.4" -from micropython_mqtt_as.mqtt_as import MQTTClient as _MQTTClient +try: + from micropython_mqtt_as.mqtt_as import MQTTClient as _MQTTClient +except ImportError: + from .mqtt_as import MQTTClient as _MQTTClient import uasyncio as asyncio import time class MQTTClient(_MQTTClient): - _ops_coros = set() + # operations return False is connection was lost and await_connection==False. + # operations return True if the operation was finished (mqtt_as doesn't return anything) + # operations raise asyncio.TimeoutError on timeout + # operations raise asyncio.CancelledError if the caller task got cancelled - async def publish(self, topic, msg, retain=False, qos=0, timeout=None, await_connection=True): - return await self._preprocessor(super().publish, topic, msg, retain, qos, - timeout=timeout, await_connection=await_connection) - - async def subscribe(self, topic, qos=0, timeout=None, await_connection=True): - await self._preprocessor(super().subscribe, topic, qos, timeout=timeout, - await_connection=await_connection) + async def _waiter(self, coro, timeout, await_connection): + # using _waiter even without a timeout as it ensures proper cancellation with self.lock + done = False - async def unsubscribe(self, topic, timeout=None, await_connection=False): - # with clean sessions a connection loss is basically a successful unsubscribe - return await self._preprocessor(super().unsubscribe, topic, timeout=timeout, - await_connection=await_connection) - - # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms - async def _connection(self): - while not self._isconnected: - await asyncio.sleep_ms(50) - - async def _operationTimeout(self, coro, *args, slot): - try: - await coro(*args) - finally: - for obj in self._ops_coros: - if obj[0] == slot: - self._ops_coros.discard(obj) - return + async def op(): + nonlocal done + try: + await coro + done = True + except Exception as e: + done = e - async def _preprocessor(self, coroutine, *args, timeout=None, await_connection=True): + task = asyncio.create_task(op()) start = time.ticks_ms() - coro = None try: - while timeout is None or time.ticks_diff(time.ticks_ms(), start) < timeout * 1000: + while not done: if not await_connection and not self._isconnected: + self.dprint("Connection lost") return False - elif coro is None: - # search for unused identifier - identifier = None - for i in range(1024): - found = False - for obj in self._ops_coros: - if obj[0] == i: - found = True - break - if not found: # id unique - identifier = i - break - # create task (if task was created outside loop, a return would cancel the - # not-started generator and prevent it from removing itself from - # self._ops_coros) - task = self._operationTimeout(coroutine, *args, slot=identifier) - asyncio.get_event_loop().create_task(task) - coro = (identifier, task) - self._ops_coros.add(coro) - elif coro not in self._ops_coros: - # coro removed, so operation was successful - self.dprint("Success on", args) - return True - await asyncio.sleep_ms(20) - self.dprint("timeout on", args) + elif timeout and time.ticks_diff(time.ticks_ms(), start) > timeout * 1000: + self.dprint("timeout in operation") + raise asyncio.TimeoutError + await asyncio.sleep_ms(40) + task = None # Task finished. finally doesn't need to cancel it. + if isinstance(done, Exception): + raise done + else: + return done except asyncio.CancelledError: - raise # the caller of this coro should be cancelled too + # operation got cancelled externally, finally: will cancel the task + raise finally: - if coro and coro in self._ops_coros: - # coro still active, cancel it + if task: async with self.lock: - asyncio.cancel(coro[1]) - # self._ops_coros.discard(coro) is done by finally in _operationTimeout - return False - # else: returns return value during process, which is True in case it was successful - return False + self.dprint("canceled with lock") + task.cancel() + + async def publish(self, topic, msg, retain=False, qos=0, timeout=None, await_connection=True): + if not await_connection and not self._isconnected: + return False + return await self._waiter(super().publish(topic, msg, retain, qos), timeout, + await_connection) + + async def subscribe(self, topic, qos=0, timeout=None, await_connection=True): + if not await_connection and not self._isconnected: + return False + return await self._waiter(super().subscribe(topic, qos), timeout, await_connection) + + async def unsubscribe(self, topic, timeout=None, await_connection=True): + if not await_connection and not self._isconnected: + return False + return await self._waiter(super().unsubscribe(topic), timeout, await_connection) diff --git a/mqtt_as_timeout_concurrent2.py b/mqtt_as_timeout_concurrent2.py deleted file mode 100644 index 1a8fd32..0000000 --- a/mqtt_as_timeout_concurrent2.py +++ /dev/null @@ -1,130 +0,0 @@ -# Author: Kevin Köck -# Copyright Kevin Köck 2019 Released under the MIT license -# Created on 2019-11-05 - -__updated__ = "2019-11-06" -__version__ = "0.1" - -try: - from micropython_mqtt_as.mqtt_as import MQTTClient as _MQTTClient -except ImportError: - from .mqtt_as import MQTTClient as _MQTTClient -import uasyncio as asyncio -import time - - -### -# wait_for for official uasyncio from https://github.com/kevinkk525/micropython_uasyncio_extension -### -class TimeoutError(Exception): - pass - - -asyncio.TimeoutError = TimeoutError - - -async def wait_for(coro, timeout): - return await wait_for_ms(coro, timeout * 1000) - - -async def wait_for_ms(coro, timeout): - canned = False - t = time.ticks_ms() - - async def killer(): - nonlocal canned - while time.ticks_diff(time.ticks_ms(), t) < timeout: - await asyncio.sleep(0) # keeps killer in runq, leaves more slots in waitq - asyncio.cancel(coro) - canned = True # used to identify if killer cancelled coro or got cancelled - # because the CancelledError will also be raised in line 33. - - kill = killer() - asyncio.ensure_future(kill) - try: - res = await coro - except asyncio.CancelledError: - if canned: # coro got canceled, not wait_for - if asyncio.DEBUG and __debug__: - asyncio.log.debug("Coro %s cancelled in wait_for", coro) - raise TimeoutError - if asyncio.DEBUG and __debug__: - asyncio.log.debug("Wait_for got cancelled awaiting %s", coro) - raise - except Exception: - raise - else: - if canned: - if asyncio.DEBUG and __debug__: - asyncio.log.debug("Coro %s cancelled in wait_for but caught Exception", coro) - raise TimeoutError - finally: - asyncio.cancel(kill) - return res - - -asyncio.wait_for = wait_for -asyncio.wait_for_ms = wait_for_ms - - -### - -class MQTTClient(_MQTTClient): - - # Await broker connection. Subclassed to reduce canceling time from 1s to 50ms - async def _connection(self): - while not self._isconnected: - await asyncio.sleep_ms(50) - - async def _killer(self, coro): - done = False - res = None # since mqtt_as doesn't return on success, None is success - - async def op(): - nonlocal res - try: - res = await coro - finally: - nonlocal done - done = True - - task = op() - asyncio.ensure_future(task) - try: - while not done: - await asyncio.sleep(0) # keep on runq - except asyncio.CancelledError: - async with self.lock: - # print("canceled with lock") - asyncio.cancel(task) - return res - - async def publish(self, topic, msg, retain=False, qos=0, timeout=None): - if timeout: - try: - return await asyncio.wait_for( - self._killer(super().publish(topic, msg, retain, qos)), timeout) - except asyncio.TimeoutError: - return False - else: - return await super().publish(topic, msg, retain, qos) - - async def subscribe(self, topic, qos=0, timeout=None): - if timeout: - try: - return await asyncio.wait_for( - self._killer(super().subscribe(topic, qos)), timeout) - except asyncio.TimeoutError: - return False - else: - return await super().subscribe(topic, qos) - - async def unsubscribe(self, topic, timeout=None): - if timeout: - try: - return await asyncio.wait_for( - self._killer(super().unsubscribe(topic)), timeout) - except asyncio.TimeoutError: - return False - else: - return await super().unsubscribe(topic) diff --git a/tests/timeout_concurrent2.py b/tests/timeout_concurrent2.py deleted file mode 100644 index 0e31934..0000000 --- a/tests/timeout_concurrent2.py +++ /dev/null @@ -1,115 +0,0 @@ -# Author: Kevin Köck -# Copyright Kevin Köck 2019 Released under the MIT license -# Created on 2019-11-05 - -__updated__ = "2019-11-05" -__version__ = "0.1" - -try: - from mqtt_as_timeout_concurrent2 import MQTTClient -except ImportError: - from ..mqtt_as_timeout_concurrent2 import MQTTClient - -import uasyncio as asyncio - -loop = asyncio.get_event_loop(waitq_len=60, runq_len=60) - - -async def publish(val, t): - print("Publishing", val, "timeout", t) - res = await client.publish("foo_topic", val, qos=1, timeout=t) - print("publish result for", val, ":", res) - - -def callback(topic, msg, retained): - print((topic, msg, retained)) - - -first = True - - -async def conn_han(client): - global first - if first: - # await client.subscribe('foo_topic', 1) - loop = asyncio.get_event_loop() - loop.create_task(publish("payload {!s}".format(1), 1)) - loop.create_task(publish("payload {!s}".format(2), 2)) - loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) - loop.create_task(publish("payload {!s}".format(4), 4)) - loop.create_task(publish("payload {!s}".format(5), 5)) - loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) - first = False - await asyncio.sleep(1) - print("Closing connection") - await client.disconnect() - await asyncio.sleep(5) - print("Publishing disconnected") - loop.create_task(publish("payload {!s}".format(1), 1)) - loop.create_task(publish("payload {!s}".format(2), 2)) - loop.create_task(client.subscribe("testtopic{!s}".format(3), qos=1, timeout=5)) - loop.create_task(publish("payload {!s}".format(4), 4)) - loop.create_task(publish("payload {!s}".format(5), 5)) - loop.create_task(client.subscribe("testtopic{!s}".format(6), qos=1, timeout=5)) - await asyncio.sleep(10) - print("Reconnecting after all timeouts") - await client.connect() - loop.create_task(publish("payload {!s}".format(8), 8)) - await asyncio.sleep(5) - print("Test done") - await client.disconnect() - - -import config -from ubinascii import hexlify -from machine import unique_id - - -async def wifi(state): - print("WIFI state", state) - - -async def eliza(*_): # e.g. via set_wifi_handler(coro): see test program - await asyncio.sleep_ms(20) - - -config_dict = { - 'client_id': hexlify(unique_id()), - 'server': config.MQTT_HOST, - 'port': config.MQTT_PORT, - 'user': config.MQTT_USER, - 'password': config.MQTT_PASSWORD, - 'keepalive': 60, - 'ping_interval': 0, - 'ssl': False, - 'ssl_params': {}, - 'response_time': 10, - 'clean_init': True, - 'clean': True, - 'max_repubs': 4, - 'will': None, - 'subs_cb': lambda *_: None, - 'wifi_coro': wifi, - 'connect_coro': eliza, - 'ssid': None, - 'wifi_pw': None, -} -config_dict['connect_coro'] = conn_han -config_dict['subs_cb'] = callback - -client = MQTTClient(**config_dict) -client.DEBUG = True - - -async def main(client): - await client.connect() - n = 0 - while True: - await asyncio.sleep(5) - - -def test(): - try: - loop.run_until_complete(main(client)) - finally: - client.close() # Prevent LmacRxBlk:1 errors