diff --git a/.hidden/VERSION b/.hidden/VERSION index d61aa22fa..534f10743 100644 --- a/.hidden/VERSION +++ b/.hidden/VERSION @@ -1 +1 @@ -{"branch": "stable7", "version": "7.1.020"} +{"branch": "wip-develop-7.2-2025", "version": "7.2.110"} \ No newline at end of file diff --git a/Classes/DomoticzDB.py b/Classes/DomoticzDB.py index 1a55cb751..5afa3a245 100644 --- a/Classes/DomoticzDB.py +++ b/Classes/DomoticzDB.py @@ -10,13 +10,7 @@ # # SPDX-License-Identifier: GPL-3.0 license -""" - Module: z_DomoticzDico.py - - Description: Retreive & Build Domoticz Dictionary - -""" - +""" This Classe allow retreiving and settings information in Domoticz via the JSON API """ import base64 import binascii @@ -24,29 +18,42 @@ import socket import ssl import time +import urllib.error import urllib.request +from urllib.parse import urlsplit -from Classes.LoggingManagement import LoggingManagement from Modules.restartPlugin import restartPluginViaDomoticzJsonApi from Modules.tools import is_domoticz_new_API -CACHE_TIMEOUT = (15 * 60) + 15 # num seconds +REQ_TIMEOUT = .750 # 750 ms Timeout + def init_domoticz_api(self): + """ + Initializes the Domoticz API by determining whether to use the new or old API format. + """ + is_new_api = is_domoticz_new_API(self) + self.logging( + "Debug", + f"Initializing Domoticz API using {'new' if is_new_api else 'old'} API format" + ) - if is_domoticz_new_API(self): - self.logging("Debug", 'Init domoticz api based on new api') - init_domoticz_api_settings( - self, + api_settings = { + "new": ( "type=command¶m=getsettings", "type=command¶m=gethardware", "type=command¶m=getdevices&rid=", - ) - else: - self.logging("Debug", 'Init domoticz api based on old api') - init_domoticz_api_settings( - self, "type=settings", "type=hardware", "type=devices&rid=" - ) + ), + "old": ( + "type=settings", + "type=hardware", + "type=devices&rid=", + ), + } + + selected_settings = api_settings["new"] if is_new_api else api_settings["old"] + init_domoticz_api_settings(self, *selected_settings) + def init_domoticz_api_settings(self, settings_api, hardware_api, devices_api): self.DOMOTICZ_SETTINGS_API = settings_api @@ -54,112 +61,140 @@ def init_domoticz_api_settings(self, settings_api, hardware_api, devices_api): self.DOMOTICZ_DEVICEST_API = devices_api -def isBase64( sb ): +def isBase64(sb): + """ + Checks if the given string is valid Base64. + :param sb: The string to check. + :return: True if the string is valid Base64, False otherwise. + """ try: return base64.b64encode(base64.b64decode(sb)).decode() == sb - except TypeError: - return False - except binascii.Error: + except (TypeError, binascii.Error): return False - -def extract_username_password( self, url_base_api ): - + + +def extract_username_password(self, url_base_api): + """ + Extracts the username, password, host, and protocol from a URL. + :param url_base_api: The URL containing credentials in the format ://:@ + :return: A tuple (username, password, host_port, proto) or (None, None, None, None) if invalid. + """ + HTTP_PROTO = "http://" # noqa: S5332 + HTTPS_PROTO = "https://" + items = url_base_api.split('@') if len(items) != 2: + self.logging("Debug", f"no credentials in the URL: {url_base_api}") return None, None, None, None - self.logging("Debug", f'Extract username/password {url_base_api} ==> {items} ') - host_port = items[1] + self.logging("Debug", f"Extracting username/password from {url_base_api} ==> {items}") + proto = None - if items[0].find("https") == 0: - proto = 'https' - items[0] = items[0][:5].lower() + items[0][5:] - item1 = items[0].replace('https://','') - usernamepassword = item1.split(':') - if len(usernamepassword) == 2: - username, password = usernamepassword - return username, password, host_port, proto - self.logging("Error", f'We are expecting a username and password but do not find it in {url_base_api} ==> {items} ==> {item1} ==> {usernamepassword}') - - elif items[0].find("http") == 0: - proto = 'http' - items[0] = items[0][:4].lower() + items[0][4:] - item1 = items[0].replace('http://','') - usernamepassword = item1.split(':') - if len(usernamepassword) == 2: - username, password = usernamepassword - return username, password, host_port, proto - self.logging("Error", f'We are expecting a username and password but do not find it in {url_base_api} ==> {items} ==> {item1} ==> {usernamepassword}') - - self.logging("Error", f'We are expecting a username and password but do not find it in {url_base_api} ==> {items} ') + credentials, host_port = items + if credentials.startswith(HTTPS_PROTO): + proto = "https" + credentials = credentials[8:] # Remove proto + elif credentials.startswith(HTTP_PROTO): + proto = "http" + credentials = credentials[7:] # Remove proto + else: + self.logging("Error", f"Unsupported protocol in URL: {url_base_api}") + return None, None, None, None + + username_password = credentials.split(':', 1) + if len(username_password) == 2: + username, password = username_password + return username, password, host_port, proto + + self.logging("Error", f"Missing username or password in {url_base_api} ==> {credentials} ==> {username_password}") return None, None, None, None -def open_and_read( self, url ): - self.logging("Log", f'opening url {url}') - - myssl_context = None +def open_and_read(self, url): + """ + Opens a URL and reads the response with optional SSL context, retries, and a timeout. + :param url: The URL to open. + :return: Response content or None if the request fails. + """ + self.logging("Debug", f"Opening URL: {url}") + + # Set up SSL context if necessary + ssl_context = None if "https" in url.lower() and not self.pluginconf.pluginConf["CheckSSLCertificateValidity"]: - myssl_context = ssl.create_default_context() - myssl_context.check_hostname=False - myssl_context.verify_mode=ssl.CERT_NONE - - retry = 3 - while retry: + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + for retries in range(3, 0, -1): try: - self.logging("Debug", f'opening url {url} with context {myssl_context}') - with urllib.request.urlopen(url, context=myssl_context) as response: + self.logging("Debug", f"Opening URL: {url} with SSL context: {ssl_context} and REQ_TIMEOUT timeout") + with urllib.request.urlopen(url, context=ssl_context, timeout=REQ_TIMEOUT) as response: return response.read() except urllib.error.HTTPError as e: - if e.code in [429,504]: # 429=too many requests, 504=gateway timeout - reason = f'{e.code} {str(e.reason)}' + if e.code in [429, 504]: + reason = f"{e.code} {e.reason}" elif isinstance(e.reason, socket.timeout): - reason = f'HTTPError socket.timeout {e.reason} - {e}' - else: - raise - except urllib.error.URLError as e: - if isinstance(e.reason, socket.timeout): - reason = f'URLError socket.timeout {e.reason} - {e}' + reason = f"HTTPError socket.timeout {e.reason} - {e}" else: raise - except socket.timeout as e: - reason = f'socket.timeout {e}' - netloc = urllib.parse.urlsplit(url).netloc # e.g. nominatim.openstreetmap.org - self.logging("Error", f'*** {netloc} {reason}; will retry') + + except (urllib.error.URLError, socket.timeout) as e: + reason = f"{type(e).__name__} {e.reason}" if hasattr(e, 'reason') else str(e) + + # Log retry information + netloc = urlsplit(url).netloc + self.logging("Error", f"*** {netloc} {reason}; retrying ({retries - 1} attempts left)") time.sleep(1) - retry -= 1 + -def domoticz_request( self, url): - self.logging("Debug",'domoticz request url: %s' %url) +def domoticz_request(self, url): + """ + Makes a request to the given Domoticz URL with optional SSL context, authentication, and a timeout. + :param url: The target URL. + :return: Response content or None if the request fails. + """ + self.logging("Debug", f"Domoticz request URL: {url}") + + # Create the request object try: request = urllib.request.Request(url) - except urllib.error.URLError as e: - self.logging("Error", "Request to %s rejected. Error: %s" %(url, e)) + except ValueError as e: + self.logging("Error", f"Invalid URL: {url}. Error: {e}") return None - self.logging("Debug",'domoticz request result: %s' %request) + # Add authorization header if available if self.authentication_str: - self.logging("Debug",'domoticz request Authorization: %s' %request) - request.add_header("Authorization", "Basic %s" % self.authentication_str) - self.logging("Debug",'domoticz request open url') + request.add_header("Authorization", f"Basic {self.authentication_str}") + self.logging("Debug", "Authorization header added to request.") - myssl_context = None - if "https" in url.lower() and not self.pluginconf.pluginConf["CheckSSLCertificateValidity"]: - myssl_context = ssl.create_default_context() - myssl_context.check_hostname=False - myssl_context.verify_mode=ssl.CERT_NONE - + # Set up SSL context if necessary + ssl_context = None + if url.lower().startswith("https") and not self.pluginconf.pluginConf["CheckSSLCertificateValidity"]: + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + # Open the URL with a 750ms timeout try: - self.logging("Debug", f'opening url {request} with context {myssl_context}') - response = urllib.request.urlopen(request, context=myssl_context) + self.logging("Debug", f"Opening URL: {url} with SSL context: {ssl_context} and REQ_TIMEOUT timeout") + with urllib.request.urlopen(request, context=ssl_context, timeout=REQ_TIMEOUT) as response: + return response.read() except urllib.error.URLError as e: - self.logging("Error", "Urlopen to %s rejected. Error: %s" %(url, e)) - return None + self.logging("Error", f"Request to {url} failed. Error: {e}") + except socket.timeout: + self.logging("Error", f"Request to {url} timed out after 750ms.") + except Exception as e: + self.logging("Error", f"Unexpected error occurred for URL {url}: {e}") - return response.read() - + return None + + def domoticz_base_url(self): + """ + Returns the base URL for the Domoticz API, either from the configuration (if we have processed it once) + or by constructing it using credentials and host information. + """ if self.url_ready: self.logging( "Debug", "domoticz_base_url - API URL ready %s Basic Authentication: %s" %(self.url_ready, self.authentication_str)) @@ -189,11 +224,17 @@ def domoticz_base_url(self): self.url_ready = url return url + class DomoticzDB_Preferences: - # sourcery skip: replace-interpolation-with-fstring - + """ interact in Read Only with the Domoticz Preferences Table (Domoticz Settings) """ + def __init__(self, api_base_url, pluginconf, log, DomoticzBuild, DomoticzMajor, DomoticzMinor): + """ + Initializes the DomoticzDB_Preferences class with the necessary parameters + and loads the preferences from the Domoticz API. + """ self.api_base_url = api_base_url + self.url_ready = None self.preferences = {} self.pluginconf = pluginconf self.log = log @@ -202,47 +243,56 @@ def __init__(self, api_base_url, pluginconf, log, DomoticzBuild, DomoticzMajor, self.DomoticzBuild = DomoticzBuild self.DomoticzMajor = DomoticzMajor self.DomoticzMinor = DomoticzMinor + + # Initialize API settings and load preferences init_domoticz_api(self) self.load_preferences() - def load_preferences(self): - # sourcery skip: replace-interpolation-with-fstring + """ + Loads preferences from the Domoticz API and stores them in the preferences attribute. + """ url = domoticz_base_url(self) - if url is None: + if not url: return - url += self.DOMOTICZ_HARDWARE_API - dz_response = domoticz_request( self, url) - if dz_response is None: - return + url += self.DOMOTICZ_HARDWARE_API + dz_response = domoticz_request(self, url) + if dz_response: + self.preferences = json.loads(dz_response) - self.preferences = json.loads( dz_response ) - def logging(self, logType, message): - # sourcery skip: replace-interpolation-with-fstring + """ + Logs messages to the specified log with the given log type. + """ self.log.logging("DZDB", logType, message) - def retreiveAcceptNewHardware(self): - # sourcery skip: replace-interpolation-with-fstring - self.logging("Debug", "retreiveAcceptNewHardware status %s" %self.preferences['AcceptNewHardware']) - return self.preferences['AcceptNewHardware'] - - def retreiveWebUserNamePassword(self): - # sourcery skip: replace-interpolation-with-fstring - webUserName = webPassword = '' - if 'WebPassword' in self.preferences: - webPassword = self.preferences['WebPassword'] - if 'WebUserName' in self.preferences: - webUserName = self.preferences['WebUserName'] - self.logging("Debug", "retreiveWebUserNamePassword %s %s" %(webUserName, webPassword)) - return webUserName, webPassword + def retrieve_accept_new_hardware(self): + """ + Retrieves the 'AcceptNewHardware' status from the preferences. + """ + accept_new_hardware = self.preferences.get('AcceptNewHardware', False) + self.logging("Debug", f"retrieve_accept_new_hardware status {accept_new_hardware}") + return accept_new_hardware + + def retrieve_web_user_name_password(self): + """ + Retrieves the web username and password from the preferences. + """ + web_user_name = self.preferences.get('WebUserName', '') + web_password = self.preferences.get('WebPassword', '') + self.logging("Debug", f"retrieve_web_user_name_password {web_user_name} {web_password}") + return web_user_name, web_password + class DomoticzDB_Hardware: + """ interact in Read Write with the Domoticz Hardware Table (Domoticz Plugins) """ + def __init__(self, api_base_url, pluginconf, hardwareID, log, pluginParameters, DomoticzBuild, DomoticzMajor, DomoticzMinor): + """ Initializes the DomoticzDB_Hardware class with the necessary parameters and loads hardware information. """ self.api_base_url = api_base_url - self.authentication_str = None self.url_ready = None + self.authentication_str = None self.hardware = {} self.HardwareID = hardwareID self.pluginconf = pluginconf @@ -256,109 +306,122 @@ def __init__(self, api_base_url, pluginconf, hardwareID, log, pluginParameters, self.load_hardware() def load_hardware(self): - # sourcery skip: replace-interpolation-with-fstring + """ Loads hardware data from the Domoticz API and stores it in the hardware attribute. """ url = domoticz_base_url(self) - if url is None: + if not url: return - url += self.DOMOTICZ_HARDWARE_API - dz_result = domoticz_request( self, url) - if dz_result is None: - return - result = json.loads( dz_result ) - - for x in result['result']: - idx = x[ "idx" ] - self.hardware[ idx ] = x + url += self.DOMOTICZ_HARDWARE_API + dz_result = domoticz_request(self, url) + if dz_result: + result = json.loads(dz_result) + for x in result.get('result', []): + self.hardware[x["idx"]] = x def logging(self, logType, message): + """ Logs messages to the specified log with the given log type. """ self.log.logging("DZDB", logType, message) - def disableErasePDM(self, webUserName, webPassword): - # sourcery skip: replace-interpolation-with-fstring - # To disable the ErasePDM, we have to restart the plugin - # This is usally done after ErasePDM + def disable_erase_pdm(self, webUserName, webPassword): + """ Disables the ErasePDM feature by restarting the plugin. """ restartPluginViaDomoticzJsonApi(self, stop=False, url_base_api=self.api_base_url) def get_loglevel_value(self): - # sourcery skip: replace-interpolation-with-fstring - if ( - self.hardware - and ("%s" %self.HardwareID) in self.hardware - and 'LogLevel' in self.hardware[ '%s' %self.HardwareID ] - ): - self.logging("Debug", "get_loglevel_value %s " %(self.hardware[ '%s' %self.HardwareID ]['LogLevel'])) - return self.hardware[ '%s' %self.HardwareID ]['LogLevel'] - return 7 + """ Retrieves the log level for the hardware, defaults to 7 if not found. """ + hardware_info = self.hardware.get(str(self.HardwareID), {}) + log_level = hardware_info.get('LogLevel', 7) + self.logging("Debug", f"get_loglevel_value {log_level}") + return log_level def multiinstances_z4d_plugin_instance(self): - # sourcery skip: replace-interpolation-with-fstring + """ Checks if there are multiple instances of the Z4D plugin running. """ self.logging("Debug", "multiinstances_z4d_plugin_instance") - if sum("Zigate" in self.hardware[ x ]["Extra"] for x in self.hardware) > 1: - return True - return False + return sum("Zigate" in x.get("Extra", "") for x in self.hardware.values()) > 1 + class DomoticzDB_DeviceStatus: + """ + Interact in Read Only with the Domoticz DeviceStatus Table (Domoticz Devices). + This mainly is used to retrieve the Adjusted Value parameters for Temp/Baro and the Motion delay. + """ + def __init__(self, api_base_url, pluginconf, hardwareID, log, DomoticzBuild, DomoticzMajor, DomoticzMinor): + """ + Initializes the DomoticzDB_DeviceStatus class with the necessary parameters. + """ self.api_base_url = api_base_url + self.url_ready = None self.HardwareID = hardwareID self.pluginconf = pluginconf self.log = log self.authentication_str = None - self.url_ready = None self.DomoticzBuild = DomoticzBuild self.DomoticzMajor = DomoticzMajor self.DomoticzMinor = DomoticzMinor + self.cache = {} # Caching device status data init_domoticz_api(self) def logging(self, logType, message): - # sourcery skip: replace-interpolation-with-fstring + """Logs messages with the specified log type.""" self.log.logging("DZDB", logType, message) - def get_device_status(self, ID): - # "http://%s:%s@127.0.0.1:%s" - # sourcery skip: replace-interpolation-with-fstring + def _get_cached_device_status(self, device_id): + """Returns the cached device status if it's still valid.""" + cached_entry = self.cache.get(device_id) + if cached_entry: + timestamp, data = cached_entry + self.logging("Debug", f"Using cached data for device ID {device_id}: {data}") + return data + return None + + def _cache_device_status(self, device_id, data): + """Caches the device status with the current timestamp.""" + self.cache[device_id] = (time.time(), data) + self.logging("Debug", f"Cached data for device ID {device_id}") + + def get_device_status(self, device_id): + """ + Retrieves the device status for a given device ID, with caching. + """ + # Check if data is already cached + cached_data = self._get_cached_device_status(device_id) + if cached_data is not None: + return cached_data + + # If not cached, make a request url = domoticz_base_url(self) - if url is None: - return - url += self.DOMOTICZ_DEVICEST_API + "%s" %ID + if not url: + return None - dz_result = domoticz_request( self, url) + url += f"{self.DOMOTICZ_DEVICEST_API}{device_id}" + dz_result = domoticz_request(self, url) if dz_result is None: return None - result = json.loads( dz_result ) - self.logging("Debug", "Result: %s" %result) + + result = json.loads(dz_result) + self.logging("Debug", f"Result: {result}") + + # Cache the result before returning + self._cache_device_status(device_id, result) return result - - def extract_AddValue(self, ID, attribute): - # sourcery skip: replace-interpolation-with-fstring - result = self.get_device_status( ID) - if result is None: - return 0 - if 'result' not in result: + def _extract_add_value(self, device_id, attribute): + """Extracts the value of a specified attribute from the device status.""" + result = self.get_device_status(device_id) + if result is None or 'result' not in result: return 0 - AdjValue = 0 - for x in result['result']: - AdjValue = x[attribute] - self.logging("Debug", "return extract_AddValue %s %s %s" % (ID, attribute, AdjValue) ) - return AdjValue - - def retreiveAddjValue_baro(self, ID): - # sourcery skip: replace-interpolation-with-fstring - return self.extract_AddValue( ID, 'AddjValue2') - - def retreiveTimeOut_Motion(self, ID): - # sourcery skip: replace-interpolation-with-fstring - """ - Retreive the TmeeOut Motion value of Device.ID - """ - return self.extract_AddValue( ID, 'AddjValue') - def retreiveAddjValue_temp(self, ID): - # sourcery skip: replace-interpolation-with-fstring - """ - Retreive the AddjValue of Device.ID - """ - return self.extract_AddValue( ID, 'AddjValue') + return next((device[attribute] for device in result['result'] if attribute in device), 0) + + def retrieve_addj_value_baro(self, device_id): + """Retrieves the AddjValue2 attribute for the given device.""" + return self._extract_add_value(device_id, 'AddjValue2') + + def retrieve_timeout_motion(self, device_id): + """Retrieves the AddjValue (motion timeout) for the given device.""" + return self._extract_add_value(device_id, 'AddjValue') + + def retrieve_addj_value_temp(self, device_id): + """Retrieves the AddjValue (temperature) for the given device.""" + return self._extract_add_value(device_id, 'AddjValue') diff --git a/Classes/GroupMgtv2/GrpDatabase.py b/Classes/GroupMgtv2/GrpDatabase.py index ddf20a3db..8fb160d19 100644 --- a/Classes/GroupMgtv2/GrpDatabase.py +++ b/Classes/GroupMgtv2/GrpDatabase.py @@ -39,12 +39,13 @@ def write_groups_list(self): """ self.logging("Debug", "Dumping: %s" % self.GroupListFileName) + self.logging("Status", f"+ Saving Group List into {self.GroupListFileName}") with open(self.GroupListFileName, "wt") as handle: json.dump(self.ListOfGroups, handle, sort_keys=True, indent=2) - if is_domoticz_db_available(self) and self.pluginconf.pluginConf["useDomoticzDatabase"]: - self.log.logging("Database", "Debug", "Save Plugin Group Db to Domoticz") - setConfigItem(Key="ListOfGroups", Value={"TimeStamp": time.time(), "b64Groups": self.ListOfGroups}) + if self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]: + self.logging("Status", "+ Saving Group List into Domoticz") + setConfigItem(Key="ListOfGroups", Attribute="b64encoded", Value={"TimeStamp": time.time(), "b64encoded": self.ListOfGroups}) def load_groups_list_from_json(self): @@ -54,39 +55,37 @@ def load_groups_list_from_json(self): if self.GroupListFileName is None: return - if is_domoticz_db_available(self) and self.pluginconf.pluginConf["useDomoticzDatabase"]: - _domoticz_grouplist = getConfigItem(Key="ListOfGroups") - - dz_timestamp = 0 - if "TimeStamp" in _domoticz_grouplist: - dz_timestamp = _domoticz_grouplist["TimeStamp"] - _domoticz_grouplist = _domoticz_grouplist["b64Groups"] - self.logging( - "Debug", - "Groups data loaded where saved on %s" - % (time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(dz_timestamp))), - ) - - txt_timestamp = 0 - if os.path.isfile(self.GroupListFileName): + # Load from Domoticz + if self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]: + _domoticz_grouplist = getConfigItem(Key="ListOfGroups", Attribute="b64encoded") + dz_timestamp = _domoticz_grouplist.get("TimeStamp",0) + _domoticz_grouplist = _domoticz_grouplist.get("b64encoded", {}) + self.logging( "Debug", "Groups data loaded where saved on %s"% (time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(dz_timestamp))),) + + # Load from Json + txt_timestamp = 0 + _json_grouplist = {} + if os.path.isfile(self.GroupListFileName): + with open(self.GroupListFileName, "rt") as handle: + _json_grouplist = json.load(handle) txt_timestamp = os.path.getmtime(self.GroupListFileName) - domoticz_log_api("%s timestamp is %s" % (self.GroupListFileName, txt_timestamp)) - if dz_timestamp < txt_timestamp: - domoticz_log_api("Dz Group is older than Json Dz: %s Json: %s" % (dz_timestamp, txt_timestamp)) - # We should load the json file - - if not isinstance(_domoticz_grouplist, dict): - _domoticz_grouplist = {} - - if not os.path.isfile(self.GroupListFileName): - self.logging("Debug", "GroupMgt - Nothing to import from %s" % self.GroupListFileName) - return - - with open(self.GroupListFileName, "rt") as handle: - self.ListOfGroups = json.load(handle) - - if is_domoticz_db_available(self) and self.pluginconf.pluginConf["useDomoticzDatabase"]: - domoticz_log_api("GroupList Loaded from Dz: %s from Json: %s" % (len(_domoticz_grouplist), len(self.ListOfGroups))) + self.logging( "Debug", "%s timestamp is %s" % (self.GroupListFileName, txt_timestamp)) + + # Check Loads + if _domoticz_grouplist and _json_grouplist : + self.logging( "Debug", "==> Sanity check : GroupList Loaded. %s entries from Domoticz, %s from Json, result: %s" % ( + len(_domoticz_grouplist), len(_json_grouplist), _domoticz_grouplist == _json_grouplist )) + + if self.pluginconf.pluginConf["useDomoticzDatabase"] and dz_timestamp > txt_timestamp: + # We should load the Domoticz file + self.ListOfGroups = _domoticz_grouplist + loaded_from = "Domoticz" + else: + # We should use Json + loaded_from = self.GroupListFileName + self.ListOfGroups = _json_grouplist + + self.logging("Status", "Z4D loads %s config entries from %s" % (len(self.ListOfGroups), loaded_from)) def build_group_list_from_list_of_devices(self): diff --git a/Classes/LoggingManagement.py b/Classes/LoggingManagement.py index 2084d3b12..c883b021e 100644 --- a/Classes/LoggingManagement.py +++ b/Classes/LoggingManagement.py @@ -290,7 +290,7 @@ def _is_to_be_logged(self, logType, module): if self.pluginconf.pluginConf[module]: return True else: - domoticz_error_api("%s debug module unknown %s" % (module, module)) + domoticz_error_api("debug module unknown: %s" % (module)) return True return False diff --git a/Classes/OTA.py b/Classes/OTA.py index ce5b90dfd..5a3318da6 100644 --- a/Classes/OTA.py +++ b/Classes/OTA.py @@ -340,6 +340,10 @@ def ota_upgrade_end_request(self, MsgData): logging(self, "Status", "OTA upgrade completed with success - %s/%s %s Version: 0x%08x Type: 0x%04x Code: 0x%04x Status: %s" % ( MsgSrcAddr, MsgEP, MsgClusterId, intMsgImageVersion, image_type, intMsgManufCode, MsgStatus)) ota_upgrade_end_response(self, MsgSQN, MsgSrcAddr, MsgEP, intMsgImageVersion, image_type, intMsgManufCode) + + # Remove "OTAUpdate" if existing and successful + self.ListOfDevices[MsgSrcAddr].pop("OTAUpdate", None) + notify_upgrade_end(self, "OK", MsgSrcAddr, MsgEP, image_type, intMsgManufCode, intMsgImageVersion) elif MsgStatus == "95": @@ -1211,32 +1215,21 @@ def ota_aync_request( self, MsgSrcAddr, MsgEP, MsgIEEE, MsgFileOffset, image_ver return True -def notify_upgrade_end( - self, - Status, - MsgSrcAddr, - MsgEP, - image_type, - intMsgManufCode, - intMsgImageVersion, - ): # OK 26/10 +def _get_device_name_by_id(self, device_id): + return next((device.Name for device in self.Devices.values() if device.DeviceID == device_id), None) + + +def notify_upgrade_end( self, Status, MsgSrcAddr, MsgEP, image_type, intMsgManufCode, intMsgImageVersion, ): + # OK 26/10 _transferTime_hh, _transferTime_mm, _transferTime_ss = convert_time(int(time.time() - self.ListInUpdate["StartTime"])) _ieee = self.ListOfDevices[MsgSrcAddr]["IEEE"] - _name = None + _name = _get_device_name_by_id(self, _ieee) _textmsg = "" - for x in self.Devices: - if self.Devices[x].DeviceID == _ieee: - _name = self.Devices[x].Name - + if Status == "OK": _textmsg = "Device: %s has been updated with firmware %s in %s hour %s min %s sec" % ( - _name, - intMsgImageVersion, - _transferTime_hh, - _transferTime_mm, - _transferTime_ss, - ) + _name, intMsgImageVersion, _transferTime_hh, _transferTime_mm, _transferTime_ss, ) logging(self, "Status", _textmsg) if "Firmware Update" in self.PluginHealth and len(self.PluginHealth["Firmware Update"]) > 0: self.PluginHealth["Firmware Update"]["Progress"] = "Success" @@ -1252,6 +1245,7 @@ def notify_upgrade_end( if "Firmware Update" in self.PluginHealth and len(self.PluginHealth["Firmware Update"]) > 0: self.PluginHealth["Firmware Update"]["Progress"] = "Aborted" + elif Status == "Failed": _textmsg = "Firmware update aborted error code %s for Device %s in %s hour %s min %s sec" % ( Status, @@ -1262,6 +1256,7 @@ def notify_upgrade_end( ) if "Firmware Update" in self.PluginHealth and len(self.PluginHealth["Firmware Update"]) > 0: self.PluginHealth["Firmware Update"]["Progress"] = "Failed" + elif Status == "More": _textmsg = "Device: %s has been updated to latest firmware in %s hour %s min %s sec, but additional Image needed" % ( _name, @@ -1385,12 +1380,12 @@ def start_upgrade_infos(self, MsgSrcAddr, intMsgImageType, intMsgManufCode, MsgF # Retrieve device name from the IEEE address _ieee = self.ListOfDevices[MsgSrcAddr]["IEEE"] - _name = next((self.Devices[x].Name for x in self.Devices if self.Devices[x].DeviceID == _ieee), None) - + _name = _get_device_name_by_id(self, _ieee) + # Estimate upload time estimated_time_for_upload = ( self.ListInUpdate["intSize"] // MsgMaxDataSize ) if self.zigbee_communication == "zigpy": - estimated_time_for_upload //= 7 + estimated_time_for_upload //= 4.5 # Convert estimated time into hours, minutes, and seconds _durhh, _durmm, _durss = convert_time(estimated_time_for_upload) @@ -1458,6 +1453,17 @@ def notify_ota_firmware_available(self, srcnwkid, manufcode, imagetype, filevers logging(self, "Status", " firmware type: %s" % _ota_available["imageType"]) logging(self, "Status", " URL to download: %s" % _ota_available["url"]) + if srcnwkid in self.ListOfDevices: + if "OTAUpdate" not in self.ListOfDevices[srcnwkid]: + self.ListOfDevices[srcnwkid]["OTAUpdate"] = {} + if imagetype in self.ListOfDevices[srcnwkid]["OTAUpdate"]: + self.ListOfDevices[srcnwkid]["OTAUpdate"][imagetype].clear() + self.ListOfDevices[srcnwkid]["OTAUpdate"][imagetype] = { + "currentversion": str(fileversion), + "newestversion" : str(_ota_available["fileVersion"]), + "url": _ota_available["url"], + } + if folder: logging(self, "Status", " Folder to store: %s" % folder) else: diff --git a/Classes/PluginConf.py b/Classes/PluginConf.py index 1f200138d..674e46f9a 100644 --- a/Classes/PluginConf.py +++ b/Classes/PluginConf.py @@ -37,8 +37,8 @@ "internetAccess": { "type": "bool", "default": 1, "current": None, "restart": 1, "hidden": False, "Advanced": False, }, "CheckSSLCertificateValidity": { "type": "bool", "default": 0, "current": None, "restart": 1, "hidden": False, "Advanced": False, }, "allowOTA": { "type": "bool", "default": 1, "current": None, "restart": 1, "hidden": True, "Advanced": False, }, - "pingDevices": { "type": "bool", "default": 1, "current": None, "restart": 1, "hidden": False, "Advanced": True, }, - "PluginAnalytics": { "type": "bool", "default": -1, "current": None, "restart": 0, "hidden": True, "Advanced": False, }, + "CheckDeviceHealth": { "type": "bool", "default": 1, "current": None, "restart": 1, "hidden": False, "Advanced": True, }, + "PluginAnalytics": { "type": "bool", "default": -1, "current": None, "restart": 0, "hidden": False, "Advanced": False, }, "DomoticzCustomMenu": { "type": "bool", "default": 1, "current": None, "restart": 1, "hidden": False, "Advanced": False, }, "NightShift": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": False, } }, @@ -128,6 +128,7 @@ "Order": 8, "param": { "deviceOffWhenTimeOut": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, + "pingDevicesFeq": { "type": "int", "default": 3600, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, "forcePollingAfterAction": { "type": "bool", "default": 1, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, "forcePassiveWidget": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, "allowForceCreationDomoDevice": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": True, "Advanced": True, }, @@ -147,7 +148,6 @@ "Order": 9, "param": { "blueLedOnOff": { "type": "bool", "default": 1, "current": None, "restart": 0, "hidden": False, "Advanced": False, }, - "pingDevicesFeq": { "type": "int", "default": 3600, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, "resetPermit2Join": { "type": "bool", "default": 1, "current": None, "restart": 0, "hidden": False, "Advanced": True, }, "Ping": {"type": "bool", "default": 1, "current": None, "restart": 0, "hidden": False, "Advanced": True}, "allowRemoveZigateDevice": { "type": "bool", "default": 1, "current": None, "restart": 0, "hidden": True, "Advanced": True, "ZigpyRadio": "" }, @@ -204,7 +204,7 @@ "Order": 12, "param": { "PosixPathUpdate": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, - "storeDomoticzDatabase": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": False,"Advanced": True,}, + "storeDomoticzDatabase": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": False,"Advanced": True,}, "useDomoticzDatabase": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": False,"Advanced": True,}, "PluginLogMode": {"type": "list","list": { "system default": 0, "0600": 0o600, "0640": 0o640, "0644": 0o644},"default": 0,"current": None,"restart": 1,"hidden": False,"Advanced": True,}, "numDeviceListVersion": {"type": "int","default": 12,"current": None,"restart": 0,"hidden": False,"Advanced": False,}, @@ -231,6 +231,7 @@ "BasicOutput": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Binding": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "CasaIA": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, + "CheckUpdate": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Cluster": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Command": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "ConfigureReporting": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, @@ -266,6 +267,7 @@ "PDM": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Pairing": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Philips": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, + "PingDevices": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "PiZigate": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "Plugin": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, "PluginTools": { "type": "bool", "default": 0, "current": None, "restart": 0, "hidden": False, "Advanced": True }, @@ -391,7 +393,6 @@ "nPDUaPDUThreshold": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,"ZigpyRadio": ""}, "forceAckOnZCL": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "DropBadAnnoucement": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, - "expJsonDatabase": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "TryFindingIeeeOfUnknownNwkid": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "enableZigpyPersistentInFile": {"type": "bool","default": 0,"current": None,"restart": 1,"hidden": False,"Advanced": True,}, "enableZigpyPersistentInMemory": {"type": "bool","default": 1,"current": None,"restart": 1,"hidden": False,"Advanced": True,}, @@ -402,12 +403,12 @@ "Order": 99, "param": { # Just for compatibility keep it but hidden ( move to Custom device "Param" section) + "CheckRequirements": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": True,"Advanced": True}, "nPDUaPDUThreshold": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,"ZigpyRadio": ""}, "rebindLivolo": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": False,}, "allowAutoPairing": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "disabledDefaultResponseFirmware": {"type": "bool","default": 0,"current": None,"restart": 1,"hidden": True,"Advanced": True,}, "logUnknownDeviceModel": {"type": "bool","default": 1,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, - "expJsonDatabase": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "forceAckOnZCL": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "ControllerInHybridMode": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, "ControllerInRawMode": {"type": "bool","default": 0,"current": None,"restart": 0,"hidden": True,"Advanced": True,}, @@ -531,12 +532,13 @@ def write_Settings(self): else: write_pluginConf[param] = self.pluginConf[param] + Domoticz.Status( f"+ Saving Plugin Configuration into {pluginConfFile}") with open(pluginConfFile, "wt") as handle: json.dump(write_pluginConf, handle, sort_keys=True, indent=2) - if is_domoticz_db_available(self) and (self.pluginConf["useDomoticzDatabase"] or self.pluginConf["storeDomoticzDatabase"]): - setConfigItem(Key="PluginConf", Value={"TimeStamp": time.time(), "b64Settings": write_pluginConf}) + Domoticz.Status("+ Saving Plugin Configuration into Domoticz") + setConfigItem(Key="PluginConf", Attribute="b64encoded", Value={"TimeStamp": time.time(), "b64encoded": write_pluginConf}) def _load_Settings(self): @@ -545,49 +547,57 @@ def _load_Settings(self): dz_timestamp = 0 if is_domoticz_db_available(self): - _domoticz_pluginConf = getConfigItem(Key="PluginConf") - if "TimeStamp" in _domoticz_pluginConf: - dz_timestamp = _domoticz_pluginConf.get("TimeStamp",0) - _domoticz_pluginConf = _domoticz_pluginConf.get("b64Settings",{}) - Domoticz.Log( - "Plugin data loaded where saved on %s" - % (time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(dz_timestamp))) - ) + configuration_entry_imported_from_Domoticz = getConfigItem(Key="PluginConf", Attribute="b64encoded") + dz_timestamp = configuration_entry_imported_from_Domoticz.get("TimeStamp", 0) + _domoticz_pluginConf = configuration_entry_imported_from_Domoticz.get( "b64encoded", {}) + + if dz_timestamp != 0: + Domoticz.Log( "Plugin data loaded where saved on %s" % (time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(dz_timestamp))) ) + if not isinstance(_domoticz_pluginConf, dict): _domoticz_pluginConf = {} txt_timestamp = 0 if os.path.isfile(self.pluginConf["filename"]): txt_timestamp = os.path.getmtime(self.pluginConf["filename"]) - Domoticz.Log("%s timestamp is %s" % (self.pluginConf["filename"], txt_timestamp)) + Domoticz.Log("%s timestamp is %s" % (self.pluginConf["filename"], txt_timestamp)) - if dz_timestamp < txt_timestamp: - Domoticz.Log("Dz PluginConf is older than Json Dz: %s Json: %s" % (dz_timestamp, txt_timestamp)) - # We should load the json file + with open(self.pluginConf["filename"], "rt") as handle: + _pluginConf = {} + try: + _pluginConf = json.load(handle) - with open(self.pluginConf["filename"], "rt") as handle: - _pluginConf = {} - try: - _pluginConf = json.load(handle) - - except json.decoder.JSONDecodeError as e: - Domoticz.Error("poorly-formed %s, not JSON: %s" % (self.pluginConf["filename"], e)) - return + except json.decoder.JSONDecodeError as e: + Domoticz.Error("poorly-formed %s, not JSON: %s" % (self.pluginConf["filename"], e)) + return - for param in _pluginConf: - self.pluginConf[param] = _pluginConf[param] + loaded_from = "Domoticz" # Check Load - if is_domoticz_db_available(self) and self.pluginConf["useDomoticzDatabase"]: - Domoticz.Log("PluginConf Loaded from Dz: %s from Json: %s" % (len(_domoticz_pluginConf), len(_pluginConf))) - if _domoticz_pluginConf: - for x in _pluginConf: - if x not in _domoticz_pluginConf: - Domoticz.Error("-- %s is missing in Dz" % x) - elif _pluginConf[x] != _domoticz_pluginConf[x]: - Domoticz.Error( - "++ %s is different in Dz: %s from Json: %s" % (x, _domoticz_pluginConf[x], _pluginConf[x]) - ) + if is_domoticz_db_available(self) and _pluginConf: + Domoticz.Log("==> Sanity check : PluginConf Loaded. %s entries from Domoticz, %s from Json" % ( + len(_domoticz_pluginConf), len(_pluginConf))) + + if is_domoticz_db_available(self) and self.pluginConf["useDomoticzDatabase"] and _domoticz_pluginConf: + for x in _pluginConf: + if x not in _domoticz_pluginConf: + Domoticz.Error("-- %s is missing in Dz" % x) + + elif _pluginConf[x] != _domoticz_pluginConf[x]: + Domoticz.Error( "++ %s is different in Dz: %s from Json: %s" % (x, _domoticz_pluginConf[x], _pluginConf[x]) ) + + if self.pluginConf["useDomoticzDatabase"] and dz_timestamp > txt_timestamp: + # We should load the Domoticz file + load_plugin_conf = _domoticz_pluginConf + loaded_from = "Domoticz" + else: + # We should use Json + load_plugin_conf = _pluginConf + + for param in load_plugin_conf: + self.pluginConf[param] = load_plugin_conf[param] + + Domoticz.Status("Z4D loads %s config entries from %s" % (len(_domoticz_pluginConf), loaded_from)) # Overwrite Zigpy parameters if we are running native Zigate if self.zigbee_communication != "zigpy": diff --git a/Classes/TransportStats.py b/Classes/TransportStats.py index 5b29c7f54..acbbf2f76 100644 --- a/Classes/TransportStats.py +++ b/Classes/TransportStats.py @@ -12,12 +12,10 @@ import json -from time import time - -from Modules.domoticzAbstractLayer import (domoticz_error_api, - domoticz_log_api, - domoticz_status_api) +import time +from pathlib import Path +from Modules.domoticzAbstractLayer import domoticz_log_api, domoticz_status_api class TransportStatistics: def __init__(self, pluginconf, log, zigbee_communication): @@ -49,7 +47,7 @@ def __init__(self, pluginconf, log, zigbee_communication): self._maxRxProcesses = self._cumulRxProcess = self._cntRxProcess = self._averageRxProcess = 0 self._max_reading_thread_timing = self._cumul_reading_thread_timing = self._cnt_reading_thread_timing = self._average_reading_thread_timing = 0 self._max_reading_zigpy_timing = self._cumul_reading_zigpy_timing = self._cnt_reading_zigpy_timing = self._average_reading_zigpy_timing = 0 - self._start = int(time()) + self._start = int(time.time()) self.TrendStats = [] self.pluginconf = pluginconf self.log = log @@ -59,12 +57,15 @@ def __init__(self, pluginconf, log, zigbee_communication): def starttime(self): return self._start + def pdm_loaded(self): self._pdmLoads += 1 + def get_pdm_loaded(self): return self._pdmLoads + def add_timing_zigpy(self, timing): self._cumul_reading_zigpy_timing += timing self._cnt_reading_zigpy_timing += 1 @@ -85,6 +86,7 @@ def add_timing_thread(self, timing): % (self._max_reading_thread_timing, self._average_reading_thread_timing) ) + def add_timing8000(self, timing): self._cumulTiming8000 += timing @@ -97,6 +99,7 @@ def add_timing8000(self, timing): % (self._maxTiming8000, self._averageTiming8000) ) + def add_timing8011(self, timing): self._cumulTiming8011 += timing @@ -109,6 +112,7 @@ def add_timing8011(self, timing): % (self._maxTiming8011, self._averageTiming8011) ) + def add_timing8012(self, timing): self._cumulTiming8012 += timing @@ -121,6 +125,7 @@ def add_timing8012(self, timing): % (self._maxTiming8012, self._averageTiming8012) ) + def add_rxTiming(self, timing): self._cumulRxProcess += timing @@ -133,151 +138,224 @@ def add_rxTiming(self, timing): % (self._maxRxProcesses, self._averageRxProcess) ) + def addPointforTrendStats(self, TimeStamp): + """ + Adds a point to the trend statistics table, tracking Rx, Tx, and Load metrics. + + Args: + TimeStamp (int): The timestamp for the data point. + Note: + The table is capped at MAX_TREND_STAT_TABLE entries, with the oldest entry removed when the limit is reached. + """ MAX_TREND_STAT_TABLE = 120 - uptime = int(time() - self._start) - Rxps = round(self._received / uptime, 2) - Txps = round(self._sent / uptime, 2) - if len(self.TrendStats) >= MAX_TREND_STAT_TABLE: - del self.TrendStats[0] - self.TrendStats.append({"_TS": TimeStamp, "Rxps": Rxps, "Txps": Txps, "Load": self._Load}) + try: + # Calculate uptime and transmission rates + uptime = int(time.time() - self._start) + if uptime <= 0: + self.log.logging("Stats", "Error", "Invalid uptime calculation: uptime must be greater than 0.") + return + + Rxps = round(self._received / uptime, 2) + Txps = round(self._sent / uptime, 2) + + # Maintain the size of the TrendStats table + if len(self.TrendStats) >= MAX_TREND_STAT_TABLE: + self.TrendStats.pop(0) + + # Append the new data point + self.TrendStats.append({ + "_TS": TimeStamp, + "Rxps": Rxps, + "Txps": Txps, + "Load": self._Load + }) + + except Exception as e: + self.log.logging("Stats", "Error", f"Failed to add point to trend stats: {e}") + def reTx(self): """ return the number of crc Errors """ return self._reTx + def crcErrors(self): " return the number of crc Errors " return self._crcErrors + def frameErrors(self): " return the number of frame errors" return self._frameErrors + def sent(self): " return he number of sent messages" return self._sent + def received(self): " return the number of received messages" return self._received + def ackReceived(self): return self._ack + def ackKOReceived(self): return self._ackKO + def dataReceived(self): return self._data + def TOstatus(self): return self._TOstatus + def TOdata(self): return self._TOdata + def clusterOK(self): return self._clusterOK + def clusterKO(self): return self._clusterKO + def APSFailure(self): return self._APSFailure + def APSAck(self): return self._APSAck + def APSNck(self): return self._APSNck + def printSummary(self): - if self.received() == 0: - return - if self.sent() == 0 or self.received() == 0: + """ + Prints a summary of plugin statistics, including transmission, + reception, and timing metrics. + """ + if self.received() == 0 or self.sent() == 0: return + + def print_with_percentage(label, value, total): + percentage = round((value / total) * 100, 2) + domoticz_status_api(f"{label}: {value} ({percentage}%)") + domoticz_status_api("Plugin statistics") domoticz_status_api(" Messages Sent:") - domoticz_status_api(" Max Load (Queue) : %s " % (self._MaxLoad)) - domoticz_status_api(" TX commands : %s" % (self.sent())) - domoticz_status_api(" TX failed : %s (%s" % (self.ackKOReceived(), round((self.ackKOReceived() / self.sent()) * 10, 2))+ "%)") + domoticz_status_api(f" Max Load (Queue) : {self._MaxLoad}") + domoticz_status_api(f" TX commands : {self.sent()}") + + print_with_percentage(" TX failed", self.ackKOReceived(), self.sent()) if self.zigbee_communication == "native": - domoticz_status_api(" TX timeout : %s (%s" % (self.TOstatus(), round((self.TOstatus() / self.sent()) * 100, 2)) + "%)") + print_with_percentage(" TX timeout", self.TOstatus(), self.sent()) - domoticz_status_api(" TX data timeout : %s (%s" % (self.TOdata(), round((self.TOdata() / self.sent()) * 100, 2)) + "%)") - domoticz_status_api(" TX reTransmit : %s (%s" % (self.reTx(), round((self.reTx() / self.sent()) * 100, 2)) + "%)") + print_with_percentage(" TX data timeout", self.TOdata(), self.sent()) + print_with_percentage(" TX reTransmit", self.reTx(), self.sent()) if self.zigbee_communication == "native": - domoticz_status_api(" TX APS Failure : %s (%s" % (self.APSFailure(), round((self.APSFailure() / self.sent()) * 100, 2))+ "%)") + print_with_percentage(" TX APS Failure", self.APSFailure(), self.sent()) - domoticz_status_api(" TX APS Ack : %s (%s" % (self.APSAck(), round((self.APSAck() / self.sent()) * 100, 2)) + "%)") - domoticz_status_api(" TX APS Nck : %s (%s" % (self.APSNck(), round((self.APSNck() / self.sent()) * 100, 2)) + "%)") + print_with_percentage(" TX APS Ack", self.APSAck(), self.sent()) + print_with_percentage(" TX APS Nck", self.APSNck(), self.sent()) domoticz_status_api(" Messages Received:") - domoticz_status_api(" RX frame : %s" % (self.received())) - domoticz_status_api(" RX clusters : %s" % (self.clusterOK())) - domoticz_status_api(" RX clusters KO : %s" % (self.clusterKO())) + domoticz_status_api(f" RX frame : {self.received()}") + domoticz_status_api(f" RX clusters : {self.clusterOK()}") + domoticz_status_api(f" RX clusters KO : {self.clusterKO()}") if self.zigbee_communication == "native": domoticz_status_api(" Coordinator reacting time on Tx (if ReactTime enabled)") - domoticz_status_api(" Max : %s sec" % (self._maxTiming8000)) - domoticz_status_api(" Average : %s sec" % (self._averageTiming8000)) - + domoticz_status_api(f" Max : {self._maxTiming8000} sec") + domoticz_status_api(f" Average : {self._averageTiming8000} sec") else: domoticz_status_api(" Plugin reacting time on Tx (if ReactTime enabled)") - domoticz_status_api(" Max : %s ms" % (self._max_reading_zigpy_timing)) - domoticz_status_api(" Average : %s ms" % (self._average_reading_zigpy_timing)) + domoticz_status_api(f" Max : {self._max_reading_zigpy_timing} ms") + domoticz_status_api(f" Average : {self._average_reading_zigpy_timing} ms") domoticz_status_api(" Plugin processing time on Rx (if ReactTime enabled)") - if self.zigbee_communication == "native": - domoticz_status_api(" Max : %s sec" % (self._maxRxProcesses)) - domoticz_status_api(" Average : %s sec" % (self._averageRxProcess)) - else: - domoticz_status_api(" Max : %s ms" % (self._maxRxProcesses)) - domoticz_status_api(" Average : %s ms" % (self._averageRxProcess)) - - t0 = self.starttime() - t1 = int(time()) - _days = 0 - _duration = t1 - t0 - _hours = _duration // 3600 - _duration = _duration % 3600 - if _hours >= 24: - _days = _hours // 24 - _hours = _hours % 24 - _min = _duration // 60 - _duration = _duration % 60 - _sec = _duration % 60 - domoticz_status_api(" Operating time : %s Hours %s Mins %s Secs" % (_hours, _min, _sec)) + timing_unit = "sec" if self.zigbee_communication == "native" else "ms" + domoticz_status_api(f" Max : {self._maxRxProcesses} {timing_unit}") + domoticz_status_api(f" Average : {self._averageRxProcess} {timing_unit}") + days, hours, mins, secs = _plugin_uptime(self.starttime()) + domoticz_status_api(" Operating time : %d Days %d Hours %d Mins %d Secs" % (days, hours, mins, secs)) - def writeReport(self): - timing = int(time()) - stats = {timing: {}} - stats[timing]["crcErrors"] = self._crcErrors - stats[timing]["frameErrors"] = self._frameErrors - stats[timing]["sent"] = self._sent - stats[timing]["received"] = self._received - stats[timing]["APS Ack"] = self._APSAck - stats[timing]["APS Nck"] = self._APSNck - stats[timing]["ack"] = self._ack - stats[timing]["ackKO"] = self._ackKO - stats[timing]["data"] = self._data - stats[timing]["TOstatus"] = self._TOstatus - stats[timing]["TOdata"] = self._TOdata - stats[timing]["clusterOK"] = self._clusterOK - stats[timing]["clusterKO"] = self._clusterKO - stats[timing]["reTx"] = self._reTx - stats[timing]["MaxLoad"] = self._MaxLoad - stats[timing]["start"] = self._start - stats[timing]["stop"] = timing - - json_filename = self.pluginconf.pluginConf["pluginReports"] + "Transport-stats.json" - with open(json_filename, "at") as json_file: - json_file.write("\n") - json.dump(stats, json_file) + def writeReport(self): + """ + Write transport statistics to a JSON file. + """ + # Collect the current timestamp + current_time = int(time.time()) + + # Prepare stats dictionary + stats = { + current_time: { + "crcErrors": self._crcErrors, + "frameErrors": self._frameErrors, + "sent": self._sent, + "received": self._received, + "APS Ack": self._APSAck, + "APS Nck": self._APSNck, + "ack": self._ack, + "ackKO": self._ackKO, + "data": self._data, + "TOstatus": self._TOstatus, + "TOdata": self._TOdata, + "clusterOK": self._clusterOK, + "clusterKO": self._clusterKO, + "reTx": self._reTx, + "MaxLoad": self._MaxLoad, + "start": self._start, + "stop": current_time, + } + } + + # Construct the JSON file path + json_filename = Path(self.pluginconf.pluginConf["pluginReports"]) / "Transport-stats.json" + + try: + # Append statistics to the JSON file + with open(json_filename, "a") as json_file: # Use 'a' for appending + json_file.write("\n") + json.dump(stats, json_file, indent=4) # Add indent for better readability + except Exception as e: + self.log.logging("Plugin", "Error", f"Failed to write transport stats: {e}") + + +def _plugin_uptime(starttime): + """ + Calculates the uptime since the given start time. + + Args: + starttime (int): The start time in seconds since the epoch. + + Returns: + tuple: Uptime in days, hours, minutes, and seconds. + """ + t1 = int(time.time()) + _duration = t1 - starttime + + _days = _duration // (24 * 3600) + _duration %= 24 * 3600 + _hours = _duration // 3600 + _duration %= 3600 + _mins = _duration // 60 + _secs = _duration % 60 + + return _days, _hours, _mins, _secs diff --git a/Classes/WebServer/WebServer.py b/Classes/WebServer/WebServer.py index 38ed9c0d2..861d6a825 100644 --- a/Classes/WebServer/WebServer.py +++ b/Classes/WebServer/WebServer.py @@ -1082,8 +1082,8 @@ def rest_zDevice(self, verb, data, parameters): if attribut == "Battery" and attribut in self.ListOfDevices[item]: if self.ListOfDevices[item]["Battery"] in ( {}, ) and "IASBattery" in self.ListOfDevices[item]: device[attribut] = str(self.ListOfDevices[item][ "IASBattery" ]) - elif isinstance( self.ListOfDevices[item]["Battery"], (int,float)): - device[attribut] = int(self.ListOfDevices[item]["Battery"]) + elif isinstance( self.ListOfDevices[item]["Battery"], int): + device[attribut] = self.ListOfDevices[item]["Battery"] device["BatteryInside"] = True elif item == "CheckParam": @@ -1530,56 +1530,102 @@ def rest_zigate_mode(self, verb, data, parameters): _response["Data"] = json.dumps("ZiGate mode: %s requested" % mode) return _response - def rest_battery_state(self, verb, data, parameters): _response = prepResponseMessage(self, setupHeadersResponse()) _response["Headers"]["Content-Type"] = "application/json; charset=utf-8" if verb == "GET": _battEnv = {"Battery":{"<30%":{}, "<50%": {}, ">50%" : {}},"Update Time":{ "Unknown": {}, "< 1 week": {}, "> 1 week": {}}} for x in self.ListOfDevices: - self.logging("Debug", f"rest_battery_state - {x}") if x == "0000": - continue + continue - battery = self.ListOfDevices[x].get("Battery") + if self.ListOfDevices[x]["ZDeviceName"] == "": + _deviceName = x + else: + _deviceName = self.ListOfDevices[x]["ZDeviceName"] - if battery is None: - continue - self.logging("Debug", f"rest_battery_state - {x} Battery found") + if "Battery" in self.ListOfDevices[x] and isinstance(self.ListOfDevices[x]["Battery"], int): + if self.ListOfDevices[x]["Battery"] > 50: + _battEnv["Battery"][">50%"][_deviceName] = {"Battery": self.ListOfDevices[x]["Battery"]} - _device_name = self.ListOfDevices[x].get("ZDeviceName", x ) + elif self.ListOfDevices[x]["Battery"] > 30: + _battEnv["Battery"]["<50%"][_deviceName] = {"Battery": self.ListOfDevices[x]["Battery"]} - if not isinstance( battery, (int, float)): - self.logging("Debug", f"rest_battery_state - {x} Battery found, but not int !! {type(battery)}") - continue - battery = int(battery) + else: + _battEnv["Battery"]["<30%"][_deviceName] = {"Battery": self.ListOfDevices[x]["Battery"]} - if self.ListOfDevices[x]["Battery"] > 50: - _battEnv["Battery"][">50%"][_device_name] = {"Battery": battery} + if "BatteryUpdateTime" in self.ListOfDevices[x]: + if (int(time.time()) - self.ListOfDevices[x]["BatteryUpdateTime"]) > 604800: # one week in seconds + _battEnv["Update Time"]["> 1 week"][_deviceName] = {"BatteryUpdateTime": self.ListOfDevices[x]["BatteryUpdateTime"]} - elif self.ListOfDevices[x]["Battery"] > 30: - _battEnv["Battery"]["<50%"][_device_name] = {"Battery": battery} + else: + _battEnv["Update Time"]["< 1 week"][_deviceName] = {"BatteryUpdateTime": self.ListOfDevices[x]["BatteryUpdateTime"]} - else: - _battEnv["Battery"]["<30%"][_device_name] = {"Battery": battery} + else: + _battEnv["Update Time"]["Unknown"][_deviceName] = "Unknown" + + _response["Data"] = json.dumps(_battEnv, sort_keys=True) + return _response + + def rest_ota_firmware_available(self, verb, data, parameters): + _response = prepResponseMessage(self, setupHeadersResponse()) + _response["Headers"]["Content-Type"] = "application/json; charset=utf-8" - if "BatteryUpdateTime" in self.ListOfDevices[x]: - if (int(time.time()) - self.ListOfDevices[x]["BatteryUpdateTime"]) > 604800: # one week in seconds - _battEnv["Update Time"]["> 1 week"][_device_name] = {"BatteryUpdateTime": self.ListOfDevices[x]["BatteryUpdateTime"]} + if verb == "GET": + _fwAvail = [] - else: - _battEnv["Update Time"]["< 1 week"][_device_name] = {"BatteryUpdateTime": self.ListOfDevices[x]["BatteryUpdateTime"]} + # Iterate over devices + for device_key, device_data in self.ListOfDevices.items(): + # Skip processing for device with key "0000" + if device_key == "0000": + continue - else: - _battEnv["Update Time"]["Unknown"][_device_name] = "Unknown" + ota_update = device_data.get("OTAUpdate") + if ota_update is None: + continue + + # Retrieve device-level information + zdevice_name = device_data.get("ZDeviceName") + model = device_data.get("Model", "Unknown") + manufacturer_name = device_data.get("Manufacturer Name", "Unknown") + manufacturer_id = device_data.get("Manufacturer", "Unknown") + ieee = device_data.get("IEEE", "Unknown") + + # Process firmware updates + for fwtype, update_info in ota_update.items(): + current_version = update_info.get("currentversion", "N/A") + newest_version = update_info.get("newestversion", "N/A") + url = update_info.get("url", "N/A") + + device = { + 'nwkid': device_key, + 'zdevicename': zdevice_name, + 'model': model, + 'manufacturername': manufacturer_name, + 'manufacturerid': manufacturer_id, + 'ieee': ieee, + 'fwtype': fwtype, + 'currentversion': current_version, + 'newestversion': newest_version, + 'url': url, + } + + self.logging("Status", f"Processing OTA update for {zdevice_name}: {device}") + _fwAvail.append(device) + + # Log collected firmware availability information + self.logging("Status", f"Collected OTA firmware availability: {_fwAvail}") + + # Include firmware data in the response + _response["Data"] = json.dumps(_fwAvail, sort_keys=True) - self.logging("Debug", f"rest_battery_state - {_battEnv}") - _response["Data"] = json.dumps(_battEnv, sort_keys=True) return _response - + + def logging(self, logType, message): self.log.logging("WebServer", logType, message) +## Helpers def dummy_zdevice_name(): return [{"Battery": "", "ConsistencyCheck": "ok", "Health": "Disabled", "IEEE": "90fd9ffffe86c7a1", "LQI": 80, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "TRADFRI bulb E27 WS clear 950lm", "Param": "{'Disabled': true, 'PowerOnAfterOffOn': 255, 'fadingOff': 0, 'moveToHueSatu': 0, 'moveToColourTemp': 0, 'moveToColourRGB': 0, 'moveToLevel': 0}", "Status": "inDB", "WidgetList": ["Zigbee - TRADFRI bulb E27 WS clear 950lm_ColorControlWW-90fd9ffffe86c7a1-01"], "ZDeviceName": "Led Ikea", "_NwkId": "ada7"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "60a423fffe529d60", "LQI": 80, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "LXEK-1", "Param": "{'PowerOnAfterOffOn': 255, 'fadingOff': 0, 'moveToHueSatu': 0, 'moveToColourTemp': 0, 'moveToColourRGB': 0, 'moveToLevel': 0}", "Status": "inDB", "WidgetList": ["Zigbee - LXEK-1_ColorControlRGBWW-60a423fffe529d60-01"], "ZDeviceName": "Led LKex", "_NwkId": "7173"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "680ae2fffe7aca89", "LQI": 80, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "TRADFRI Signal Repeater", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - TRADFRI Signal Repeater_Voltage-680ae2fffe7aca89-01"], "ZDeviceName": "Repeater", "_NwkId": "a5ee"}, {"Battery": 16.0, "ConsistencyCheck": "ok", "Health": "Not seen last 24hours", "IEEE": "90fd9ffffeea89e8", "LQI": 25, "MacCapa": ["RFD", "Battery"], "Model": "TRADFRI remote control", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - TRADFRI remote control_Ikea_Round_5b-90fd9ffffeea89e8-01"], "ZDeviceName": "Remote Tradfri", "_NwkId": "cee1"}, {"Battery": 100, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "000d6f0011087079", "LQI": 116, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "WarningDevice", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - WarningDevice_AlarmWD-000d6f0011087079-01"], "ZDeviceName": "IAS Sirene", "_NwkId": "2e33"}, {"Battery": 53, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "54ef441000298533", "LQI": 76, "MacCapa": ["RFD", "Battery"], "Model": "lumi.magnet.acn001", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - lumi.magnet.acn001_Door-54ef441000298533-01"], "ZDeviceName": "Lumi Door", "_NwkId": "bb45"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "00047400008aff8b", "LQI": 80, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "Shutter switch with neutral", "Param": "{'netatmoInvertShutter': 0, 'netatmoLedShutter': 0}", "Status": "inDB", "WidgetList": ["Zigbee - Shutter switch with neutral_Venetian-00047400008aff8b-01"], "ZDeviceName": "Inter Shutter Legrand", "_NwkId": "06ab"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "000474000082a54f", "LQI": 18, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "Dimmer switch wo neutral", "Param": "{'netatmoEnableDimmer': 1, 'PowerOnAfterOffOn': 255, 'BallastMaxLevel': 254, 'BallastMinLevel': 1}", "Status": "inDB", "WidgetList": ["Zigbee - Dimmer switch wo neutral_LvlControl-000474000082a54f-01"], "ZDeviceName": "Inter Dimmer Legrand", "_NwkId": "9c25"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "00047400001f09a4", "LQI": 80, "MacCapa": ["FFD", "RxonIdle", "MainPower"], "Model": "Micromodule switch", "Param": "{'PowerOnAfterOffOn': 255}", "Status": "inDB", "WidgetList": ["Zigbee - Micromodule switch_Switch-00047400001f09a4-01"], "ZDeviceName": "Micromodule Legrand", "_NwkId": "8706"}, {"Battery": "", "ConsistencyCheck": "ok", "Health": "", "IEEE": "00158d0003021601", "LQI": 0, "MacCapa": ["RFD", "Battery"], "Model": "lumi.sensor_motion.aq2", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - lumi.sensor_motion.aq2_Motion-00158d0003021601-01", "Zigbee - lumi.sensor_motion.aq2_Lux-00158d0003021601-01"], "ZDeviceName": "Lumi Motion", "_NwkId": "6f81"}, {"Battery": 100, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "0015bc001a01aa27", "LQI": 83, "MacCapa": ["RFD", "Battery"], "Model": "MOSZB-140", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - MOSZB-140_Motion-0015bc001a01aa27-23", "Zigbee - MOSZB-140_Tamper-0015bc001a01aa27-23", "Zigbee - MOSZB-140_Voltage-0015bc001a01aa27-23", "Zigbee - MOSZB-140_Temp-0015bc001a01aa27-26", "Zigbee - MOSZB-140_Lux-0015bc001a01aa27-27"], "ZDeviceName": "Motion frient", "_NwkId": "b9bc"}, {"Battery": 63, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "00158d000323dabe", "LQI": 61, "MacCapa": ["RFD", "Battery"], "Model": "lumi.sensor_switch", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - lumi.sensor_switch_SwitchAQ2-00158d000323dabe-01"], "ZDeviceName": "Lumi Switch (rond)", "_NwkId": "a029"}, {"Battery": 100.0, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "000d6ffffea1e6da", "LQI": 94, "MacCapa": ["RFD", "Battery"], "Model": "TRADFRI onoff switch", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - TRADFRI onoff switch_SwitchIKEA-000d6ffffea1e6da-01"], "ZDeviceName": "OnOff Ikea", "_NwkId": "c6ca"}, {"Battery": 100.0, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "000b57fffe2c0dde", "LQI": 87, "MacCapa": ["RFD", "Battery"], "Model": "TRADFRI wireless dimmer", "Param": "{}", "Status": "inDB", "WidgetList": ["Zigbee - TRADFRI wireless dimmer_GenericLvlControl-000b57fffe2c0dde-01"], "ZDeviceName": "Dim Ikea", "_NwkId": "6c43"}, {"Battery": 100, "ConsistencyCheck": "ok", "Health": "Live", "IEEE": "588e81fffe35f595", "LQI": 80, "MacCapa": ["RFD", "Battery"], "Model": "Wiser2-Thermostat", "Param": "{'WiserLockThermostat': 0, 'WiserRoomNumber': 1}", "Status": "inDB", "WidgetList": ["Zigbee - Wiser2-Thermostat_Temp+Hum-588e81fffe35f595-01", "Zigbee - Wiser2-Thermostat_Humi-588e81fffe35f595-01", "Zigbee - Wiser2-Thermostat_Temp-588e81fffe35f595-01", "Zigbee - Wiser2-Thermostat_ThermoSetpoint-588e81fffe35f595-01", "Zigbee - Wiser2-Thermostat_Valve-588e81fffe35f595-01"], "ZDeviceName": "Wiser Thermostat", "_NwkId": "5a00"}] diff --git a/Classes/WebServer/com.py b/Classes/WebServer/com.py index 2bea81a7b..538f57ced 100644 --- a/Classes/WebServer/com.py +++ b/Classes/WebServer/com.py @@ -154,7 +154,7 @@ def handle_client(self, client_socket, client_addr): while self.running: try: # Let's receive the first chunck (to get the headers) - data = receive_data(self, client_socket).decode('utf-8') + data = receive_data(self, client_socket).decode('utf-8', errors='ignore') if not data: self.logging("Debug", f"no data from {client_addr}") @@ -189,8 +189,8 @@ def handle_client(self, client_socket, client_addr): break except Exception as e: - self.logging("Log", f"Unexpected error with {client_addr}: {e}") - self.logging("Log", f"{traceback.format_exc()}") + self.logging("Error", f"handle_client Unexpected error with {client_addr}: {e} from method: {method} path: {path} content_length: {content_length} len_body: {len(body)} headers: {headers}") + self.logging("Error", f"{traceback.format_exc()}") break finally: diff --git a/Classes/ZigpyTransport/AppBellows.py b/Classes/ZigpyTransport/AppBellows.py index 0bd462b14..f5f546224 100644 --- a/Classes/ZigpyTransport/AppBellows.py +++ b/Classes/ZigpyTransport/AppBellows.py @@ -65,6 +65,7 @@ async def startup(self, statistics, HardwareID, pluginconf, use_of_zigpy_persist self.shutting_down = False self.restarting = False + self.radio_lost_cnt = 0 try: await self.connect() diff --git a/Classes/ZigpyTransport/AppDeconz.py b/Classes/ZigpyTransport/AppDeconz.py index ea38b64fb..9d44a6046 100644 --- a/Classes/ZigpyTransport/AppDeconz.py +++ b/Classes/ZigpyTransport/AppDeconz.py @@ -62,6 +62,7 @@ async def startup(self, statistics, HardwareID, pluginconf, use_of_zigpy_persist self.shutting_down = False self.restarting = False + self.radio_lost_cnt = 0 await asyncio.sleep( 3 ) diff --git a/Classes/ZigpyTransport/AppGeneric.py b/Classes/ZigpyTransport/AppGeneric.py index b3eba417e..bf5b3bad2 100644 --- a/Classes/ZigpyTransport/AppGeneric.py +++ b/Classes/ZigpyTransport/AppGeneric.py @@ -159,16 +159,16 @@ async def _create_backup(self) -> None: def connection_lost(self, exc: Exception) -> None: """Handle connection lost event.""" - LOGGER.warning( "+ Connection to the radio was lost (trying to recover): %s %r" %(type(exc), exc) ) - super(type(self),self).connection_lost(exc) + if exc is None or self.restarting or self.shutting_down: + # exc is None: Connection closed cleanly. do nothing + # self.restarting: Restart in progress + return - if not self.shutting_down and not self.restarting and isinstance( exc, (serial.serialutil.SerialException, TimeoutError)): - LOGGER.error( "++++++++++++++++++++++ Connection to coordinator failed on Serial, let's restart the plugin") - LOGGER.warning( f"--> : self.shutting_down: {self.shutting_down}, {self.restarting}") + LOGGER.error( f"+ Connection to the radio was lost due to {type(exc)}, restart plugin") - self.restarting = True - self.callBackRestartPlugin() + self.restarting = True + self.callBackRestartPlugin() # Trigger plugin restart def _retreive_previous_backup(self): @@ -294,8 +294,15 @@ def wrapper(self, packet): if t_start: t_end = int(1000 * time.time()) t_elapse = t_end - t_start - self.statistics.add_rxTiming(t_elapse) - self.log.logging("TransportZigpy", "Log", f"| (packet_received) | {t_elapse} | {packet.src.address.serialize()[::-1].hex()} | {packet.profile_id} | {packet.lqi} | {packet.rssi} |") + self.statistics.add_rxTiming(t_elapse) + self.log.logging( + "TransportZigpy", + "Log", + f"| (packet_received) | {t_elapse} | {f'{packet.tsn:#02x}' if packet.tsn is not None else 'N/A'} | " + f"0x{packet.src.address.serialize()[::-1].hex()} | {f'{packet.profile_id:#04x}' if packet.profile_id is not None else 'N/A'} | " + f"{packet.lqi} | {packet.rssi} |" + ) + return wrapper diff --git a/Classes/ZigpyTransport/AppZnp.py b/Classes/ZigpyTransport/AppZnp.py index 21328d60c..5cc0995e7 100644 --- a/Classes/ZigpyTransport/AppZnp.py +++ b/Classes/ZigpyTransport/AppZnp.py @@ -66,6 +66,7 @@ async def startup(self, statistics, HardwareID, pluginconf, use_of_zigpy_persist self.shutting_down = False self.restarting = False + self.radio_lost_cnt = 0 # Pipiche : 24-Oct-2022 Disabling CONF_MAX_CONCURRENT_REQUESTS so the default will be used ( 16 ) # self.znp_config[znp_conf.CONF_MAX_CONCURRENT_REQUESTS] = 2 diff --git a/Classes/ZigpyTransport/plugin_encoders.py b/Classes/ZigpyTransport/plugin_encoders.py index ef22fea41..ddfe43da1 100644 --- a/Classes/ZigpyTransport/plugin_encoders.py +++ b/Classes/ZigpyTransport/plugin_encoders.py @@ -119,7 +119,8 @@ def build_plugin_8011_frame_content(self, nwkid, status, lqi): # MsgSEQ = MsgData[12:14] if MsgLen > 12 else None lqi = lqi or 0x00 - frame_payload = "%02x" % status + nwkid + frame_payload = "%02x" % status + frame_payload += nwkid return encapsulate_plugin_frame("8011", frame_payload, "%02x" % lqi) def build_plugin_8014_frame_content(self, nwkid, payload): diff --git a/Classes/ZigpyTransport/zigpyThread.py b/Classes/ZigpyTransport/zigpyThread.py index 49efba4b7..891082742 100644 --- a/Classes/ZigpyTransport/zigpyThread.py +++ b/Classes/ZigpyTransport/zigpyThread.py @@ -825,6 +825,7 @@ async def _send_and_retry(self, Function, destination, Profile, Cluster, _nwkid, try: self.log.logging("TransportZigpy", "Debug", f"_send_and_retry: {_ieee} {Profile:X} {Cluster:X} - AckIsDisable: {ack_is_disable} extended_timeout: {extended_timeout} Attempts: {attempt}/{max_retry}") result, _ = await zigpy_request(self, destination, Profile, Cluster, sEp, dEp, sequence, payload, ack_is_disable=ack_is_disable, use_ieee=use_ieee, extended_timeout=extended_timeout) + self.radio_lost_cnt = 0 except (asyncio.exceptions.TimeoutError, asyncio.exceptions.CancelledError, AttributeError, DeliveryError) as e: error_log_message = f"Warning while submitting - {Function} {_ieee}/0x{_nwkid} 0x{Profile:X} 0x{Cluster:X} payload: {payload} AckIsDisable: {ack_is_disable} Retry: {attempt}/{max_retry} with exception: '{e}' ({type(e)}))" @@ -844,6 +845,7 @@ async def _send_and_retry(self, Function, destination, Profile, Cluster, _nwkid, error_log_message = f"_send_and_retry - Unexpected Exception - {Function} {_ieee}/0x{_nwkid} 0x{Profile:X} 0x{Cluster:X} payload: {payload} AckIsDisable: {ack_is_disable} RETRY: {attempt}/{max_retry} ({error})" self.log.logging("TransportZigpy", "Error", error_log_message) result = 0xB6 + break else: # Success diff --git a/Modules/basicOutputs.py b/Modules/basicOutputs.py index 55a799cb1..8064260af 100644 --- a/Modules/basicOutputs.py +++ b/Modules/basicOutputs.py @@ -638,17 +638,37 @@ def set_poweron_afteroffon(self, key, OnOffMode=0xFF): del self.ListOfDevices[key]["Ep"][EPout]["0006"][attribute] return write_attribute( self, key, ZIGATE_EP, EPout, cluster_id, manuf_id, manuf_spec, attribute, data_type, data, ackIsDisabled=True, ) -def handle_unknow_device( self, Nwkid): - # This device is unknown, and we don't have the IEEE to check if there is a device coming with a new sAddr - # Will request in the next hearbeat to for a IEEE request + +def handle_unknow_device(self, Nwkid): + """ + Handles an unknown device by attempting to look up its IEEE address. + + If an IEEE address is found for the given network ID (Nwkid), logs the result + and removes the Nwkid from the unknown devices list. Otherwise, attempts to + find a match for the Nwkid and ensures it is tracked for future processing. + + Args: + Nwkid (str): The network ID of the device. + + Returns: + None + """ + # Attempt to lookup IEEE address for the given network ID ieee = lookupForIEEE(self, Nwkid, True) if ieee: - self.log.logging("Input", "Debug", "Found IEEE for short address: %s is %s" % (Nwkid, ieee)) + self.log.logging("Input", "Debug", f"Found IEEE for short address: {Nwkid} is {ieee}") + + # Remove from UnknownDevices if present if Nwkid in self.UnknownDevices: self.UnknownDevices.remove(Nwkid) else: + # Try finding a matching IEEE for the Nwkid try_to_find_ieee_matching_nwkid(self, Nwkid) - self.UnknownDevices.append(Nwkid) + + # Add to UnknownDevices if not already present + if Nwkid not in self.UnknownDevices: + self.UnknownDevices.append(Nwkid) + def try_to_find_ieee_matching_nwkid(self, nwkid): diff --git a/Modules/checkingUpdate.py b/Modules/checkingUpdate.py index 621a89aed..9a436d447 100644 --- a/Modules/checkingUpdate.py +++ b/Modules/checkingUpdate.py @@ -11,9 +11,6 @@ # SPDX-License-Identifier: GPL-3.0 license # Use DNS TXT to check latest version available on gitHub -# - stable -# - beta - import dns.resolver import requests @@ -33,66 +30,122 @@ def check_plugin_version_against_dns(self, zigbee_communication, branch, zigate_model): - self.log.logging("Plugin", "Debug", f"check_plugin_version_against_dns {zigbee_communication} {branch} {zigate_model}") + """ + Checks the plugin version against DNS TXT records and fetches firmware details if applicable. + + Args: + zigbee_communication (str): Communication type ('native' or 'zigpy'). + branch (str): Current plugin branch (e.g., 'master', 'beta'). + zigate_model (str): Zigate model identifier. - plugin_version = None - plugin_version = _get_dns_txt_record(self, PLUGIN_TXT_RECORD) - self.log.logging("Plugin", "Debug", f"check_plugin_version_against_dns {plugin_version}") + Returns: + tuple: Plugin version, firmware major version, firmware minor version. + """ + self.log.logging("CheckUpdate", "Debug", f"check_plugin_version_against_dns {zigbee_communication} {branch} {zigate_model}") + # Fetch and parse plugin version DNS record + plugin_version = _fetch_and_parse_dns_record(self, PLUGIN_TXT_RECORD, "Plugin") if plugin_version is None: - # Something weird happened - self.log.logging("Plugin", "Error", "Unable to get access to plugin expected version. Is Internet access available ?") + self.log.logging("CheckUpdate", "Error", "Unable to access plugin version. Is Internet access available?") return (0, 0, 0) - plugin_version_dict = _parse_dns_txt_record( plugin_version) - self.log.logging("Plugin", "Debug", f"check_plugin_version_against_dns {plugin_version} {plugin_version_dict}") + self.log.logging("CheckUpdate", "Debug", f" plugin version: >{plugin_version}") - # If native communication (zigate) let's find the zigate firmware - firmware_version = None + firmware_version_dict = {} if zigbee_communication == "native": - zigate_plugin_record = ZIGATE_DNS_RECORDS.get(zigate_model) - firmware_version = _get_dns_txt_record(self, zigate_plugin_record) - firmware_version_dict = _parse_dns_txt_record(firmware_version) - self.log.logging("Plugin", "Debug", f"check_plugin_version_against_dns {firmware_version} {firmware_version_dict}") + firmware_version = _fetch_and_parse_dns_record(self, ZIGATE_DNS_RECORDS.get(zigate_model), "Firmware") + if firmware_version: + firmware_version_dict = firmware_version - self.log.logging("Plugin", "Debug", f"check_plugin_version_against_dns {plugin_version} {plugin_version_dict}") + self.log.logging("CheckUpdate", "Debug", f" firmware version: >{firmware_version_dict}") + + # Determine the response based on communication type and branch support + if zigbee_communication == "native" and branch in plugin_version and "firmMajor" in firmware_version_dict and "firmMinor" in firmware_version_dict: + return (plugin_version[branch], firmware_version_dict["firmMajor"], firmware_version_dict["firmMinor"]) - if zigbee_communication == "native" and branch in plugin_version_dict and "firmMajor" in firmware_version_dict and "firmMinor" in firmware_version_dict: - return (plugin_version_dict[branch], firmware_version_dict["firmMajor"], firmware_version_dict["firmMinor"]) + if zigbee_communication == "zigpy" and branch in plugin_version: + return (plugin_version[branch], 0, 0) - if zigbee_communication == "zigpy" and branch in plugin_version_dict: - return (plugin_version_dict[branch], 0, 0) - - self.log.logging("Plugin", "Error", f"You are running {branch}-{plugin_version}, a NOT SUPPORTED version. ") + self.log.logging("CheckUpdate", "Error", f"You are running on branch: {branch}, which is NOT SUPPORTED.") return (0, 0, 0) +def _fetch_and_parse_dns_record(self, record_name, record_type): + """ + Fetches and parses a DNS TXT record. + + Args: + record_name (str): The name of the DNS TXT record. + record_type (str): Type of record (e.g., 'Plugin', 'Firmware') for logging. + + Returns: + dict or None: Parsed DNS record as a dictionary, or None if unavailable. + """ + if not record_name: + self.log.logging("CheckUpdate", "Error", f"{record_type} DNS record not found.") + return None + + self.log.logging("CheckUpdate", "Debug", f"Fetching {record_type} DNS record: {record_name}") + record = _get_dns_txt_record(self, record_name) + if record is None: + self.log.logging("CheckUpdate", "Error", f"Failed to fetch {record_type} DNS record: {record_name}") + return None + + self.log.logging("CheckUpdate", "Debug", f"Fetching {record_type} DNS record: {record_name} = {record}") + + parsed_record = _parse_dns_txt_record(record) + self.log.logging("CheckUpdate", "Debug", f"Fetched and parsed {record_type} DNS record: {parsed_record}") + return parsed_record + + def _get_dns_txt_record(self, record, timeout=DNS_REQ_TIMEOUT): + """ + Fetch a DNS TXT record. + + Args: + record (str): The DNS record to fetch. + timeout (int): Timeout for the DNS query in seconds. + + Returns: + str or None: The DNS TXT record as a string, or None if unavailable. + """ if not self.internet_available: + self.log.logging("CheckUpdate", "Error", f"Internet unavailable, skipping DNS resolution for {record}") return None try: + # Attempt to resolve the DNS TXT record result = dns.resolver.resolve(record, "TXT", tcp=True, lifetime=timeout).response.answer[0] return str(result[0]).strip('"') except dns.resolver.Timeout: - error_message = f"DNS resolution timed out for {record} after {timeout} second" - self.internet_available = False + _handle_dns_error(self, f"DNS resolution timed out for {record} after {timeout} second(s)", fatal=True) except dns.resolver.NoAnswer: - error_message = f"DNS TXT record not found for {record}" + _handle_dns_error(self, f"DNS TXT record not found for {record}") except dns.resolver.NoNameservers: - error_message = f"No nameservers found for {record}" - self.internet_available = False + _handle_dns_error(self, f"No nameservers found for {record}", fatal=True) except Exception as e: - error_message = f"An unexpected error occurred while resolving DNS TXT record for {record}: {e}" + _handle_dns_error(self, f"Unexpected error while resolving DNS TXT record for {record}: {e}") - self.log.logging("Plugin", "Error", error_message) return None +def _handle_dns_error(self, message, fatal=False): + """ + Handle DNS errors with consistent logging. + + Args: + message (str): The error message to log. + fatal (bool): If True, set internet_available to False. + """ + self.log.logging("CheckUpdate", "Error", message) + if fatal: + self.internet_available = False + + def _parse_dns_txt_record(txt_record): version_dict = {} if txt_record and txt_record != "": @@ -102,39 +155,72 @@ def _parse_dns_txt_record(txt_record): def is_plugin_update_available(self, currentVersion, availVersion): - if availVersion == 0: - return False + """ + Check if a plugin update is available. - currentMaj, currentMin, currentUpd = currentVersion.split(".") - availMaj, availMin, availUpd = availVersion.split(".") + Args: + currentVersion (str): Current plugin version (e.g., "1.0.0"). + availVersion (str): Available plugin version (e.g., "1.1.0"). - if availMaj > currentMaj: - self.log.logging("Plugin", "Status", "Zigbee4Domoticz plugin: upgrade available: %s" %availVersion) - return True + Returns: + bool: True if an update is available, False otherwise. + """ + if availVersion == "0": + return False - if availMaj == currentMaj and ( - availMin == currentMin - and availUpd > currentUpd - or availMin > currentMin - ): - self.log.logging("Plugin", "Status", "Zigbee4Domoticz plugin: upgrade available: %s" %availVersion) + if _is_newer_version(self, currentVersion, availVersion): + self.log.logging("CheckUpdate", "Status", f"Zigbee4Domoticz plugin: upgrade available: {availVersion}") return True + return False +def _is_newer_version(self, currentVersion, availVersion): + """ + Compare two version strings to determine if the second is newer. + + Args: + currentVersion (str): Current version string (e.g., "1.0.0"). + availVersion (str): Available version string (e.g., "1.1.0"). + + Returns: + bool: True if availVersion is newer than currentVersion, False otherwise. + """ + current = tuple(map(int, currentVersion.split("."))) + available = tuple(map(int, availVersion.split("."))) + return available > current + + def is_zigate_firmware_available(self, currentMajorVersion, currentFirmwareVersion, availfirmMajor, availfirmMinor): if not (availfirmMinor and currentFirmwareVersion): return False if int(availfirmMinor, 16) > int(currentFirmwareVersion, 16): - self.log.logging("Plugin", "Debug", "Zigate Firmware update available") + self.log.logging("CheckUpdate", "Status", "Zigate Firmware update available") return True return False -def is_internet_available(): +def is_internet_available(self): + """ + Check if the internet is available by sending a GET request to a reliable website. + + Returns: + bool: True if the internet is available, False otherwise. + """ + url = "https://www.google.com" + timeout = 3 + try: - response = requests.get("https://www.google.com", timeout=3) - # Check if the status code is a success code (2xx) - return response.status_code == 200 + response = requests.get(url, timeout=timeout) + # Return True if the status code indicates success (2xx) + return 200 <= response.status_code < 300 except requests.ConnectionError: + # Handle cases where the connection fails + return False + except requests.Timeout: + # Handle timeout errors + return False + except requests.RequestException as e: + # Handle other request exceptions + self.log.logging( "Plugin", "Status",f"Unexpected error while checking internet availability: {e}") return False diff --git a/Modules/command.py b/Modules/command.py index c5380d893..ce851d3fb 100644 --- a/Modules/command.py +++ b/Modules/command.py @@ -44,7 +44,7 @@ schneider_temp_Setcurrent) from Modules.switchSelectorWidgets import SWITCH_SELECTORS from Modules.thermostats import thermostat_Mode, thermostat_Setpoint -from Modules.tools import get_deviceconf_parameter_value +from Modules.tools import get_device_nickname, get_deviceconf_parameter_value from Modules.tuya import (tuya_curtain_lvl, tuya_curtain_openclose, tuya_dimmer_dimmer, tuya_dimmer_onoff, tuya_energy_onoff, tuya_garage_door_action, @@ -173,9 +173,18 @@ def domoticz_command(self, Devices, DeviceID, Unit, Nwkid, Command, Level, Color forceUpdateDev = SWITCH_SELECTORS.get(DeviceType, {}).get("ForceUpdate", False) if DeviceType not in ACTIONATORS and not self.pluginconf.pluginConf.get("forcePassiveWidget"): - self.log.logging("Command", "Log", f"mgtCommand - You are trying to action not allowed for Device: {widget_name} Type: {ClusterTypeList} and DeviceType: {DeviceType} Command: {Command} Level:{Level}", Nwkid) + zdevice_name = get_device_nickname(self, NwkId=Nwkid) + self.log.logging( + "Command", + "Log", + ( + f"mgtCommand - You are trying to action a Domoticz widget '{widget_name}' belonging to a 'remote' type of device '{zdevice_name}', which is not actionable by default. " + f"Please use the plugin WebUI advanced settings menu to enable the 'forcePassiveWidget: Allow Domoticz actions on widgets from remote devices' if that is what you need." + ), + Nwkid + ) return - + health_value = self.ListOfDevices.get(Nwkid, {}).get("Health") if health_value == "Not Reachable": self.ListOfDevices.setdefault(Nwkid, {})["Health"] = "" diff --git a/Modules/database.py b/Modules/database.py index 2c3b561dc..c1e92e6cc 100644 --- a/Modules/database.py +++ b/Modules/database.py @@ -127,7 +127,8 @@ "IASBattery", "Operating Time", "DelayBindingAtPairing", - "CertifiedDevice" + "CertifiedDevice", + "OTAUpdate" ) MANUFACTURER_ATTRIBUTES = ( @@ -142,48 +143,53 @@ ) -def LoadDeviceList(self): - # Load DeviceList.txt into ListOfDevices - # - ListOfDevices_from_Domoticz = None +def load_plugin_database(self): + """ Load plugin database in memory. """ - # This can be enabled only with Domoticz version 2021.1 build 1395 and above, otherwise big memory leak + ListOfDevices_imported_from_Domoticz = loaded_from = None + list_of_device_txt_filename = Path( self.pluginconf.pluginConf["pluginData"] ) / self.DeviceListName - if self.pluginconf.pluginConf["useDomoticzDatabase"] and Modules.tools.is_domoticz_db_available(self): - ListOfDevices_from_Domoticz, saving_time = _read_DeviceList_Domoticz(self) - self.log.logging( - "Database", - "Debug", - "Database from Dz is recent: %s Loading from Domoticz Db" - % is_domoticz_recent(self, saving_time, self.pluginconf.pluginConf["pluginData"] + self.DeviceListName) - ) + if self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]: + ListOfDevices_imported_from_Domoticz, saving_time = load_plugin_database_from_domoticz(self) + backup_domoticz_more_recent = is_timestamp_recent_than_filename(self, saving_time, list_of_device_txt_filename ) if os.path.exists(list_of_device_txt_filename) else True + self.log.logging( "Database", "Log", "Database from Dz is recent: %s Loading from Domoticz Db" % backup_domoticz_more_recent) res = "Success" - _pluginConf = Path( self.pluginconf.pluginConf["pluginData"] ) - _DeviceListFileName = _pluginConf / self.DeviceListName - if os.path.isfile(_DeviceListFileName): - res = loadTxtDatabase(self, _DeviceListFileName) - else: - # Do not exist + if os.path.isfile(list_of_device_txt_filename): + # Loading plugin database from txt file and populate the ListOfDevices dictionary self.ListOfDevices = {} - return True + res = load_plugin_database_txt_format(self, list_of_device_txt_filename) - self.log.logging("Database", "Status", "Z4D loads %s entries from %s" % (len(self.ListOfDevices), _DeviceListFileName)) - if ListOfDevices_from_Domoticz: - self.log.logging( "Database", "Log", "Plugin Database loaded - BUT NOT USE - from Dz: %s from DeviceList: %s, checking deltas " % ( len(ListOfDevices_from_Domoticz), len(self.ListOfDevices), ), ) + # Keep the Size of the DeviceList in order to check changes + self.log.logging("Database", "Log", "LoadDeviceList - DeviceList filename : %s" % list_of_device_txt_filename) - self.log.logging("Database", "Debug", "LoadDeviceList - DeviceList filename : %s" % _DeviceListFileName) - Modules.tools.helper_versionFile(_DeviceListFileName, self.pluginconf.pluginConf["numDeviceListVersion"]) + # Manage versioning for a file by creating sequentially numbered backups. + Modules.tools.helper_versionFile(list_of_device_txt_filename, self.pluginconf.pluginConf["numDeviceListVersion"]) + loaded_from = list_of_device_txt_filename - # Keep the Size of the DeviceList in order to check changes - self.DeviceListSize = os.path.getsize(_DeviceListFileName) + # At that stage, we have loaded from Domoticz and from txt file. + if ListOfDevices_imported_from_Domoticz and self.ListOfDevices: + self.log.logging( "Database", "Log", "Sanity check. Plugin database loaded from Domoticz. %s from domoticz, %s from txt file" % ( + len(ListOfDevices_imported_from_Domoticz), len(self.ListOfDevices), ), ) + + if ListOfDevices_imported_from_Domoticz and self.pluginconf.pluginConf["useDomoticzDatabase"] and backup_domoticz_more_recent: + # We will use the Domoticz import. + self.ListOfDevices = {} + self.ListOfDevices = ListOfDevices_imported_from_Domoticz + loaded_from = "Domoticz" + + if loaded_from: + self.log.logging("Database", "Status", "Z4D loads %s entries from %s" % (len(self.ListOfDevices), loaded_from)) cleanup_table_entries( self) + hacks_after_loading(self) + reset_data_structuture_after_load(self) + load_new_param_definition(self) - if self.pluginconf.pluginConf["ZigpyTopologyReport"]: - # Cleanup the old Topology data - remove_legacy_topology_datas(self) - + return res + + +def hacks_after_loading(self): for addr in self.ListOfDevices: # Fixing mistake done in the code. fixing_consumption_lumi(self, addr) @@ -229,6 +235,12 @@ def LoadDeviceList(self): # We need to adjust the Model to the right mode update_zlinky_device_model_if_needed(self, addr) + +def reset_data_structuture_after_load(self): + if self.pluginconf.pluginConf["ZigpyTopologyReport"]: + # Cleanup the old Topology data + remove_legacy_topology_datas(self) + if self.pluginconf.pluginConf["resetReadAttributes"]: self.pluginconf.pluginConf["resetReadAttributes"] = False self.pluginconf.write_Settings() @@ -237,12 +249,10 @@ def LoadDeviceList(self): self.pluginconf.pluginConf["resetConfigureReporting"] = False self.pluginconf.write_Settings() - load_new_param_definition(self) - - return res +def load_plugin_database_txt_format(self, dbName): + """ Load the plugin databqse from .txt format into self.ListOfDevices """ -def loadTxtDatabase(self, dbName): res = "Success" with open(dbName, "r", encoding='utf-8') as myfile2: self.log.logging("Database", "Debug", "Open : %s" % dbName) @@ -275,130 +285,139 @@ def loadTxtDatabase(self, dbName): continue else: nb += 1 - CheckDeviceList(self, key, val) + load_plugin_database_txt_entry(self, key, val) return res -def _read_DeviceList_Domoticz(self): +def load_plugin_database_from_domoticz(self): + """ Load plugin databse from Domoticz return a tuple dictionary and backup timestamp""" - ListOfDevices_from_Domoticz = getConfigItem(Key="ListOfDevices", Attribute="Devices") - time_stamp = 0 - if "TimeStamp" in ListOfDevices_from_Domoticz: - time_stamp = ListOfDevices_from_Domoticz.get("TimeStamp",0) - ListOfDevices_from_Domoticz = ListOfDevices_from_Domoticz.get("Devices",{}) - self.log.logging( - "Database", - "Log", - "Plugin data found on DZ with date %s" - % (time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(time_stamp))), - ) + configuration_entry_imported_from_Domoticz = getConfigItem(Key="ListOfDevices", Attribute="b64encoded") + self.log.logging( "Database", "Debug", f"-- load_plugin_database_from_domoticz loaded {str((configuration_entry_imported_from_Domoticz))}" ) - self.log.logging( - "Database", "Debug", "Load from Dz: %s %s" % (len(ListOfDevices_from_Domoticz), ListOfDevices_from_Domoticz) - ) - if not isinstance(ListOfDevices_from_Domoticz, dict): - ListOfDevices_from_Domoticz = {} - else: - for x in list(ListOfDevices_from_Domoticz): - self.log.logging("Database", "Debug", "--- Loading %s" % (x)) - - for attribute in list(ListOfDevices_from_Domoticz[x]): - if attribute not in (MANDATORY_ATTRIBUTES + MANUFACTURER_ATTRIBUTES + BUILD_ATTRIBUTES): - self.log.logging("Database", "Debug", "xxx Removing attribute: %s for %s" % (attribute, x)) - del ListOfDevices_from_Domoticz[x][attribute] + time_stamp = configuration_entry_imported_from_Domoticz.get("TimeStamp", 0) + listofdevices_imported_from_domoticz = configuration_entry_imported_from_Domoticz.get( "b64encoded", {}) + + if not isinstance(listofdevices_imported_from_domoticz, dict): + listofdevices_imported_from_domoticz = {} + + self.log.logging( "Database", "Log", f"-- load_plugin_database_from_domoticz loaded {len(listofdevices_imported_from_domoticz)} from {(time.strftime('%A, %Y-%m-%d %H:%M:%S', time.localtime(time_stamp)))}" ) - return (ListOfDevices_from_Domoticz, time_stamp) + # dictionary cleanup + for x in list(listofdevices_imported_from_domoticz): + self.log.logging("Database", "Debug", "--- Loading %s" % (x)) + for attribute in list(listofdevices_imported_from_domoticz[x]): + if attribute not in (MANDATORY_ATTRIBUTES + MANUFACTURER_ATTRIBUTES + BUILD_ATTRIBUTES): + self.log.logging("Database", "Debug", "xxx Removing attribute: %s for %s" % (attribute, x)) + del listofdevices_imported_from_domoticz[x][attribute] -def is_domoticz_recent(self, dz_timestamp, device_list_txt_filename): + return (listofdevices_imported_from_domoticz, time_stamp) - txt_timestamp = 0 - if os.path.isfile(device_list_txt_filename): - txt_timestamp = os.path.getmtime(device_list_txt_filename) - self.log.logging("Database", "Log", "%s timestamp is %s" % (device_list_txt_filename, txt_timestamp)) +def is_timestamp_recent_than_filename(self, dz_timestamp, device_list_txt_filename): + # Get the timestamp of the text file if it exists, else default to 0 + txt_timestamp = os.path.getmtime(device_list_txt_filename) if os.path.isfile(device_list_txt_filename) else 0 + + # Log only when dz_timestamp is more recent than txt_timestamp if dz_timestamp > txt_timestamp: - self.log.logging("Database", "Log", "Dz is more recent than Txt Dz: %s Txt: %s" % (dz_timestamp, txt_timestamp)) + self.log.logging("Database", "Debug", f"{device_list_txt_filename} timestamp: {txt_timestamp}") + self.log.logging("Database", "Debug", f"Dz timestamp {dz_timestamp} is more recent than Txt timestamp {txt_timestamp}") return True return False -def WriteDeviceList(self, count): # sourcery skip: merge-nested-ifs - if self.HBcount < count: +def save_plugin_database(self, count): # sourcery skip: merge-nested-ifs + # In case count = -1 we request to force a write also on the Domoticz Database (when onStop is called) + # When count = 0 we force a write + + self.log.logging("Database", "Debug", "save_plugin_database %s %s" %(self.HBcount, count)) + if count != -1 and self.HBcount < count: self.HBcount = self.HBcount + 1 return - self.log.logging("Database", "Debug", "WriteDeviceList %s %s" %(self.HBcount, count)) if self.pluginconf.pluginConf["pluginData"] is None or self.DeviceListName is None: - self.log.logging("Database", "Error", "WriteDeviceList - self.pluginconf.pluginConf['pluginData']: %s , self.DeviceListName: %s" % ( + self.log.logging("Database", "Error", "save_plugin_database - self.pluginconf.pluginConf['pluginData']: %s , self.DeviceListName: %s" % ( self.pluginconf.pluginConf["pluginData"], self.DeviceListName)) return - if self.pluginconf.pluginConf["expJsonDatabase"]: - _write_DeviceList_json(self) - - _write_DeviceList_txt(self) + save_plugin_database_txt_format(self) - if ( - Modules.tools.is_domoticz_db_available(self) - and ( self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]) - ): - if _write_DeviceList_Domoticz(self) is None: + if ( count == -1 and ( self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]) ): + if save_plugin_database_into_domoticz(self) is None: # An error occured. Probably Dz.Configuration() is not available. - _write_DeviceList_txt(self) + save_plugin_database_txt_format(self) self.HBcount = 0 -def _write_DeviceList_txt(self): +def save_plugin_database_txt_format(self): # Write in classic format ( .txt ) _pluginData = Path( self.pluginconf.pluginConf["pluginData"] ) _DeviceListFileName = _pluginData / self.DeviceListName + self.log.logging("Database", "Status", f"+ Saving plugin database into {_DeviceListFileName}") try: - self.log.logging("Database", "Debug", "Write %s = %s" % (_DeviceListFileName, str(self.ListOfDevices))) + self.log.logging("Database", "Debug", "save_plugin_database_txt_format %s = %s" % (_DeviceListFileName, str(self.ListOfDevices))) with open(_DeviceListFileName, "wt", encoding='utf-8') as file: for key in self.ListOfDevices: try: file.write(key + " : " + str(self.ListOfDevices[key]) + "\n") except UnicodeEncodeError: - self.log.logging( "Database", "Error", "UnicodeEncodeError while while saving %s : %s on file" %( + self.log.logging( "Database", "Error", "save_plugin_database_txt_format UnicodeEncodeError while while saving %s : %s on file" %( key, self.ListOfDevices[key])) continue except ValueError: - self.log.logging( "Database", "Error", "ValueError while saving %s : %s on file" %( + self.log.logging( "Database", "Error", "save_plugin_database_txt_format ValueError while saving %s : %s on file" %( key, self.ListOfDevices[key])) continue except IOError: - self.log.logging( "Database", "Error", "IOError while writing to plugin Database %s" % _DeviceListFileName) + self.log.logging( "Database", "Error", "save_plugin_database_txt_format IOError while writing to plugin Database %s" % _DeviceListFileName) continue - self.log.logging("Database", "Debug", "WriteDeviceList - flush Plugin db to %s" % _DeviceListFileName) + self.log.logging("Database", "Debug", "save_plugin_database_txt_format - flush Plugin db to %s" % _DeviceListFileName) except FileNotFoundError: - self.log.logging( "Database", "Error", "WriteDeviceList - File not found >%s<" %_DeviceListFileName) + self.log.logging( "Database", "Error", "save_plugin_database_txt_format - File not found >%s<" %_DeviceListFileName) except IOError: - self.log.logging( "Database", "Error", "Error while Writing plugin Database %s" % _DeviceListFileName) + self.log.logging( "Database", "Error", "save_plugin_database_txt_format Error while Writing plugin Database %s" % _DeviceListFileName) -def _write_DeviceList_json(self): - _pluginData = Path( self.pluginconf.pluginConf["pluginData"] ) -# Incorrect error issue -# _DeviceListFileName = _pluginData / self.DeviceListName[:-3] + "json" - _DeviceListFileName = _pluginData / (self.DeviceListName[:-3] + "json") - self.log.logging("Database", "Debug", "Write %s = %s" % (_DeviceListFileName, str(self.ListOfDevices))) - with open(_DeviceListFileName, "wt") as file: - json.dump(self.ListOfDevices, file, sort_keys=True, indent=2) - self.log.logging("Database", "Debug", "WriteDeviceList - flush Plugin db to %s" % _DeviceListFileName) +def save_plugin_database_into_domoticz(self): + self.log.logging("Database", "Status", "+ Saving plugin database into Domoticz") + + plugin_dictionnary = self.ListOfDevices.copy() + self.log.logging("Database", "Log", f"save_plugin_database_into_domoticz - dumping {len(plugin_dictionnary)} entry in Domoticz Configuration" ) + + return setConfigItem( Key="ListOfDevices", Attribute="b64encoded", Value={"TimeStamp": time.time(), "b64encoded": plugin_dictionnary} ) + + +def write_coordinator_backup_domoticz(self, coordinator_backup): + self.log.logging("Database", "Log", "write_coordinator_backup_domoticz - saving coordinator data to Domoticz Sqlite Db type: %s" %type(coordinator_backup)) + self.log.logging("Database", "Status", "+ Saving Coordinator database into Domoticz") + return setConfigItem( Key="CoordinatorBackup", Attribute="b64encoded", Value={"TimeStamp": time.time(), "b64encoded": coordinator_backup } ) -def _write_DeviceList_Domoticz(self): - ListOfDevices_for_save = self.ListOfDevices.copy() - self.log.logging("Database", "Log", "WriteDeviceList - flush Plugin db to %s" % "Domoticz") - return setConfigItem( Key="ListOfDevices", Attribute="Devices", Value={"TimeStamp": time.time(), "Devices": ListOfDevices_for_save} ) + +def read_coordinator_backup_domoticz(self): + + coordinator_backup = getConfigItem(Key="CoordinatorBackup", Attribute="b64encoded") + self.log.logging( "Database", "Debug", "Coordinator backup entry from Domoticz %s" % str(coordinator_backup)) + + time_stamp = coordinator_backup["TimeStamp"] if "TimeStamp" in coordinator_backup else 0 + coordinator_network_backup = coordinator_backup["b64encoded"] if "b64encoded" in coordinator_backup else None + + self.log.logging( "Database", "Debug", "Coordinator data found on DZ with date %s" % ( + time.strftime("%A, %Y-%m-%d %H:%M:%S", time.localtime(time_stamp))), ) + + if coordinator_network_backup: + self.log.logging( "Database", "Debug", "Load from Dz: %s %s %s" % ( + len(coordinator_network_backup), str(coordinator_network_backup), type(coordinator_network_backup) )) + + return (coordinator_network_backup, time_stamp) def importDeviceConf(self): @@ -488,6 +507,7 @@ def checkDevices2LOD(self, Devices): if self.ListOfDevices[nwkid]["Status"] == "inDB": self.ListOfDevices[nwkid]["ConsistencyCheck"] = next(("ok" for dev in Devices if Devices[dev].DeviceID == self.ListOfDevices[nwkid]["IEEE"]), "not in DZ") + def checkListOfDevice2Devices(self, Devices): for widget_idx, widget_info in self.ListOfDomoticzWidget.items(): self.log.logging("Database", "Debug", f"checkListOfDevice2Devices - {widget_idx} {type(widget_idx)} - {widget_info} {type(widget_info)}") @@ -501,7 +521,7 @@ def checkListOfDevice2Devices(self, Devices): continue if device_id not in self.IEEE2NWK: - self.log.logging("Database", "Log", f"checkListOfDevice2Devices - {widget_name} not found in the plugin!") + self.log.logging("Database", "Log", f"Domoticz widget: '{widget_name}' not found in the plugin database!") continue nwkid = self.IEEE2NWK[device_id] @@ -522,13 +542,13 @@ def saveZigateNetworkData(self, nkwdata): self.log.logging("Database", "Error", "Error while writing Zigate Network Details%s" % json_filename) -def CheckDeviceList(self, key, val): +def load_plugin_database_txt_entry(self, key, val): """ This function is call during DeviceList load """ - self.log.logging("Database", "Debug", "CheckDeviceList - Address search : " + str(key), key) - self.log.logging("Database", "Debug2", "CheckDeviceList - with value : " + str(val), key) + self.log.logging("Database", "Debug", "load_plugin_database_txt_entry - Address search : " + str(key), key) + self.log.logging("Database", "Debug2", "load_plugin_database_txt_entry - with value : " + str(val), key) DeviceListVal = eval(val) # Do not load Devices in State == 'unknown' or 'left' @@ -543,7 +563,7 @@ def CheckDeviceList(self, key, val): if key in self.ListOfDevices: # Suspect - self.log.logging("Database", "Error", "CheckDeviceList - Object %s already in the plugin Db !!!" % key) + self.log.logging("Database", "Error", "load_plugin_database_txt_entry - Object %s already in the plugin Db !!!" % key) return if Modules.tools.DeviceExist(self, key, DeviceListVal.get("IEEE", "")): @@ -567,13 +587,13 @@ def CheckDeviceList(self, key, val): elif key == "0000": # Reduce the number of Attributes loaded for Zigate self.log.logging( - "Database", "Debug", "CheckDeviceList - Zigate (IEEE) = %s Load Zigate Attributes" % DeviceListVal["IEEE"] + "Database", "Debug", "load_plugin_database_txt_entry - Zigate (IEEE) = %s Load Zigate Attributes" % DeviceListVal["IEEE"] ) IMPORT_ATTRIBUTES = list(set(CIE_ATTRIBUTES)) self.log.logging("Database", "Debug", "--> Attributes loaded: %s" % IMPORT_ATTRIBUTES) else: self.log.logging( - "Database", "Debug", "CheckDeviceList - DeviceID (IEEE) = %s Load Full Attributes" % DeviceListVal["IEEE"] + "Database", "Debug", "load_plugin_database_txt_entry - DeviceID (IEEE) = %s Load Full Attributes" % DeviceListVal["IEEE"] ) IMPORT_ATTRIBUTES = list(set(MANDATORY_ATTRIBUTES + BUILD_ATTRIBUTES + MANUFACTURER_ATTRIBUTES)) @@ -603,7 +623,7 @@ def CheckDeviceList(self, key, val): self.log.logging( "Database", "Debug", - "CheckDeviceList - DeviceID (IEEE) = " + str(DeviceListVal["IEEE"]) + " for NetworkID = " + str(key), + "load_plugin_database_txt_entry - DeviceID (IEEE) = " + str(DeviceListVal["IEEE"]) + " for NetworkID = " + str(key), key, ) if DeviceListVal["IEEE"]: @@ -613,7 +633,7 @@ def CheckDeviceList(self, key, val): self.log.logging( "Database", "Log", - "CheckDeviceList - IEEE = " + str(DeviceListVal["IEEE"]) + " for NWKID = " + str(key), + "load_plugin_database_txt_entry - IEEE = " + str(DeviceListVal["IEEE"]) + " for NWKID = " + str(key), key, ) @@ -890,7 +910,7 @@ def cleanup_table_entries( self): break idx += 1 - + def profalux_fix_remote_device_model(self): for x in self.ListOfDevices: @@ -933,7 +953,7 @@ def hack_ts0601(self, nwkid): return hack_ts0601_error(self, nwkid, model_name, manufacturer=manuf_name) - + def hack_ts0601_error(self, nwkid, model, manufacturer=None): # Looks like we have a TS0601 and something wrong !!! self.log.logging("Tuya", "Error", "This device is not correctly configured, please contact us with the here after information") @@ -949,7 +969,8 @@ def hack_ts0601_rename_model( self, nwkid, modelName, manufacturer_name): if self.ListOfDevices[ nwkid ][ 'Model' ] != suggested_model: self.log.logging("Tuya", "Status", "Z4D adjusts Model name from %s to %s" %( modelName, suggested_model)) self.ListOfDevices[ nwkid ][ 'Model' ] = suggested_model - + + def cleanup_ota(self, nwkid): if "OTAUpgrade" not in self.ListOfDevices[ nwkid ]: diff --git a/Modules/domoMaj.py b/Modules/domoMaj.py index db499ae2d..523c40e79 100644 --- a/Modules/domoMaj.py +++ b/Modules/domoMaj.py @@ -1695,7 +1695,7 @@ def calculate_baro_forecast(baroValue): def baro_adjustement_value(self, Devices, NwkId, DeviceId, Device_Unit): if self.domoticzdb_DeviceStatus: try: - return round(self.domoticzdb_DeviceStatus.retreiveAddjValue_baro(domo_read_Device_Idx(self, Devices, DeviceId, Device_Unit,)), 1) + return round(self.domoticzdb_DeviceStatus.retrieve_addj_value_baro(domo_read_Device_Idx(self, Devices, DeviceId, Device_Unit,)), 1) except Exception as e: self.log.logging("Widget", "Error", "Error while trying to get Adjusted Value for Baro %s %s" % ( NwkId, e), NwkId) @@ -1704,7 +1704,7 @@ def baro_adjustement_value(self, Devices, NwkId, DeviceId, Device_Unit): def temp_adjustement_value(self, Devices, NwkId, DeviceId, Device_Unit): if self.domoticzdb_DeviceStatus: try: - return round(self.domoticzdb_DeviceStatus.retreiveAddjValue_temp(domo_read_Device_Idx(self, Devices, DeviceId, Device_Unit,)), 1) + return round(self.domoticzdb_DeviceStatus.retrieve_addj_value_temp(domo_read_Device_Idx(self, Devices, DeviceId, Device_Unit,)), 1) except Exception as e: self.log.logging("Widget", "Error", "Error while trying to get Adjusted Value for Temp %s %s" % ( NwkId, e), NwkId) diff --git a/Modules/domoTools.py b/Modules/domoTools.py index 995e0eed5..27867dd93 100644 --- a/Modules/domoTools.py +++ b/Modules/domoTools.py @@ -224,7 +224,7 @@ def retreive_reset_delays(self, nwkid): def reset_motion(self, Devices, NwkId, WidgetType, DeviceId_, Unit_, SignalLevel, BatteryLvl, ID, now, lastupdate, TimedOut): nValue, sValue = domo_read_nValue_sValue(self, Devices, DeviceId_, Unit_) - if nValue == 0 and sValue == "Off" or (now - lastupdate) < TimedOut or (self.domoticzdb_DeviceStatus and self.domoticzdb_DeviceStatus.retreiveTimeOut_Motion(ID) > 0): + if nValue == 0 and sValue == "Off" or (now - lastupdate) < TimedOut or (self.domoticzdb_DeviceStatus and self.domoticzdb_DeviceStatus.retrieve_timeout_motion(ID) > 0): return domo_update_api(self, Devices, DeviceId_, Unit_, nValue=0, sValue="Off") diff --git a/Modules/domoticzAbstractLayer.py b/Modules/domoticzAbstractLayer.py index f1f4f2ed2..b91138169 100644 --- a/Modules/domoticzAbstractLayer.py +++ b/Modules/domoticzAbstractLayer.py @@ -15,7 +15,11 @@ Description: Set of functions which abstract Domoticz Legacy and Extended framework API """ +import ast +import json import time +from base64 import b64decode, b64encode + #import DomoticzEx as Domoticz #DOMOTICZ_EXTENDED_API = True# import Domoticz as Domoticz @@ -81,9 +85,6 @@ def setConfigItem(Key=None, Attribute="", Value=None): def getConfigItem(Key=None, Attribute="", Default=None): - - domoticz_log_api("Loading %s - %s from Domoticz sqlite Db" %( Key, Attribute)) - if Default is None: Default = {} Value = Default @@ -103,16 +104,89 @@ def getConfigItem(Key=None, Attribute="", Default=None): def prepare_dict_for_storage(dict_items, Attribute): + """ + Prepares the dictionary for storage by Base64-encoding the specified attribute. - from base64 import b64encode + Args: + dict_items (dict): The dictionary containing the attribute to be encoded. + Attribute (str): The key in the dictionary to be Base64-encoded. + Returns: + dict: The modified dictionary with the specified attribute Base64-encoded and a "Version" key added. + """ if Attribute in dict_items: - dict_items[Attribute] = b64encode(str(dict_items[Attribute]).encode("utf-8")) - dict_items["Version"] = 1 + try: + # Serialize the attribute value to JSON first, then Base64 encode (Version 2) + json_str = json.dumps(dict_items[Attribute]) + + # Convert the attribute value to a string, then encode to Base64 + dict_items[Attribute] = b64encode(json_str.encode("utf-8")).decode("utf-8") + + except (TypeError, ValueError) as e: + # Log or raise an error if serialization fails + domoticz_error_api(f"Error during JSON serialization: {e} for data {dict_items[Attribute]}") + return {} + + dict_items["Version"] = 3 return dict_items def repair_dict_after_load(b64_dict, Attribute): + """ + Repairs the dictionary after loading by Base64-decoding the specified attribute. + + Args: + b64_dict (dict): The dictionary containing the Base64-encoded attribute. + Attribute (str): The key in the dictionary to be decoded. + + Returns: + dict: The modified dictionary with the specified attribute Base64-decoded. + """ + if b64_dict in ("", {}): + return {} + + if "Version" not in b64_dict: + domoticz_log_api("repair_dict_after_load - Not supported storage") + return {} + + _version = b64_dict["Version"] + if _version == 1: + return _repair_dict_after_loadV1(b64_dict, Attribute) + + elif _version == 3: + return _repair_dict_after_loadV3(b64_dict, Attribute) + + domoticz_error_api(f"repair_dict_after_load - Unknown version number: {_version} ({type(_version)})") + # Return an empty dict + return {} + + +def _repair_dict_after_loadV3(b64_dict, Attribute): + if Attribute not in b64_dict: + domoticz_error_api(f"_repair_dict_after_loadV2 - {Attribute} not found in {b64_dict} ") + return {} + + try: + # Decode the Base64-encoded attribute value + decoded_data = b64decode(b64_dict[Attribute]).decode('utf-8') + try: + # Attempt JSON decoding + b64_dict[Attribute] = json.loads(decoded_data) + + except json.JSONDecodeError: + # If it's not JSON, use literal_eval as a fallback (for Python dict-like strings) + b64_dict[Attribute] = ast.literal_eval(decoded_data) + + except (json.JSONDecodeError, ValueError) as e: + domoticz_error_api(f"Error during JSON decoding: {e}") + + except Exception as e: + domoticz_error_api(f"Unexpected error during Base64 decode: {e}") + + return b64_dict + + +def _repair_dict_after_loadV1(b64_dict, Attribute): if b64_dict in ("", {}): return {} if "Version" not in b64_dict: diff --git a/Modules/heartbeat.py b/Modules/heartbeat.py index 9a5d10a77..1df6baca9 100755 --- a/Modules/heartbeat.py +++ b/Modules/heartbeat.py @@ -88,66 +88,9 @@ PING_DEVICE_VIA_GROUPID = 3567 // HEARTBEAT # Secondes ( 59minutes et 45 secondes ) FIRST_PING_VIA_GROUP = 127 // HEARTBEAT +# Retry intervals: Retry #1 (30s), Retry #2 (120s), Retry #3 (300s) +PING_RETRY_INTERVALS = [25, 115, 295] -#def attributeDiscovery(self, NwkId): -# -# rescheduleAction = False -# # If Attributes not yet discovered, let's do it -# -# if "ConfigSource" not in self.ListOfDevices[NwkId]: -# return False -# -# if self.ListOfDevices[NwkId]["ConfigSource"] == "DeviceConf": -# return False -# -# if "Attributes List" in self.ListOfDevices[NwkId] and len(self.ListOfDevices[NwkId]["Attributes List"]) > 0: -# return False -# -# if "Attributes List" not in self.ListOfDevices[NwkId]: -# self.ListOfDevices[NwkId]["Attributes List"] = {'Ep': {}} -# if "Request" not in self.ListOfDevices[NwkId]["Attributes List"]: -# self.ListOfDevices[NwkId]["Attributes List"]["Request"] = {} -# -# for iterEp in list(self.ListOfDevices[NwkId]["Ep"]): -# if iterEp == "ClusterType": -# continue -# if iterEp not in self.ListOfDevices[NwkId]["Attributes List"]["Request"]: -# self.ListOfDevices[NwkId]["Attributes List"]["Request"][iterEp] = {} -# -# for iterCluster in list(self.ListOfDevices[NwkId]["Ep"][iterEp]): -# if iterCluster in ("Type", "ClusterType", "ColorMode"): -# continue -# if iterCluster not in self.ListOfDevices[NwkId]["Attributes List"]["Request"][iterEp]: -# self.ListOfDevices[NwkId]["Attributes List"]["Request"][iterEp][iterCluster] = 0 -# -# if self.ListOfDevices[NwkId]["Attributes List"]["Request"][iterEp][iterCluster] != 0: -# continue -# -# if not self.busy and self.ControllerLink.loadTransmit() <= MAX_LOAD_ZIGATE: -# if int(iterCluster, 16) < 0x0FFF: -# getListofAttribute(self, NwkId, iterEp, iterCluster) -# # getListofAttributeExtendedInfos(self, NwkId, EpOut, cluster, start_attribute=None, manuf_specific=None, manuf_code=None) -# elif ( -# "Manufacturer" in self.ListOfDevices[NwkId] -# and len(self.ListOfDevices[NwkId]["Manufacturer"]) == 4 -# and is_hex(self.ListOfDevices[NwkId]["Manufacturer"]) -# ): -# getListofAttribute( -# self, -# NwkId, -# iterEp, -# iterCluster, -# manuf_specific="01", -# manuf_code=self.ListOfDevices[NwkId]["Manufacturer"], -# ) -# # getListofAttributeExtendedInfos(self, NwkId, EpOut, cluster, start_attribute=None, manuf_specific=None, manuf_code=None) -# -# self.ListOfDevices[NwkId]["Attributes List"]["Request"][iterEp][iterCluster] = time.time() -# -# else: -# rescheduleAction = True -# -# return rescheduleAction def attributeDiscovery(self, NwkId): # If Attributes not yet discovered, let's do it @@ -424,199 +367,255 @@ def pollingDeviceStatus(self, NwkId): return False -def checkHealth(self, NwkId): +def is_check_device_health_not_needed(self, NwkId): + """ + Check and update the health status of a device in the network. - # Checking current state of the this Nwk - if "Health" not in self.ListOfDevices[NwkId]: - self.ListOfDevices[NwkId]["Health"] = "" - - if self.ListOfDevices[NwkId]["Health"] == "Disabled": + Args: + NwkId (str): The network ID of the device to check. + + Returns: + bool: False if a check is required, True otherwise. + + This method: + - Ensures the "Health" and "Stamp" fields are initialized for the device. + - Updates the device's health to "unknown" if it lacks a valid health status. + - Checks if a "Live" device has been inactive for more than 21,200 seconds and flags it as "Not seen last 24hours". + - Logs a message when a "Live" device appears to be out of the network. + """ + # Retrieve the device or initialize as an empty dictionary + device = self.ListOfDevices.get(NwkId, {}) + health = device.setdefault("Health", "") + + # Initialize Health and Stamp fields if missing + if "Stamp" not in device or "LastSeen" not in device["Stamp"]: + device["Health"] = "unknown" + + stamp = device.setdefault("Stamp", {"LastPing": 0, "LastSeen": 0}) + stamp.setdefault("LastSeen", 0) + + if health not in {"Disabled", "Live", "Not Reachable", "Not seen last 24hours"}: return False - - if "Stamp" not in self.ListOfDevices[NwkId]: - self.ListOfDevices[NwkId]["Stamp"] = {'LastPing': 0, 'LastSeen': 0} - self.ListOfDevices[NwkId]["Health"] = "unknown" - if "LastSeen" not in self.ListOfDevices[NwkId]["Stamp"]: - self.ListOfDevices[NwkId]["Stamp"]["LastSeen"] = 0 - self.ListOfDevices[NwkId]["Health"] = "unknown" + # Check if device is live but hasn't been seen recently + current_time = int(time.time()) + if device["Health"] == "Live" and current_time > ( stamp["LastSeen"] + 3600 * 12): + # That is 12 hours we didn't receive any message from the device which was Live! + z_device_name = device.get("ZDeviceName", f"NwkId: {NwkId}") + self.log.logging( + "PingDevices", + "Debug", + f"Device Health - {z_device_name}, IEEE: {device.get('IEEE')}, Model: {device.get('Model')} seems to be out of the network" + ) + device["Health"] = "Not seen last 24hours" - if ( - int(time.time()) > (self.ListOfDevices[NwkId]["Stamp"]["LastSeen"] + 21200) - and self.ListOfDevices[NwkId]["Health"] == "Live" - ): - if "ZDeviceName" in self.ListOfDevices[NwkId]: - self.log.logging("Heartbeat", "Debug", "Device Health - %s NwkId: %s,Ieee: %s , Model: %s seems to be out of the network" % ( - self.ListOfDevices[NwkId]["ZDeviceName"], NwkId, self.ListOfDevices[NwkId]["IEEE"], self.ListOfDevices[NwkId]["Model"],)) - else: - self.log.logging("Heartbeat", "Debug", "Device Health - NwkId: %s,Ieee: %s , Model: %s seems to be out of the network" % ( - NwkId, self.ListOfDevices[NwkId]["IEEE"], self.ListOfDevices[NwkId]["Model"]) ) - self.ListOfDevices[NwkId]["Health"] = "Not seen last 24hours" + # Return whether the device is not flagged as Not Reachable + return device["Health"] != "Not Reachable" - # If device flag as Not Reachable, don't do anything - return ( "Health" not in self.ListOfDevices[NwkId] or self.ListOfDevices[NwkId]["Health"] != "Not Reachable") +def retry_ping_device_in_bad_health(self, NwkId): + """ + Handle retry logic for pinging a device with bad health status. -def pingRetryDueToBadHealth(self, NwkId): + Args: + NwkId (str): The network ID of the device to check. + This method: + - Initializes the "pingDeviceRetry" structure if missing. + - Resets retry logic for devices with outdated or missing timestamp data. + - Performs up to three retries (at increasing intervals) if the device is in a bad health state. + - Logs details of each retry attempt and calls ping-related methods as needed. + """ now = int(time.time()) - # device is on Non Reachable state - self.log.logging("Heartbeat", "Debug", "--------> ping Retry Check %s" % NwkId, NwkId) - if "pingDeviceRetry" not in self.ListOfDevices[NwkId]: - self.ListOfDevices[NwkId]["pingDeviceRetry"] = {"Retry": 0, "TimeStamp": now} - if self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] == 0: - return + self.log.logging("Heartbeat", "Debug", f"--------> retry_ping_device_in_bad_health - ping Retry Check {NwkId}", NwkId) + + # Initialize or reset "pingDeviceRetry" if missing + retry_info = self.ListOfDevices.setdefault(NwkId, {}).setdefault( "pingDeviceRetry", {"Retry": 0, "TimeStamp": now} ) + retry = retry_info["Retry"] + last_timestamp = retry_info.get("TimeStamp", now) + + # Reset retry info for devices missing a timestamp (legacy data handling) + if retry > 0 and "TimeStamp" not in retry_info: + retry_info["Retry"] = 0 + retry_info["TimeStamp"] = now + + # Log current retry details + self.log.logging( "PingDevices", "Debug", f"--------> retry_ping_device_in_bad_health - ping Retry Check {NwkId} Retry: {retry} Gap: {now - last_timestamp}", NwkId, ) + + if retry < len(PING_RETRY_INTERVALS): + interval = PING_RETRY_INTERVALS[retry] + if (self.ControllerLink.loadTransmit() == 0) and now > (last_timestamp + interval): + ping_while_in_bad_health(self, retry, NwkId, retry_info, now) + - if "Retry" in self.ListOfDevices[NwkId]["pingDeviceRetry"] and "TimeStamp" not in self.ListOfDevices[NwkId]["pingDeviceRetry"]: - # This could be due to a previous version without TimeStamp - self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] = 0 - self.ListOfDevices[NwkId]["pingDeviceRetry"]["TimeStamp"] = now +def ping_while_in_bad_health(self, retry, NwkId, retry_info, now): + """ + Handle device ping attempts when the device is in bad health. - lastTimeStamp = self.ListOfDevices[NwkId]["pingDeviceRetry"]["TimeStamp"] - retry = self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] + Args: + retry (int): The current retry count. + NwkId (str): The network ID of the device. + retry_info (dict): A dictionary containing retry-related information. + now (int): The current timestamp. + This function logs the retry attempt, updates the retry information, + and sends a ping to the device after attempting a network address request. + """ + # Log the retry attempt self.log.logging( - "Heartbeat", + "PingDevices", "Debug", - "--------> ping Retry Check %s Retry: %s Gap: %s" % (NwkId, retry, now - lastTimeStamp), - NwkId, + f"--------> ping_while_in_bad_health - Ping Retry {retry + 1} for Device {NwkId}", + NwkId ) - # Retry #1 - if ( - retry == 0 - and self.ControllerLink.loadTransmit() == 0 - and now > (lastTimeStamp + 30) - ): # 30s - self.log.logging("Heartbeat", "Debug", "--------> ping Retry 1 Check %s" % NwkId, NwkId) - self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] += 1 - self.ListOfDevices[NwkId]["pingDeviceRetry"]["TimeStamp"] = now - lookup_ieee = self.ListOfDevices[ NwkId ]['IEEE'] - zdp_NWK_address_request(self, "0000", lookup_ieee) - submitPing(self, NwkId) - return - # Retry #2 - if ( - retry == 1 - and self.ControllerLink.loadTransmit() == 0 - and now > (lastTimeStamp + 120) - ): # 30 + 120s - # Let's retry - self.log.logging("Heartbeat", "Debug", "--------> ping Retry 2 Check %s" % NwkId, NwkId) - self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] += 1 - self.ListOfDevices[NwkId]["pingDeviceRetry"]["TimeStamp"] = now - lookup_ieee = self.ListOfDevices[ NwkId ]['IEEE'] - zdp_NWK_address_request(self, "fffd", lookup_ieee) - submitPing(self, NwkId) - return + # Update retry information + retry_info["Retry"] += 1 + retry_info["TimeStamp"] = now - # Retry #3 - if ( - retry == 2 - and self.ControllerLink.loadTransmit() == 0 - and now > (lastTimeStamp + 300) - ): # 30 + 120 + 300 - # Let's retry - self.log.logging("Heartbeat", "Debug", "--------> ping Retry 3 (last) Check %s" % NwkId, NwkId) - self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"] += 1 - self.ListOfDevices[NwkId]["pingDeviceRetry"]["TimeStamp"] = now - lookup_ieee = self.ListOfDevices[ NwkId ]['IEEE'] - zdp_NWK_address_request(self, "FFFD", lookup_ieee) - submitPing(self, NwkId) + # Get the IEEE address of the device + lookup_ieee = self.ListOfDevices[NwkId]["IEEE"] + + # Perform a network address request, using a broadcast or direct address + target_address = "FFFD" if retry > 0 else "0000" + zdp_NWK_address_request(self, target_address, lookup_ieee) + + # Send a ping to the device + send_ping_to_device(self, NwkId) -def pingDevices(self, NwkId, health, checkHealthFlag, mainPowerFlag): +def ping_devices(self, NwkId, health, checkHealthFlag, mainPowerFlag): + """ + Manage the pinging of devices based on various conditions and configurations. + Args: + NwkId (str): The network ID of the device to ping. + health (bool): The health status of the device. + checkHealthFlag (bool): Flag indicating whether to perform a health check before pinging. + mainPowerFlag (bool): Flag indicating whether the device has main power. + + This method: + - Skips pinging if group ping is enabled. + - Logs device and retry information. + - Skips pinging based on main power, TuyaPing, or blacklist configurations. + - Avoids unnecessary pings for recently active devices. + - Handles retry logic for unhealthy devices. + - Pings devices if all conditions are met. + """ + # Check for group ping configuration if self.pluginconf.pluginConf["pingViaGroup"]: - self.log.logging( "Heartbeat", "Debug", "No direct pinDevices as Group ping is enabled" , NwkId, ) + self.log.logging("PingDevices", "Debug", "No direct ping_devices as Group ping is enabled", NwkId) return - - if "pingDeviceRetry" in self.ListOfDevices[NwkId]: - self.log.logging( "Heartbeat", "Debug", "------> pinDevices %s health: %s, checkHealth: %s, mainPower: %s, retry: %s" % ( - NwkId, health, checkHealthFlag, mainPowerFlag, self.ListOfDevices[NwkId]["pingDeviceRetry"]["Retry"]), NwkId, ) - else: - self.log.logging( "Heartbeat", "Debug", "------> pinDevices %s health: %s, checkHealth: %s, mainPower: %s" % ( - NwkId, health, checkHealthFlag, mainPowerFlag), NwkId, ) + # Log device and retry information + retry_info = self.ListOfDevices.get(NwkId, {}).get("pingDeviceRetry", {}) + retry = retry_info.get("Retry", "N/A") + self.log.logging( + "PingDevices", + "Debug", + f"------> ping_devices {NwkId} health: {health}, is_check_device_health_not_needed: {checkHealthFlag}, " + f"mainPower: {mainPowerFlag}, retry: {retry}", + NwkId, + ) + + # Skip if the device lacks main power if not mainPowerFlag: return - if ( - "Param" in self.ListOfDevices[NwkId] - and "TuyaPing" in self.ListOfDevices[NwkId]["Param"] - and int(self.ListOfDevices[NwkId]["Param"]["TuyaPing"]) == 1 - ): + # Skip based on TuyaPing configuration + params = self.ListOfDevices[NwkId].get("Param", {}) + if int(params.get("TuyaPing", 0)) == 1: self.log.logging( - "Heartbeat", + "PingDevices", "Debug", - "------> pingDevice disabled for %s as TuyaPing enabled %s" - % ( - NwkId, - self.ListOfDevices[NwkId]["Param"]["TuyaPing"], - ), + f"------> ping_devices disabled for {NwkId} as TuyaPing is enabled", NwkId, ) return - if ( - "Param" in self.ListOfDevices[NwkId] - and "pingBlackListed" in self.ListOfDevices[NwkId]["Param"] - and int(self.ListOfDevices[NwkId]["Param"]["pingBlackListed"]) == 1 - ): + # Skip based on ping blacklist configuration + if int(params.get("pingBlackListed", 0)) == 1: self.log.logging( - "Heartbeat", + "PingDevices", "Debug", - "------> pingDevice disabled for %s as pingBlackListed enabled %s" - % ( - NwkId, - self.ListOfDevices[NwkId]["Param"]["pingBlackListed"], - ), + f"------> ping_devices disabled for {NwkId} as pingBlackListed is enabled", NwkId, ) return now = int(time.time()) + stamp = self.ListOfDevices[NwkId].get("Stamp", {}) + plugin_ping_frequency = self.pluginconf.pluginConf["pingDevicesFeq"] + last_message_time = stamp.get("time", 0) - if ( - "time" in self.ListOfDevices[NwkId]["Stamp"] - and now < self.ListOfDevices[NwkId]["Stamp"]["time"] + self.pluginconf.pluginConf["pingDevicesFeq"] - ): - # If we have received a message since less than 1 hours, then no ping to be done ! - self.log.logging("Heartbeat", "Debug", "------> %s no need to ping as we received a message recently " % (NwkId,), NwkId) + # Skip if a message was received recently + if now < (last_message_time + plugin_ping_frequency): + self.log.logging( + "PingDevices", + "Debug", + f"------> ping_devices {NwkId} no need to ping as we received a message recently", + NwkId, + ) return + # Handle retry logic for unhealthy devices if not health: - pingRetryDueToBadHealth(self, NwkId) + retry_ping_device_in_bad_health(self, NwkId) return - if "LastPing" not in self.ListOfDevices[NwkId]["Stamp"]: - self.ListOfDevices[NwkId]["Stamp"]["LastPing"] = 0 - lastPing = self.ListOfDevices[NwkId]["Stamp"]["LastPing"] - lastSeen = self.ListOfDevices[NwkId]["Stamp"]["LastSeen"] - if checkHealthFlag and now > (lastPing + 60) and self.ControllerLink.loadTransmit() == 0: - submitPing(self, NwkId) + # Initialize LastPing if missing + last_ping = stamp.setdefault("LastPing", 0) + last_seen = stamp.get("LastSeen", 0) + + # Check and perform health ping if applicable + if checkHealthFlag and now > (last_ping + 60) and self.ControllerLink.loadTransmit() == 0: + send_ping_to_device(self, NwkId) return - self.log.logging( "Heartbeat", "Debug", "------> pinDevice %s time: %s LastPing: %s LastSeen: %s Freq: %s" % ( - NwkId, now, lastPing, lastSeen, self.pluginconf.pluginConf["pingDevicesFeq"]), NwkId, ) - if ( - (now > (lastPing + self.pluginconf.pluginConf["pingDevicesFeq"])) - and (now > (lastSeen + self.pluginconf.pluginConf["pingDevicesFeq"])) - and self.ControllerLink.loadTransmit() == 0 - ): + # Log details before the final ping decision + self.log.logging( + "PingDevices", + "Debug", + f"------> ping_devices {NwkId} time: {now}, LastPing: {last_ping}, " + f"LastSeen: {last_seen}, Freq: {plugin_ping_frequency}", + NwkId, + ) + + # Perform ping based on time and frequency + perform_ping = ( + (now > (last_ping + plugin_ping_frequency)) + and (now > (last_seen + plugin_ping_frequency)) + and (self.ControllerLink.loadTransmit() == 0) + ) - self.log.logging( "Heartbeat", "Debug", "------> pinDevice %s time: %s LastPing: %s LastSeen: %s Freq: %s" % ( - NwkId, now, lastPing, lastSeen, self.pluginconf.pluginConf["pingDevicesFeq"]), NwkId, ) + if perform_ping: + self.log.logging( + "PingDevices", + "Debug", + f"------> ping_devices {NwkId} time: {now}, LastPing: {last_ping}, " + f"LastSeen: {last_seen}, Freq: {plugin_ping_frequency}", + NwkId, + ) + send_ping_to_device(self, NwkId) - submitPing(self, NwkId) +def send_ping_to_device(self, NwkId): + """ + Send a ping to a device to verify its connectivity and update the last ping timestamp. -def submitPing(self, NwkId): - # Pinging devices to check they are still Alive - self.log.logging("Heartbeat", "Debug", "------------> call readAttributeRequest %s" % NwkId, NwkId) + Args: + NwkId (str): The network ID of the device to ping. + + This method: + - Logs the initiation of a ping operation. + - Updates the "LastPing" timestamp in the device's record. + - Calls the `ping_device_with_read_attribute` method to perform the actual ping operation. + """ + self.log.logging("PingDevices", "Debug", f"------------> send_ping_to_device - call readAttributeRequest {NwkId}", NwkId) self.ListOfDevices[NwkId]["Stamp"]["LastPing"] = int(time.time()) ping_device_with_read_attribute(self, NwkId) + def hr_process_device(self, Devices, NwkId): # Begin # Normalize Hearbeat value if needed @@ -630,11 +629,11 @@ def hr_process_device(self, Devices, NwkId): # Check if this is a Main powered device or Not. Source of information are: MacCapa and PowerSource _mainPowered = mainPoweredDevice(self, NwkId) _checkHealth = self.ListOfDevices[NwkId]["Health"] == "" - health = checkHealth(self, NwkId) + health = is_check_device_health_not_needed(self, NwkId) # Pinging devices to check they are still Alive - if self.pluginconf.pluginConf["pingDevices"]: - pingDevices(self, NwkId, health, _checkHealth, _mainPowered) + if self.pluginconf.pluginConf["CheckDeviceHealth"]: + ping_devices(self, NwkId, health, _checkHealth, _mainPowered) # Check if we are in the process of provisioning a new device. If so, just stop if self.CommiSSionning: @@ -779,54 +778,6 @@ def clear_last_polling_data(self, NwkId): self.ListOfDevices[NwkId].pop(key, None) -#def process_read_attributes(self, NwkId, model): -# self.log.logging( "Heartbeat", "Debug", f"process_read_attributes - for {NwkId} {model}") -# process_next_ep_later = False -# now = int(time.time()) # Will be used to trigger ReadAttributes -# -# device_infos = self.ListOfDevices[NwkId] -# for ep in device_infos["Ep"]: -# if ep == "ClusterType": -# continue -# -# if model == "lumi.ctrl_neutral1" and ep != "02" : # All Eps other than '02' are blacklisted -# continue -# -# if model == "lumi.ctrl_neutral2" and ep not in ("02", "03"): -# continue -# -# for Cluster in READ_ATTRIBUTES_REQUEST: -# # We process ALL available clusters for a particular EndPoint -# -# if ( Cluster not in READ_ATTRIBUTES_REQUEST or Cluster not in device_infos["Ep"][ep] ): -# continue -# -# if self.busy or self.ControllerLink.loadTransmit() > MAX_LOAD_ZIGATE: -# self.log.logging( "Heartbeat", "Debug", "process_read_attributes - %s skip ReadAttribute for now ... system too busy (%s/%s)" % ( -# NwkId, self.busy, self.ControllerLink.loadTransmit()), NwkId, ) -# process_next_ep_later = True -# -# if READ_ATTRIBUTES_REQUEST[Cluster][1] in self.pluginconf.pluginConf: -# timing = self.pluginconf.pluginConf[READ_ATTRIBUTES_REQUEST[Cluster][1]] -# else: -# self.log.logging( "Heartbeat", "Error", "proprocess_read_attributescessKnownDevices - missing timing attribute for Cluster: %s - %s" % ( -# Cluster, READ_ATTRIBUTES_REQUEST[Cluster][1]), NwkId ) -# continue -# -# # Let's check the timing -# if not is_time_to_perform_work(self, "ReadAttributes", NwkId, ep, Cluster, now, timing): -# continue -# -# self.log.logging( "Heartbeat", "Debug", "process_read_attributes - %s/%s and time to request ReadAttribute for %s" % ( -# NwkId, ep, Cluster), NwkId, ) -# -# func = READ_ATTRIBUTES_REQUEST[Cluster][0] -# func(self, NwkId) -# -# if process_next_ep_later: -# return True -# return False - def process_read_attributes(self, NwkId, model): self.log.logging("Heartbeat", "Debug", f"process_read_attributes - for {NwkId} {model}") now = int(time.time()) diff --git a/Modules/matomo_request.py b/Modules/matomo_request.py index ba0798fe8..f5b7577d7 100644 --- a/Modules/matomo_request.py +++ b/Modules/matomo_request.py @@ -20,6 +20,7 @@ from Modules.tools import how_many_devices import distro import requests +import socket # Matomo endpoint details MATOMO_URL = "https://z4d.pipiche.net/matomo.php" @@ -123,6 +124,10 @@ def matomo_coordinator_initialisation(self): send_matomo_request( self, action_name="Coordinator Action", event_category="Coordinator", event_action="NewNetwork", event_name="Coordinator Formed new network" ) +def matomo_coordinator_restart_after_error(self): + send_matomo_request( self, action_name="Coordinator Action", event_category="Coordinator", event_action="RestartAfterError", event_name="Coordinator Restart after commmunication error" ) + + def matomo_plugin_shutdown(self): send_matomo_request( self, action_name="Plugin Action", event_category="Plugin", event_action="Shutdown", event_name="Plugin Shutdown" ) @@ -154,7 +159,11 @@ def send_matomo_request(self, action_name, custom_variable=None, custom_dimensio event_action (str, optional): Action for the event (e.g., "Opt-Out"). event_name (str, optional): Name of the event (e.g., "User Opted Out"). """ - + + if socket.gethostname() == 'rasp': + self.log.logging( "Matomo", "Error", f"send_matomo_request - Development system, nothing to send to matomo {socket.gethostname()}") + return + client_id = get_clientid(self) self.log.logging( "Matomo", "Debug", f"send_matomo_request - Clien_id {client_id}") if client_id is None: diff --git a/Modules/tools.py b/Modules/tools.py index 4ced7cdde..0919313ca 100644 --- a/Modules/tools.py +++ b/Modules/tools.py @@ -22,7 +22,7 @@ import time from pathlib import Path -from Modules.database import WriteDeviceList +from Modules.database import save_plugin_database from Modules.pluginDbAttributes import STORE_CONFIGURE_REPORTING from Modules.zigateConsts import HEARTBEAT from Modules.domoticzAbstractLayer import domo_read_Device_Idx, domo_read_Name @@ -309,7 +309,7 @@ def reconnectNWkDevice(self, new_NwkId, IEEE, old_NwkId): del self.ListOfDevices[new_NwkId][STORE_CONFIGURE_REPORTING] self.ListOfDevices[new_NwkId]["Heartbeat"] = "0" - WriteDeviceList(self, 0) + save_plugin_database(self, 0) self.log.logging("PluginTools", "Status", "NetworkID: %s is replacing %s for object: %s" % (new_NwkId, old_NwkId, IEEE)) return True @@ -425,15 +425,31 @@ def initDeviceInList(self, Nwkid): def timeStamped(self, key, Type): + """ + Updates the timestamp information for a device. + + Args: + key (str): The unique identifier (key) for the device in `ListOfDevices`. + Type (int): The message type or event type to be logged. + + This method: + - Ensures the device exists in `ListOfDevices` before updating. + - Initializes the "Stamp" field if it is missing. + - Updates the current time in both UNIX timestamp and formatted string formats. + - Logs the message type in hexadecimal format. + """ + # Ensure the device exists in ListOfDevices if key not in self.ListOfDevices: return - if "Stamp" not in self.ListOfDevices[key]: - self.ListOfDevices[key]["Stamp"] = {"LasteSeen": {}, "Time": {}, "MsgType": {}} - self.ListOfDevices[key]["Stamp"]["time"] = time.time() - self.ListOfDevices[key]["Stamp"]["Time"] = datetime.datetime.fromtimestamp(time.time()).strftime( - "%Y-%m-%d %H:%M:%S" - ) - self.ListOfDevices[key]["Stamp"]["MsgType"] = "%4x" % (Type) + + # Initialize the "Stamp" dictionary if not present + stamp = self.ListOfDevices[key].setdefault("Stamp", {"LastSeen": {}, "Time": {}, "MsgType": {}}) + + # Update the timestamp and message type + current_time = time.time() + stamp["time"] = current_time + stamp["Time"] = datetime.datetime.fromtimestamp(current_time).strftime("%Y-%m-%d %H:%M:%S") + stamp["MsgType"] = f"{Type:04x}" # Used by zcl/zdpRawCommands @@ -471,28 +487,38 @@ def updSQN(self, key, newSQN): def updLQI(self, key, LQI): + """ + Update the Link Quality Indicator (LQI) for a device. + Args: + key (str): The unique identifier of the device. + LQI (str): The LQI value in hexadecimal format. + + This function ensures the LQI value is updated and maintains a rolling history + of the last 10 LQI values for the device. + """ + # Ensure the device exists in the list if key not in self.ListOfDevices: return - if "LQI" not in self.ListOfDevices[key]: - self.ListOfDevices[key]["LQI"] = {} + # Initialize LQI fields if not present + device_info = self.ListOfDevices[key] + device_info.setdefault("LQI", {}) + device_info.setdefault("RollingLQI", []) - if LQI == "00": + # Skip invalid LQI values + if LQI == "00" or not is_hex(LQI): return - if is_hex(LQI): # Check if the LQI is Correct + # Convert the LQI to an integer and update + lqi_value = int(LQI, 16) + device_info["LQI"] = lqi_value - self.ListOfDevices[key]["LQI"] = int(LQI, 16) - - if "RollingLQI" not in self.ListOfDevices[key]: - self.ListOfDevices[key]["RollingLQI"] = [] - - if len(self.ListOfDevices[key]["RollingLQI"]) > 10: - del self.ListOfDevices[key]["RollingLQI"][0] - self.ListOfDevices[key]["RollingLQI"].append(int(LQI, 16)) - - return + # Maintain a rolling history of up to 10 LQI values + rolling_lqi = device_info["RollingLQI"] + if len(rolling_lqi) >= 10: + rolling_lqi.pop(0) # Remove the oldest value + rolling_lqi.append(lqi_value) def upd_RSSI(self, nwkid, rssi_value): @@ -1498,7 +1524,23 @@ def print_stack( self ): def helper_copyfile(source, dest, move=True): + """ + Copy or move a file from a source path to a destination path. + + This function uses the `shutil` module to move or copy files. If `shutil.move` + or `shutil.copy` fails (for example, if the file types are incompatible), it + attempts to perform a line-by-line copy. + Args: + source (str): Path to the source file. + dest (str): Path to the destination file or directory. + move (bool, optional): If True, the file is moved (deleted from source after copying). + If False, the file is copied. Defaults to True. + + Raises: + Exception: Any exception raised by `shutil.move` or `shutil.copy` that cannot be handled + by the line-by-line copy fallback will propagate up. + """ try: import shutil @@ -1513,7 +1555,25 @@ def helper_copyfile(source, dest, move=True): def helper_versionFile(source, nbversion): - + """ + Manage versioning for a file by creating sequentially numbered backups. + + This function creates versioned copies of a given file by appending incremental + numbers to the filename. If `nbversion` is greater than 1, it shifts previous + versions by incrementing their version numbers before creating a new version. + The newest version is always `source-01`. + + Args: + source (str): Path to the source file to be versioned. + nbversion (int): Number of versions to maintain. If `nbversion` is 0, + no action is taken. If 1, only a single version (`source-01`) + is created. For values greater than 1, each prior version is + shifted up by 1, deleting the oldest if `nbversion` limit + is exceeded + + Returns: + None + """ source = str(source) if nbversion == 0: return diff --git a/Modules/zigpyBackup.py b/Modules/zigpyBackup.py index b3e8a11cf..7db94fe33 100644 --- a/Modules/zigpyBackup.py +++ b/Modules/zigpyBackup.py @@ -15,6 +15,9 @@ from pathlib import Path import Modules.tools +from Modules.database import (is_timestamp_recent_than_filename, + read_coordinator_backup_domoticz, + write_coordinator_backup_domoticz) def handle_zigpy_backup(self, backup): @@ -25,34 +28,69 @@ def handle_zigpy_backup(self, backup): _pluginData = Path( self.pluginconf.pluginConf["pluginData"] ) _coordinator_backup = _pluginData / ("Coordinator-%02d.backup" %self.HardwareID ) + self.log.logging("TransportZigpy", "Debug", "Backups: %s" %backup) if os.path.exists(_coordinator_backup): Modules.tools.helper_versionFile(_coordinator_backup, self.pluginconf.pluginConf["numDeviceListVersion"]) try: + self.log.logging("Database", "Status", f"+ Saving Coordinator database into {_coordinator_backup}") with open(_coordinator_backup, "wt") as file: file.write(json.dumps((backup.as_dict()))) - self.log.logging("TransportZigpy", "Status", "Coordinator backup is available: %s" %_coordinator_backup) + self.log.logging("TransportZigpy", "Debug", "Coordinator backup is available: %s" %_coordinator_backup) except IOError: self.log.logging("TransportZigpy", "Error", "Error while Writing Coordinator backup %s" % _coordinator_backup) + if self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]: + write_coordinator_backup_domoticz(self, json.dumps((backup.as_dict())) ) + def handle_zigpy_retreive_last_backup( self ): - - # Return the last backup + """ Return the last coordinator backup from txt or domoticz""" + _pluginData = Path( self.pluginconf.pluginConf["pluginData"] ) _coordinator_backup = _pluginData / ("Coordinator-%02d.backup" %self.HardwareID) - if not os.path.exists(_coordinator_backup): - return None - - with open(_coordinator_backup, "r") as _coordinator: - self.log.logging("TransportZigpy", "Debug", "Open : %s" % _coordinator_backup) - try: - return json.load(_coordinator) - except json.JSONDecodeError: - return None - except Exception: - return None - return None \ No newline at end of file + + file_latest_coordinator_backup_record = None + + # Retreive the coordinator backup from Text file + if os.path.exists(_coordinator_backup): + with open(_coordinator_backup, "r") as _coordinator: + self.log.logging("TransportZigpy", "Debug", "Open : %s" % _coordinator_backup) + loaded_from = _coordinator_backup + try: + file_latest_coordinator_backup_record = json.load(_coordinator) + except (json.JSONDecodeError, Exception): + file_latest_coordinator_backup_record = None + + # Retreive the coordinator backup from Domoticz Configuration record + if (self.pluginconf.pluginConf["useDomoticzDatabase"] or self.pluginconf.pluginConf["storeDomoticzDatabase"]): + latest_coordinator_backup = read_coordinator_backup_domoticz(self) + self.log.logging("TransportZigpy", "Debug", "handle_zigpy_retreive_last_backup - Retreive latest_coordinator_backup %s (%s)" %( + str(latest_coordinator_backup), type(latest_coordinator_backup))) + + dz_latest_coordinator_backup_record, dz_latest_coordinator_backup_timestamp = latest_coordinator_backup + backup_domoticz_more_recent = is_timestamp_recent_than_filename(self, dz_latest_coordinator_backup_timestamp, _coordinator_backup ) if os.path.exists(_coordinator_backup) else True + + if isinstance(dz_latest_coordinator_backup_record, str): + dz_latest_coordinator_backup_record = json.loads(dz_latest_coordinator_backup_record) + + self.log.logging("TransportZigpy", "Debug", "handle_zigpy_retreive_last_backup - Retreive latest Coordinator data from Domoticz : (%s) %s" %( + type(dz_latest_coordinator_backup_record),dz_latest_coordinator_backup_record)) + + self.log.logging( "Database", "Debug", "Coordinator Backup from Domoticz is recent: %s " % ( + is_timestamp_recent_than_filename(self, dz_latest_coordinator_backup_timestamp, _coordinator_backup) )) + + if file_latest_coordinator_backup_record != dz_latest_coordinator_backup_record: + self.log.logging("TransportZigpy", "Error", f"==> Sanity check : Domoticz Coordinator Backup versus File Backup NOT equal!! Domoticz: {dz_latest_coordinator_backup_record} {_coordinator_backup}: {file_latest_coordinator_backup_record}") + + # At that stage, we have loaded from Domoticz and from txt file. + if dz_latest_coordinator_backup_record and self.pluginconf.pluginConf["useDomoticzDatabase"] and backup_domoticz_more_recent: + # We will use the Domoticz import. + self.log.logging("Database", "Status", "Z4D loads coordinator backup from Domoticz") + return dz_latest_coordinator_backup_record + + self.log.logging("Database", "Status", f"Z4D loads coordinator backup from {_coordinator_backup}") + return file_latest_coordinator_backup_record diff --git a/OTAFirmware/LEGRAND/README.md b/OTAFirmware/LEGRAND/README.md index 2d103ef0d..66b910a3e 100644 --- a/OTAFirmware/LEGRAND/README.md +++ b/OTAFirmware/LEGRAND/README.md @@ -1,7 +1,8 @@ Legrand Netatmo provides the latest Zigbee 3.0 firmware available. -https://developer.legrand.com/documentation/operating-manual/ -https://developer.legrand.com/documentation/firmwares-download/ +https://developer.legrand.com/production-firmware-download/ NLM -> Microdule NLL -> Light switch with Neutral +NLF -> Dimmer switch w/o neutral +NLV -> Shutter switch with neutral diff --git a/Tools/check_and_remove_duplicates.py b/Tools/check_and_remove_duplicates.py new file mode 100644 index 000000000..00cae07a8 --- /dev/null +++ b/Tools/check_and_remove_duplicates.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + +import argparse +import importlib.metadata +import os +import shutil + + +def find_duplicates(directory): + """ + Find and list all duplicate versions of packages in the specified directory. + + Args: + directory (str): The directory containing the Python modules. + + Returns: + dict: A dictionary where the keys are package names and the values are lists of duplicate versions. + """ + installed_packages = {} + for dist in importlib.metadata.distributions(path=[directory]): + package_name = dist.metadata['Name'] + package_version = dist.version + if package_name in installed_packages: + installed_packages[package_name].append(package_version) + else: + installed_packages[package_name] = [package_version] + + return { + pkg: versions + for pkg, versions in installed_packages.items() + if len(versions) > 1 + } + +def remove_duplicates(directory, duplicates, keep_version): + """ + Remove duplicate versions of packages, keeping the specified version. + + Args: + directory (str): The directory containing the Python modules. + duplicates (dict): A dictionary where the keys are package names and the values are lists of duplicate versions. + keep_version (str): The version to keep. + """ + + for pkg, versions in duplicates.items(): + for version in versions: + print(f"Removing {pkg}=={version} from {directory}") + package_dir = os.path.join(directory, pkg + '-' + version + '.dist-info') + if os.path.exists(package_dir): + shutil.rmtree(package_dir) + else: + print(f"Directory {package_dir} does not exist.") + + +def main(): + """ + Main function to check for and remove duplicate versions of Python modules. + + The script accepts a directory path and an optional flag to remove duplicates. + """ + parser = argparse.ArgumentParser(description="Check for and remove duplicate versions of Python modules.") + parser.add_argument('directory', type=str, help="The directory containing the Python modules.") + parser.add_argument('--remove-duplicates', action='store_true', help="Remove duplicate versions of modules.") + args = parser.parse_args() + + directory = args.directory + remove_duplicates_flag = args.remove_duplicates + + duplicates = find_duplicates(directory) + + if duplicates: + print("Duplicate versions found:") + for pkg, versions in duplicates.items(): + print(f"{pkg}: {versions}") + + if remove_duplicates_flag: + for pkg, versions in duplicates.items(): + keep_version = versions[0] # Keep the first version found + remove_duplicates(directory, {pkg: versions}, keep_version) + else: + print("No duplicate versions found.") + +if __name__ == "__main__": + main() diff --git a/Tools/extract_configuration_record.py b/Tools/extract_configuration_record.py new file mode 100755 index 000000000..5b76de52d --- /dev/null +++ b/Tools/extract_configuration_record.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: badz & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + +import sqlite3 +import json +import base64 +from datetime import datetime +import argparse + +def decode_base64_to_dict(b64_data): + """ + Decodes a base64-encoded JSON string to a dictionary. + + Parameters: + b64_data (str): Base64-encoded JSON string. + + Returns: + dict: Decoded JSON dictionary if decoding is successful; otherwise, an empty dictionary. + """ + try: + # Decode base64 to a JSON string and then load it as a dictionary + decoded_json = base64.b64decode(b64_data).decode('utf-8') + return json.loads(decoded_json) + except (base64.binascii.Error, UnicodeDecodeError, json.JSONDecodeError): + print("Warning: Could not decode base64 or parse JSON") + return {} + + +def decode_configuration(config_str, main_attribut=None): + """ + Decodes the Configuration JSON string and processes its fields based on known Attribut values. + + The function checks the `Version` of each attribute and decodes `b64encoded` only if `Version` is 2. + Otherwise, it leaves `b64encoded` as a base64-encoded string. + + Parameters: + config_str (str): JSON string from the Configuration column of the database. + main_attribut (str, optional): The main attribute (e.g., "ListOfDevices") to filter and print. Default is None to process all. + + Returns: + dict: Processed configuration dictionary with decoded fields where applicable. + """ + try: + # Decode JSON string to a dictionary + config_data = json.loads(config_str) if config_str else {} + + # If a specific main attribute is provided, filter to process just that one + attributes_to_process = [main_attribut] if main_attribut else [ "ListOfGroups", "ListOfDevices", "PluginConf", "CoordinatorBackup"] + + for attribut_key in attributes_to_process: + if attribut_key in config_data: + attribut_data = config_data[attribut_key] + + # Decode TimeStamp to a readable format + timestamp = attribut_data.get("TimeStamp") + if timestamp: + attribut_data["TimeStamp"] = datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') + + # Handle the "b64encoded" field based on Version + version = attribut_data.get("Version") + b64_data = attribut_data.get("b64encoded") + + # Only decode b64encoded if Version is 3 + if b64_data and version == 3: + attribut_data["b64encoded"] = decode_base64_to_dict(b64_data) + else: + attribut_data["b64encoded"] = b64_data # Leave as base64 string + + # Convert Version to string if present + if version is not None: + attribut_data["Version"] = str(version) + + return config_data + except json.JSONDecodeError: + print("Warning: Could not decode Configuration JSON") + return {} + + +def fetch_hardware_records(database_path, main_attribut=None): + """ + Connects to the SQLite database, retrieves records from the Hardware table, + decodes the Configuration field, and prints each record in JSON format. + + Parameters: + database_path (str): Path to the SQLite database file. + main_attribut (str, optional): The main attribute (e.g., "ListOfDevices") to filter and print. Default is None to print all. + + Output: + None: The function prints each record in a structured JSON format to the console. + """ + # Connect to the SQLite database + conn = sqlite3.connect(database_path) + cursor = conn.cursor() + + # Query to extract ID, Name, and Configuration from the Hardware table + query = "SELECT ID, Name, Configuration FROM Hardware" + + try: + # Execute the query and fetch all results + cursor.execute(query) + records = cursor.fetchall() + + # Display each record in JSON format + for record in records: + record_id = record[0] + name = record[1] + config_str = record[2] + + # Decode the Configuration field, optionally filtering to a specific main attribute + configuration = decode_configuration(config_str, main_attribut) + + # Create a dictionary for JSON output + record_data = { + "ID": record_id, + "Name": name, + "Configuration": configuration + } + + # Print the record data as a JSON string + print(json.dumps(record_data, indent=4)) + except sqlite3.Error as e: + print(f"An error occurred: {e}") + finally: + # Close the database connection + conn.close() + + +def main(): + """ + Main function to parse command-line arguments and call the appropriate function to fetch hardware records. + """ + # Set up argument parsing + parser = argparse.ArgumentParser(description="Fetch hardware records from an SQLite database.") + parser.add_argument("database", help="Path to the SQLite database file") + parser.add_argument("--main_attribut", choices=[ "ListOfGroups", "ListOfDevices", "PluginConf", "CoordinatorBackup"], + help="Specify which main attribute to print (default is to print all)") + + # Parse arguments + args = parser.parse_args() + + # Fetch and print hardware records based on the provided arguments + fetch_hardware_records(args.database, args.main_attribut) + +# Run the script +if __name__ == "__main__": + main() diff --git a/Tools/plugin-auto-upgrade.sh b/Tools/plugin-auto-upgrade.sh index 19dfc3836..4ff86c42d 100755 --- a/Tools/plugin-auto-upgrade.sh +++ b/Tools/plugin-auto-upgrade.sh @@ -85,16 +85,30 @@ check_and_activate_venv() { # Function to install python3-pip on Debian if necessary install_pip_on_debian() { + # Check if lsb_release command exists if command -v lsb_release &> /dev/null; then - DISTRIB_ID=$(lsb_release -is) - DISTRIB_RELEASE=$(lsb_release -rs) - if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then - if ! command -v pip3 &> /dev/null; then - echo "pip3 is not installed. Installing python3-pip..." - sudo apt-get update - sudo apt-get install -y python3-pip + # Get distribution ID and release number + DISTRIB_ID=$(lsb_release -is 2>/dev/null) + DISTRIB_RELEASE=$(lsb_release -rs 2>/dev/null) + + # Check if the distribution ID and release number were retrieved successfully + if [ -n "$DISTRIB_ID" ] && [ -n "$DISTRIB_RELEASE" ]; then + if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then + if ! command -v pip3 &> /dev/null; then + echo "pip3 is not installed. Installing python3-pip..." + sudo apt-get update + sudo apt-get install -y python3-pip + else + echo "pip3 is already installed." + fi + else + echo "This script is intended for Debian 12 only." fi + else + echo "Failed to retrieve distribution information." fi + else + echo "lsb_release command not found. This script requires lsb_release to determine the distribution." fi } @@ -129,6 +143,41 @@ update_git_config() { fi } +check_and_remove_duplicates() { + # Get the directory of the current script + SCRIPT_DIR=$(dirname "$(realpath "$0")") + + # Print the directory (for debugging purposes) + echo "Script directory: $SCRIPT_DIR" + python3 $SCRIPT_DIR/check_and_remove_duplicates.py $VENV_PATH --remove-duplicates +} + +check_and_remove_duplicates() { + # Get the directory of the current script + SCRIPT_DIR=$(dirname "$(realpath "$0")") + + # Print the directory (for debugging purposes) + echo "Script directory: $SCRIPT_DIR" + + # Check if the Python script exists + if [ ! -f "$SCRIPT_DIR/check_and_remove_duplicates.py" ]; then + echo "Error: Python script not found at $SCRIPT_DIR/check_and_remove_duplicates.py" + return 1 + fi + + # Launch the Python script + python3 "$SCRIPT_DIR/check_and_remove_duplicates.py" "$VENV_PATH" --remove-duplicates + + # Check if the Python script executed successfully + if [ $? -ne 0 ]; then + echo "Error: Python script execution failed" + return 1 + fi + + echo "Duplicates removed successfully" +} + + # Function to update python modules update_python_modules() { echo " " @@ -169,6 +218,7 @@ set_pip_options check_and_activate_venv print_version_info update_git_config +check_and_remove_duplicates update_python_modules echo " " diff --git a/Z4D_decoders/z4d_decoder_IEEE_Addr_Rsp.py b/Z4D_decoders/z4d_decoder_IEEE_Addr_Rsp.py index 18d7bf711..2902afe68 100644 --- a/Z4D_decoders/z4d_decoder_IEEE_Addr_Rsp.py +++ b/Z4D_decoders/z4d_decoder_IEEE_Addr_Rsp.py @@ -1,57 +1,105 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: zaraki673 & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + from Modules.basicOutputs import handle_unknow_device -from Modules.domoTools import lastSeenUpdate from Modules.errorCodes import DisplayStatusCode -from Modules.tools import (DeviceExist, loggingMessages, timeStamped, +from Modules.tools import (DeviceExist, loggingMessages, zigpy_plugin_sanity_check) def Decode8041(self, Devices, MsgData, MsgLQI): + """ + Decode an 8041 IEEE Address response and process the received information. + + Args: + Devices (dict): Dictionary of devices managed by the system. + MsgData (str): The raw message data received from the network. + MsgLQI (str): Link Quality Indicator (LQI) of the received message. + + This method: + - Extracts and parses the IEEE address response. + - Logs the details of the response. + - Validates the response data against known devices. + - Updates timestamps and device records or handles unknown devices. + """ MsgSequenceNumber = MsgData[:2] MsgDataStatus = MsgData[2:4] MsgIEEE = MsgData[4:20] + + # Log and exit on invalid status if MsgDataStatus != '00': - self.log.logging('Input', 'Debug', 'Decode8041 - Reception of IEEE Address response for %s with status %s' % (MsgIEEE, MsgDataStatus)) + self.log.logging( 'Input', 'Debug', f"Decode8041 - Reception of IEEE Address response for {MsgIEEE} with status {MsgDataStatus}" ) return - + MsgShortAddress = MsgData[20:24] - extendedResponse = False - - if len(MsgData) > 24: - extendedResponse = True + extendedResponse = len(MsgData) > 24 + + if extendedResponse: MsgNumAssocDevices = MsgData[24:26] MsgStartIndex = MsgData[26:28] MsgDeviceList = MsgData[28:] - - if extendedResponse: - self.log.logging('Input', 'Debug', 'Decode8041 - IEEE Address response, Sequence number: %s Status: %s IEEE: %s NwkId: %s nbAssociated Devices: %s StartIdx: %s DeviceList: %s' % (MsgSequenceNumber, DisplayStatusCode(MsgDataStatus), MsgIEEE, MsgShortAddress, MsgNumAssocDevices, MsgStartIndex, MsgDeviceList)) + self.log.logging( + 'Input', 'Debug', + f"Decode8041 - IEEE Address response, Sequence number: {MsgSequenceNumber}, " + f"Status: {DisplayStatusCode(MsgDataStatus)}, IEEE: {MsgIEEE}, NwkId: {MsgShortAddress}, " + f"nbAssociated Devices: {MsgNumAssocDevices}, StartIdx: {MsgStartIndex}, DeviceList: {MsgDeviceList}" + ) - if MsgShortAddress == '0000' and self.ControllerIEEE and (MsgIEEE != self.ControllerIEEE): - self.log.logging('Input', 'Error', 'Decode8041 - Receive an IEEE: %s with a NwkId: %s something wrong !!!' % (MsgIEEE, MsgShortAddress)) + # Validate addresses and log inconsistencies + if MsgShortAddress == '0000' and self.ControllerIEEE and MsgIEEE != self.ControllerIEEE: + self.log.logging( + 'Input', 'Error', + f"Decode8041 - Received an IEEE: {MsgIEEE} with NwkId: {MsgShortAddress} - something is wrong!" + ) return - - elif self.ControllerIEEE and MsgIEEE == self.ControllerIEEE and (MsgShortAddress != '0000'): - self.log.logging('Input', 'Log', 'Decode8041 - Receive an IEEE: %s with a NwkId: %s something wrong !!!' % (MsgIEEE, MsgShortAddress)) + elif self.ControllerIEEE and MsgIEEE == self.ControllerIEEE and MsgShortAddress != '0000': + self.log.logging( + 'Input', 'Log', + f"Decode8041 - Received an IEEE: {MsgIEEE} with NwkId: {MsgShortAddress} - something is wrong!" + ) return - if MsgShortAddress in self.ListOfDevices and 'IEEE' in self.ListOfDevices[MsgShortAddress] and (self.ListOfDevices[MsgShortAddress]['IEEE'] == MsgShortAddress): - self.log.logging('Input', 'Debug', 'Decode8041 - Receive an IEEE: %s with a NwkId: %s' % (MsgIEEE, MsgShortAddress)) - timeStamped(self, MsgShortAddress, 32833) + # Handle known devices + if (MsgShortAddress in self.ListOfDevices + and 'IEEE' in self.ListOfDevices[MsgShortAddress] + and self.ListOfDevices[MsgShortAddress]['IEEE'] == MsgIEEE + ): + self.log.logging( + 'Input', 'Debug', + f"Decode8041 - Received an IEEE: {MsgIEEE} with NwkId: {MsgShortAddress}" + ) loggingMessages(self, '8041', MsgShortAddress, MsgIEEE, MsgLQI, MsgSequenceNumber) - lastSeenUpdate(self, Devices, NwkId=MsgShortAddress) return + # Handle reconnection for devices known by IEEE if MsgIEEE in self.IEEE2NWK: - self.log.logging('Input', 'Debug', 'Decode8041 - Receive an IEEE: %s with a NwkId: %s, will try to reconnect' % (MsgIEEE, MsgShortAddress)) - + self.log.logging( + 'Input', 'Debug', + f"Decode8041 - Received an IEEE: {MsgIEEE} with NwkId: {MsgShortAddress}, will try to reconnect" + ) if not DeviceExist(self, Devices, MsgShortAddress, MsgIEEE): if not zigpy_plugin_sanity_check(self, MsgShortAddress): handle_unknow_device(self, MsgShortAddress) - self.log.logging('Input', 'Log', 'Decode8041 - Not able to reconnect (unknown device) %s %s' % (MsgIEEE, MsgShortAddress)) + self.log.logging( + 'Input', 'Log', + f"Decode8041 - Unable to reconnect (unknown device) {MsgIEEE} {MsgShortAddress}" + ) return - timeStamped(self, MsgShortAddress, 32833) loggingMessages(self, '8041', MsgShortAddress, MsgIEEE, MsgLQI, MsgSequenceNumber) - lastSeenUpdate(self, Devices, NwkId=MsgShortAddress) return - self.log.logging('Input', 'Log', 'WARNING - Decode8041 - Receive an IEEE: %s with a NwkId: %s, not known by the plugin' % (MsgIEEE, MsgShortAddress)) \ No newline at end of file + # Handle unknown devices + self.log.logging( + 'Input', 'Log', + f"WARNING - Decode8041 - Received an IEEE: {MsgIEEE} with NwkId: {MsgShortAddress}, not known by the plugin" + ) diff --git a/Z4D_decoders/z4d_decoder_IEEE_addr_req.py b/Z4D_decoders/z4d_decoder_IEEE_addr_req.py index abc8ed351..69121aba0 100644 --- a/Z4D_decoders/z4d_decoder_IEEE_addr_req.py +++ b/Z4D_decoders/z4d_decoder_IEEE_addr_req.py @@ -1,34 +1,62 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: zaraki673 & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + import struct from Modules.sendZigateCommand import raw_APS_request def Decode0041(self, Devices, MsgData, MsgLQI): - self.log.logging('Input', 'Debug', 'Decode0041 - IEEE_addr_req: %s %s %s' % (MsgData, self.ControllerNWKID, self.ControllerIEEE)) + self.log.logging('Input', 'Debug', f"Decode0041 - IEEE_addr_req: {MsgData} {self.ControllerNWKID} {self.ControllerIEEE}") + + if not self.ControllerIEEE or self.ControllerNWKID in (None, "ffff"): + return + + # Parse message data + sqn, srcNwkId, srcEp, nwkid, reqType, startIndex = ( MsgData[:2], MsgData[2:6], MsgData[6:8], MsgData[8:12], MsgData[12:14], MsgData[14:16] ) + + # Log parsed details + self.log.logging('Input', 'Debug', f" source req SrcNwkId: {srcNwkId} NwkId: {nwkid} Type: {reqType} Idx: {startIndex}") - sqn = MsgData[:2] - srcNwkId = MsgData[2:6] - srcEp = MsgData[6:8] - nwkid = MsgData[8:12] - reqType = MsgData[12:14] - startIndex = MsgData[14:16] - self.log.logging('Input', 'Debug', ' source req nwkid: %s' % srcNwkId) - self.log.logging('Input', 'Debug', ' request NwkId : %s' % nwkid) - self.log.logging('Input', 'Debug', ' request Type : %s' % reqType) - self.log.logging('Input', 'Debug', ' request Idx : %s' % startIndex) Cluster = '8001' + status, payload = _generate_response_payload(self, nwkid, sqn) + + self.log.logging('Input', 'Debug', f"Decode0041 - response payload: {payload}") + raw_APS_request(self, srcNwkId, '00', Cluster, '0000', payload, zigpyzqn=sqn, zigate_ep='00') + + +def _generate_response_payload(self, nwkid, sqn): + """Generate the response payload based on the requested nwkid.""" if nwkid == self.ControllerNWKID: status = '00' - controller_ieee = '%016x' % struct.unpack('Q', struct.pack('>Q', int(self.ControllerIEEE, 16)))[0] - controller_nwkid = '%04x' % struct.unpack('H', struct.pack('>H', int(self.ControllerNWKID, 16)))[0] - payload = sqn + status + controller_ieee + controller_nwkid + '00' + ieee = _format_ieee(self, self.ControllerIEEE) + nwk_id = _format_nwkid(self, self.ControllerNWKID) + payload = sqn + status + ieee + nwk_id + '00' elif nwkid in self.ListOfDevices: status = '00' - device_ieee = '%016x' % struct.unpack('Q', struct.pack('>Q', int(self.ListOfDevices[nwkid]['IEEE'], 16)))[0] - device_nwkid = '%04x' % struct.unpack('H', struct.pack('>H', int(self.ControllerNWKID, 16)))[0] - payload = sqn + status + device_ieee + device_nwkid + '00' + ieee = _format_ieee(self, self.ListOfDevices[nwkid]['IEEE']) + nwk_id = _format_nwkid(self, self.ControllerNWKID) + payload = sqn + status + ieee + nwk_id + '00' else: status = '81' payload = sqn + status + nwkid - self.log.logging('Input', 'Debug', 'Decode0041 - response payload: %s' % payload) - raw_APS_request(self, srcNwkId, '00', Cluster, '0000', payload, zigpyzqn=sqn, zigate_ep='00') \ No newline at end of file + return status, payload + + +def _format_ieee(self, ieee): + """Format the IEEE address to 16-character hex string.""" + return f"{int(ieee, 16):016x}" + + +def _format_nwkid(self, nwkid): + """Format the NWK ID to 4-character hex string.""" + return f"{int(nwkid, 16):04x}" diff --git a/Z4D_decoders/z4d_decoder_NWK_addr_req.py b/Z4D_decoders/z4d_decoder_NWK_addr_req.py index a26bc7636..1e2c30dce 100644 --- a/Z4D_decoders/z4d_decoder_NWK_addr_req.py +++ b/Z4D_decoders/z4d_decoder_NWK_addr_req.py @@ -1,21 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: zaraki673 & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + import struct from Modules.sendZigateCommand import raw_APS_request def Decode0040(self, Devices, MsgData, MsgLQI): + """ + Decode a NWK address request (0040) and prepare a response. + + Args: + Devices (dict): The list of devices. + MsgData (str): The received message data. + MsgLQI (str): The Link Quality Indicator for the message. + + This function processes a NWK address request message, prepares the appropriate + response, and sends the response back. + """ + # Log incoming message details self.log.logging('Input', 'Debug', 'Decode0040 - NWK_addr_req: %s' % MsgData) + + # Extract relevant fields from MsgData sqn = MsgData[:2] srcNwkId = MsgData[2:6] srcEp = MsgData[6:8] ieee = MsgData[8:24] reqType = MsgData[24:26] startIndex = MsgData[26:28] - self.log.logging('Input', 'Debug', ' source req nwkid: %s' % srcNwkId) - self.log.logging('Input', 'Debug', ' request IEEE : %s' % ieee) - self.log.logging('Input', 'Debug', ' request Type : %s' % reqType) - self.log.logging('Input', 'Debug', ' request Idx : %s' % startIndex) + + # Log the extracted fields + self.log.logging('Input', 'Debug', f" source req nwkid: {srcNwkId}") + self.log.logging('Input', 'Debug', f" request IEEE : {ieee}") + self.log.logging('Input', 'Debug', f" request Type : {reqType}") + self.log.logging('Input', 'Debug', f" request Idx : {startIndex}") + + # Define the cluster ID for the request Cluster = '8000' + + # Prepare the payload based on the IEEE address if ieee == self.ControllerIEEE: controller_ieee = '%016x' % struct.unpack('Q', struct.pack('>Q', int(self.ControllerIEEE, 16)))[0] controller_nwkid = '%04x' % struct.unpack('H', struct.pack('>H', int(self.ControllerNWKID, 16)))[0] @@ -29,5 +61,9 @@ def Decode0040(self, Devices, MsgData, MsgLQI): else: status = '81' payload = sqn + status + ieee - self.log.logging('Input', 'Debug', 'Decode0040 - response payload: %s' % payload) - raw_APS_request(self, srcNwkId, '00', Cluster, '0000', payload, zigpyzqn=sqn, zigate_ep='00') \ No newline at end of file + + # Log the response payload + self.log.logging('Input', 'Debug', f'Decode0040 - response payload: {payload}') + + # Send the response back using raw APS request + raw_APS_request(self, srcNwkId, '00', Cluster, '0000', payload, zigpyzqn=sqn, zigate_ep='00') diff --git a/Z4D_decoders/z4d_decoder_Node_Desc_Rsp.py b/Z4D_decoders/z4d_decoder_Node_Desc_Rsp.py index ea2f79506..fd6107fd8 100644 --- a/Z4D_decoders/z4d_decoder_Node_Desc_Rsp.py +++ b/Z4D_decoders/z4d_decoder_Node_Desc_Rsp.py @@ -1,94 +1,116 @@ -from Modules.tools import (ReArrangeMacCapaBasedOnModel, decodeMacCapa, updLQI) +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: zaraki673 & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + +from Modules.tools import ReArrangeMacCapaBasedOnModel, decodeMacCapa, updLQI + def Decode8042(self, Devices, MsgData, MsgLQI): + """ + Decode an 8042 Node Descriptor response and update device information. + + Args: + Devices (dict): Dictionary of devices managed by the system. + MsgData (str): The raw message data received from the network. + MsgLQI (str): Link Quality Indicator (LQI) of the received message. + + This method parses the node descriptor data and updates the device records with + relevant information such as manufacturer, capabilities, and logical type. + """ sequence = MsgData[:2] status = MsgData[2:4] addr = MsgData[4:8] - if status == '00': - manufacturer = MsgData[8:12] - max_rx = MsgData[12:16] - max_tx = MsgData[16:20] - server_mask = MsgData[20:24] - descriptor_capability = MsgData[24:26] - mac_capability = MsgData[26:28] - max_buffer = MsgData[28:30] - bit_field = MsgData[30:34] + + # Handle invalid status codes if status != '00': - self.log.logging('Input', 'Debug', 'Decode8042 - Reception of Node Descriptor for %s with status %s' % (addr, status)) + self.log.logging( + 'Input', 'Debug', + f"Decode8042 - Reception of Node Descriptor for {addr} with status {status}" + ) return - - self.log.logging('Input', 'Debug', 'Decode8042 - Reception Node Descriptor for: ' + addr + ' SEQ: ' + sequence + ' Status: ' + status + ' manufacturer:' + manufacturer + ' mac_capability: ' + str(mac_capability) + ' bit_field: ' + str(bit_field), addr) + + # Extract descriptor details + manufacturer = MsgData[8:12] + max_rx = MsgData[12:16] + max_tx = MsgData[16:20] + server_mask = MsgData[20:24] + descriptor_capability = MsgData[24:26] + mac_capability = MsgData[26:28] + max_buffer = MsgData[28:30] + bit_field = MsgData[30:34] + + self.log.logging( + 'Input', 'Debug', + f"Decode8042 - Reception Node Descriptor for: {addr}, SEQ: {sequence}, " + f"Status: {status}, Manufacturer: {manufacturer}, MAC Capability: {mac_capability}, Bit Field: {bit_field}", + addr + ) + + # Initialize device record if not present if addr == '0000' and addr not in self.ListOfDevices: - self.ListOfDevices[addr] = {} - self.ListOfDevices[addr]['Ep'] = {} + self.ListOfDevices[addr] = {'Ep': {}} if addr not in self.ListOfDevices: - self.log.logging('Input', 'Log', 'Decode8042 receives a message from a non existing device %s' % addr) + self.log.logging( + 'Input', 'Log', + f"Decode8042 received a message from a non-existing device {addr}" + ) return - + + # Update device details updLQI(self, addr, MsgLQI) - - self.ListOfDevices[addr]['_rawNodeDescriptor'] = MsgData[8:] - self.ListOfDevices[addr]['Max Buffer Size'] = max_buffer - self.ListOfDevices[addr]['Max Rx'] = max_rx - self.ListOfDevices[addr]['Max Tx'] = max_tx - self.ListOfDevices[addr]['macapa'] = mac_capability - self.ListOfDevices[addr]['bitfield'] = bit_field - self.ListOfDevices[addr]['server_mask'] = server_mask - self.ListOfDevices[addr]['descriptor_capability'] = descriptor_capability + self.ListOfDevices[addr].update({ + '_rawNodeDescriptor': MsgData[8:], + 'Max Buffer Size': max_buffer, + 'Max Rx': max_rx, + 'Max Tx': max_tx, + 'macapa': mac_capability, + 'bitfield': bit_field, + 'server_mask': server_mask, + 'descriptor_capability': descriptor_capability, + }) + + # Rearrange MAC capability and decode capabilities mac_capability = ReArrangeMacCapaBasedOnModel(self, addr, mac_capability) capabilities = decodeMacCapa(mac_capability) - - if 'Able to act Coordinator' in capabilities: - AltPAN = 1 - - else: - AltPAN = 0 - - if 'Main Powered' in capabilities: - PowerSource = 'Main' - - else: - PowerSource = 'Battery' - - if 'Full-Function Device' in capabilities: - DeviceType = 'FFD' - - else: - DeviceType = 'RFD' - - if 'Receiver during Idle' in capabilities: - ReceiveonIdle = 'On' - - else: - ReceiveonIdle = 'Off' - - self.log.logging('Input', 'Debug', 'Decode8042 - Alternate PAN Coordinator = ' + str(AltPAN), addr) - self.log.logging('Input', 'Debug', 'Decode8042 - Receiver on Idle = ' + str(ReceiveonIdle), addr) - self.log.logging('Input', 'Debug', 'Decode8042 - Power Source = ' + str(PowerSource), addr) - self.log.logging('Input', 'Debug', 'Decode8042 - Device type = ' + str(DeviceType), addr) + + # Determine device properties + AltPAN = 'Able to act Coordinator' in capabilities + PowerSource = 'Main' if 'Main Powered' in capabilities else 'Battery' + DeviceType = 'FFD' if 'Full-Function Device' in capabilities else 'RFD' + ReceiveOnIdle = 'On' if 'Receiver during Idle' in capabilities else 'Off' + + # Log device properties + self.log.logging('Input', 'Debug', f"Decode8042 - Alternate PAN Coordinator = {AltPAN}", addr) + self.log.logging('Input', 'Debug', f"Decode8042 - Receiver on Idle = {ReceiveOnIdle}", addr) + self.log.logging('Input', 'Debug', f"Decode8042 - Power Source = {PowerSource}", addr) + self.log.logging('Input', 'Debug', f"Decode8042 - Device Type = {DeviceType}", addr) + + # Parse bit fields to determine logical type bit_fieldL = int(bit_field[2:4], 16) bit_fieldH = int(bit_field[:2], 16) - self.log.logging('Input', 'Debug', 'Decode8042 - bit_fieldL = %s bit_fieldH = %s' % (bit_fieldL, bit_fieldH)) - LogicalType = bit_fieldL & 15 - - if LogicalType == 0: - LogicalType = 'Coordinator' - - elif LogicalType == 1: - LogicalType = 'Router' - - elif LogicalType == 2: - LogicalType = 'End Device' - - self.log.logging('Input', 'Debug', 'Decode8042 - bit_field = ' + str(bit_fieldL) + ': ' + str(bit_fieldH), addr) - self.log.logging('Input', 'Debug', 'Decode8042 - Logical Type = ' + str(LogicalType), addr) - - if 'Manufacturer' not in self.ListOfDevices[addr] or self.ListOfDevices[addr]['Manufacturer'] in ('', {}): - self.ListOfDevices[addr]['Manufacturer'] = manufacturer - - if 'Status' not in self.ListOfDevices[addr] or self.ListOfDevices[addr]['Status'] != 'inDB': - self.ListOfDevices[addr]['Manufacturer'] = manufacturer - self.ListOfDevices[addr]['DeviceType'] = str(DeviceType) - self.ListOfDevices[addr]['LogicalType'] = str(LogicalType) - self.ListOfDevices[addr]['PowerSource'] = str(PowerSource) - self.ListOfDevices[addr]['ReceiveOnIdle'] = str(ReceiveonIdle) \ No newline at end of file + LogicalType = ['Coordinator', 'Router', 'End Device'][bit_fieldL & 15] if (bit_fieldL & 15) < 3 else 'Unknown' + + self.log.logging('Input', 'Debug', f"Decode8042 - bit_field = {bit_fieldL}:{bit_fieldH}", addr) + self.log.logging('Input', 'Debug', f"Decode8042 - Logical Type = {LogicalType}", addr) + + # Update or initialize device attributes + device_record = self.ListOfDevices[addr] + device_record.setdefault('Manufacturer', manufacturer) + if device_record.get('Status') != 'inDB': + device_record.update( + { + 'Manufacturer': manufacturer, + 'DeviceType': DeviceType, + 'LogicalType': str(LogicalType), + 'PowerSource': PowerSource, + 'ReceiveOnIdle': ReceiveOnIdle, + } + ) diff --git a/Z4D_decoders/z4d_decoder_Nwk_Addr_Rsp.py b/Z4D_decoders/z4d_decoder_Nwk_Addr_Rsp.py index fa761f4ee..8cd9bb2a6 100644 --- a/Z4D_decoders/z4d_decoder_Nwk_Addr_Rsp.py +++ b/Z4D_decoders/z4d_decoder_Nwk_Addr_Rsp.py @@ -1,7 +1,19 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Implementation of Zigbee for Domoticz plugin. +# +# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee +# (C) 2015-2024 +# +# Initial authors: zaraki673 & pipiche38 +# +# SPDX-License-Identifier: GPL-3.0 license + + from Modules.basicOutputs import handle_unknow_device -from Modules.domoTools import lastSeenUpdate from Modules.errorCodes import DisplayStatusCode -from Modules.tools import (DeviceExist, loggingMessages, timeStamped, +from Modules.tools import (DeviceExist, loggingMessages, zigpy_plugin_sanity_check) from Modules.zb_tables_management import store_NwkAddr_Associated_Devices from Z4D_decoders.z4d_decoder_helpers import \ @@ -9,57 +21,81 @@ def Decode8040(self, Devices, MsgData, MsgLQI): - self.log.logging('Input', 'Debug', 'Decode8040 - payload %s' % MsgData) + """ + Decodes the 8040 message received from the network and handles associated logic. + + Args: + Devices (dict): A dictionary of known devices. + MsgData (str): The payload of the message to decode. + MsgLQI (int): The Link Quality Indicator of the message. + + Returns: + None + """ + + self.log.logging('Input', 'Debug', f'Decode8040 - payload {MsgData}') MsgSequenceNumber = MsgData[:2] MsgDataStatus = MsgData[2:4] MsgIEEE = MsgData[4:20] - self.log.logging('Input', 'Debug', 'Decode8040 - Reception of Network Address response %s with status %s' % (MsgIEEE, MsgDataStatus)) + + self.log.logging('Input', 'Debug', f'Decode8040 - Reception of Network Address response {MsgIEEE} with status {MsgDataStatus}') + if MsgDataStatus != '00': return + MsgShortAddress = MsgData[20:24] - extendedResponse = False - - if len(MsgData) > 26: - extendedResponse = True + extendedResponse = len(MsgData) > 26 + MsgNumAssocDevices, MsgStartIndex, MsgDeviceList = None, None, None + + if extendedResponse: MsgNumAssocDevices = int(MsgData[24:26], 16) MsgStartIndex = int(MsgData[26:28], 16) MsgDeviceList = MsgData[28:] - - self.log.logging('Input', 'Debug', 'Network Address response, [%s] Status: %s Ieee: %s NwkId: %s' % (MsgSequenceNumber, DisplayStatusCode(MsgDataStatus), MsgIEEE, MsgShortAddress)) + + self.log.logging( + 'Input', 'Debug', + f'Network Address response, [{MsgSequenceNumber}] Status: {DisplayStatusCode(MsgDataStatus)} ' + f'Ieee: {MsgIEEE} NwkId: {MsgShortAddress}' + ) if extendedResponse: - self.log.logging('Input', 'Debug', ' , Nb Associated Devices: %s Idx: %s Device List: %s' % (MsgNumAssocDevices, MsgStartIndex, MsgDeviceList)) + self.log.logging('Input', 'Debug', f'Nb Associated Devices: {MsgNumAssocDevices} Idx: {MsgStartIndex} Device List: {MsgDeviceList}') if MsgStartIndex + len(MsgDeviceList) // 4 != MsgNumAssocDevices: - self.log.logging('Input', 'Debug', 'Decode 8040 - Receive an IEEE: %s with a NwkId: %s but would need to continue to get all associated devices' % (MsgIEEE, MsgShortAddress)) - Network_Address_response_request_next_index(self, MsgShortAddress, MsgIEEE, MsgStartIndex, len(MsgDeviceList) // 4) + self.log.logging( + 'Input', 'Debug', + f'Decode 8040 - Receive an IEEE: {MsgIEEE} with a NwkId: {MsgShortAddress} ' + f'but would need to continue to get all associated devices' + ) + Network_Address_response_request_next_index( self, MsgShortAddress, MsgIEEE, MsgStartIndex, len(MsgDeviceList) // 4 ) - if MsgShortAddress in self.ListOfDevices and 'IEEE' in self.ListOfDevices[MsgShortAddress] and (self.ListOfDevices[MsgShortAddress]['IEEE'] == MsgIEEE): - self.log.logging('Input', 'Debug', 'Decode 8041 - Receive an IEEE: %s with a NwkId: %s' % (MsgIEEE, MsgShortAddress)) + ieee_matches = (MsgShortAddress in self.ListOfDevices) and (self.ListOfDevices[MsgShortAddress].get('IEEE') == MsgIEEE) + + if ieee_matches: + self.log.logging( 'Input', 'Debug', f'Decode 8041 - Receive an IEEE: {MsgIEEE} with a NwkId: {MsgShortAddress}' ) if extendedResponse: store_NwkAddr_Associated_Devices(self, MsgShortAddress, MsgStartIndex, MsgDeviceList) - timeStamped(self, MsgShortAddress, 32833) loggingMessages(self, '8040', MsgShortAddress, MsgIEEE, MsgLQI, MsgSequenceNumber) - lastSeenUpdate(self, Devices, NwkId=MsgShortAddress) return if MsgIEEE in self.IEEE2NWK: - self.log.logging('Input', 'Log', 'Decode 8040 - Receive an IEEE: %s with a NwkId: %s, will try to reconnect' % (MsgIEEE, MsgShortAddress)) - - if not DeviceExist(self, Devices, MsgShortAddress, MsgIEEE): + self.log.logging( 'Input', 'Debug', f'Decode 8040 - Receive an IEEE: {MsgIEEE} with a NwkId: {MsgShortAddress}, will try to reconnect' ) - if not zigpy_plugin_sanity_check(self, MsgShortAddress): - handle_unknow_device(self, MsgShortAddress) - self.log.logging('Input', 'Debug', 'Decode 8040 - Not able to reconnect (unknown device)') + if not DeviceExist(self, Devices, MsgShortAddress, MsgIEEE) and not zigpy_plugin_sanity_check(self, MsgShortAddress): + handle_unknow_device(self, MsgShortAddress) + self.log.logging( 'Input', 'Debug', 'Decode 8040 - Not able to reconnect (unknown device)' ) return if extendedResponse: store_NwkAddr_Associated_Devices(self, MsgShortAddress, MsgStartIndex, MsgDeviceList) - timeStamped(self, MsgShortAddress, 32833) loggingMessages(self, '8040', MsgShortAddress, MsgIEEE, MsgLQI, MsgSequenceNumber) - lastSeenUpdate(self, Devices, NwkId=MsgShortAddress) + return - self.log.logging('Input', 'Error', 'Decode 8040 - Receive an IEEE: %s with a NwkId: %s, seems not known by the plugin' % (MsgIEEE, MsgShortAddress)) \ No newline at end of file + self.log.logging( + 'Input', 'Error', + f'Decode 8040 - Receive an IEEE: {MsgIEEE} with a NwkId: {MsgShortAddress}, ' + f'seems not known by the plugin' + ) diff --git a/Z4D_decoders/z4d_decoder_Zigate_Cmd_Rsp.py b/Z4D_decoders/z4d_decoder_Zigate_Cmd_Rsp.py index f3a92075e..dd3a88b21 100644 --- a/Z4D_decoders/z4d_decoder_Zigate_Cmd_Rsp.py +++ b/Z4D_decoders/z4d_decoder_Zigate_Cmd_Rsp.py @@ -65,7 +65,7 @@ def Decode8011(self, Devices, MsgData, MsgLQI, TransportInfos=None): MsgLen = len(MsgData) MsgStatus = MsgData[:2] MsgSrcAddr = MsgData[2:6] - + if MsgSrcAddr not in self.ListOfDevices: if not zigpy_plugin_sanity_check(self, MsgSrcAddr): self.log.logging('Input', 'Debug', f"Decode8011 - not zigpy_plugin_sanity_check {MsgSrcAddr} {MsgStatus}") @@ -97,11 +97,10 @@ def Decode8011(self, Devices, MsgData, MsgLQI, TransportInfos=None): if not _powered: return - + self.log.logging('Input', 'Debug', f"Decode8011 - Timedout {MsgSrcAddr}") timedOutDevice(self, Devices, NwkId=MsgSrcAddr) - set_health_state(self, MsgSrcAddr, MsgData[8:12], MsgStatus) diff --git a/constraints.txt b/constraints.txt index c0952656b..f9f4092d1 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,7 +1,7 @@ -zigpy==0.73.1 +zigpy==0.75.1 zigpy_znp==0.13.1 zigpy_deconz==0.24.1 -bellows==0.42.5 +bellows==0.43.0 dnspython==2.6.1 pyserial>=3.5 charset-normalizer==2.0.11 diff --git a/plugin.py b/plugin.py index 3a78a9991..93c50459b 100644 --- a/plugin.py +++ b/plugin.py @@ -11,7 +11,7 @@ # SPDX-License-Identifier: GPL-3.0 license """ - +

Plugin Zigbee for domoticz



Informations


@@ -134,19 +134,20 @@ is_plugin_update_available, is_zigate_firmware_available) from Modules.command import domoticz_command -from Modules.database import (LoadDeviceList, WriteDeviceList, - checkDevices2LOD, checkListOfDevice2Devices, - import_local_device_conf) +from Modules.database import (checkDevices2LOD, checkListOfDevice2Devices, + import_local_device_conf, load_plugin_database, + save_plugin_database) from Modules.domoticzAbstractLayer import (domo_read_Name, find_legacy_DeviceID_from_unit, how_many_legacy_slot_available, + is_device_ieee_in_domoticz_db, is_domoticz_extended, load_list_of_domoticz_widget) from Modules.heartbeat import processListOfDevices from Modules.input import zigbee_receive_message from Modules.matomo_request import (matomo_coordinator_initialisation, + matomo_coordinator_restart_after_error, matomo_plugin_analytics_infos, - matomo_plugin_restart, matomo_plugin_shutdown, matomo_plugin_started) from Modules.paramDevice import initialize_device_settings @@ -402,13 +403,10 @@ def onStart(self): self.zigbee_communication, self.VersionNewFashion, self.DomoticzMajor, self.DomoticzMinor, Parameters["HomeFolder"], self.HardwareID ) - self.pluginconf.pluginConf["useDomoticzDatabase"] = False - self.pluginconf.pluginConf["storeDomoticzDatabase"] = False - if self.internet_available is None: - self.internet_available = is_internet_available() + self.internet_available = is_internet_available(self) - if self.internet_available: + if self.internet_available and self.pluginconf.pluginConf.get("CheckRequirements", True): if check_requirements( Parameters[ "HomeFolder"] ): # Check_requirements() return True if requirements not meet. self.onStop() @@ -498,7 +496,7 @@ def onStart(self): self.DomoticzMajor, self.DomoticzMinor, ) - self.WebUsername, self.WebPassword = self.domoticzdb_Preferences.retreiveWebUserNamePassword() + self.WebUsername, self.WebPassword = self.domoticzdb_Preferences.retrieve_web_user_name_password() self.adminWidgets = AdminWidgets( self.log , self.pluginconf, self.pluginParameters, self.ListOfDomoticzWidget, Devices, self.ListOfDevices, self.HardwareID, self.IEEE2NWK) self.adminWidgets.updateStatusWidget(Devices, "Starting up") @@ -527,7 +525,7 @@ def onStart(self): # Import DeviceList.txt Filename is : DeviceListName self.log.logging("Plugin", "Status", "Z4D loading database") - if LoadDeviceList(self) == "Failed": + if load_plugin_database(self) == "Failed": self.log.logging("Plugin", "Error", "Something wennt wrong during the import of Load of Devices ...") self.log.logging( @@ -619,8 +617,15 @@ def onStop(self): # Flush ListOfDevices if self.log: - self.log.logging("Plugin", "Log", "Flushing plugin database onto disk") - WriteDeviceList(self, 0) # write immediatly + self.log.logging("Plugin", "Status", "Flushing to disk") + + save_plugin_database(self, -1) # write immediatly + # Save ListGroups + if self.groupmgt: + self.groupmgt.write_groups_list() + + # Save PluginConf + self.pluginconf.write_Settings() # Uninstall Z4D custom UI from Domoticz uninstall_Z4D_to_domoticz_custom_ui() @@ -642,9 +647,10 @@ def onStop(self): if self.pluginconf and self.webserver: self.webserver.onStop() + # Save plugin database if self.PDMready and self.pluginconf: - WriteDeviceList(self, 0) + save_plugin_database(self, -1) # Print and save statistics if configured if self.PDMready and self.pluginconf and self.statistics: @@ -670,44 +676,35 @@ def onStop(self): def onDeviceRemoved(self, Unit): - # def onDeviceRemoved(self, DeviceID, Unit): + """ + Handles the removal of a device or group based on the Unit provided. + """ + # Early exit if the controller is not initialized if not self.ControllerIEEE: - self.log.logging( "Plugin", "Error", "onDeviceRemoved - too early, coordinator and plugin initialisation not completed", ) + self.log.logging( + "Plugin", + "Error", + "onDeviceRemoved - too early, coordinator and plugin initialization not completed", + ) + return if self.log: self.log.logging("Plugin", "Debug", "onDeviceRemoved called") - if not is_domoticz_extended(): - DeviceID = find_legacy_DeviceID_from_unit(self, Devices, Unit) - - device_name = domo_read_Name( self, Devices, DeviceID, Unit, ) - - # Let's check if this is End Node, or Group related. - if DeviceID in self.IEEE2NWK: - NwkId = self.IEEE2NWK[DeviceID] - - self.log.logging("Plugin", "Status", f"Removing Device {DeviceID} {device_name} in progress") - fullyremoved = removeDeviceInList(self, Devices, DeviceID, Unit) - - # We might have to remove also the Device from Groups - if fullyremoved: - if self.groupmgt: - self.groupmgt.RemoveNwkIdFromAllGroups(NwkId) - - # sending a Leave Request to device, so the device will send a leave - leaveRequest(self, ShortAddr=NwkId, IEEE=DeviceID) + # Determine DeviceID for legacy or extended cases + DeviceID = ( + find_legacy_DeviceID_from_unit(self, Devices, Unit) + if not is_domoticz_extended() + else None + ) - # for a remove in case device didn't send the leave - zigate_remove_device(self, str(self.ControllerIEEE), str(DeviceID) ) - self.log.logging( "Plugin", "Status", f"Request device {device_name} -> {DeviceID} to be removed from coordinator" ) + device_name = domo_read_Name(self, Devices, DeviceID, Unit) - self.log.logging("Plugin", "Debug", f"ListOfDevices :After REMOVE {self.ListOfDevices}") - load_list_of_domoticz_widget(self, Devices) - return - - if self.groupmgt and DeviceID in self.groupmgt.ListOfGroups: - self.log.logging("Plugin", "Status", f"Request device {DeviceID} to be remove from Group(s)") - self.groupmgt.FullRemoveOfGroup(Unit, DeviceID) + # Handle removal of end devices or groups + if DeviceID in self.IEEE2NWK: + _remove_device_entry(self, DeviceID, Unit, device_name) + elif self.groupmgt and DeviceID in self.groupmgt.ListOfGroups: + _remove_group_entry(self, Unit, DeviceID) def onConnect(self, Connection, Status, Description): @@ -807,7 +804,7 @@ def restart_plugin(self): self.log.logging("Plugin", "Error", error_message) self.adminWidgets.updateNotificationWidget(Devices, error_message) if self.internet_available and self.pluginconf.pluginConf["MatomoOptIn"]: - matomo_plugin_restart(self) + matomo_coordinator_restart_after_error(self) restartPluginViaDomoticzJsonApi(self, stop=False, url_base_api=Parameters["Mode5"]) @@ -923,11 +920,11 @@ def onHeartbeat(self): # Write the ListOfDevice every 15 minutes or immediatly if we have remove or added a Device if len(Devices) == prevLenDevices: - WriteDeviceList(self, ( (15 * 60) // HEARTBEAT) ) + save_plugin_database(self, ( (15 * 60) // HEARTBEAT) ) else: self.log.logging("Plugin", "Debug", "Devices size has changed , let's write ListOfDevices on disk") - WriteDeviceList(self, 0) # write immediatly + save_plugin_database(self, 0) # write immediatly networksize_update(self) _trigger_coordinator_backup( self ) @@ -1087,19 +1084,18 @@ def _start_zigpy_ZNP(self): from Classes.ZigpyTransport.Transport import ZigpyTransport - #self.pythonModuleVersion["zigpy"] = (zigpy.__version__) - # https://github.com/zigpy/zigpy-znp/issues/205 - #self.pythonModuleVersion["zigpy_znp"] = import_version( 'zigpy-znp' ) - check_python_modules_version( self ) self.zigbee_communication = "zigpy" self.pluginParameters["Zigpy"] = True self.log.logging("Plugin", "Status", "Z4D starting ZNP") - self.ControllerLink= ZigpyTransport( - self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "znp", Parameters["SerialPort"] - ) - self.ControllerLink.open_cie_connection() + try: + self.ControllerLink= ZigpyTransport( + self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "znp", Parameters["SerialPort"] + ) + self.ControllerLink.open_cie_connection() + except Exception as e: + self.log.logging("Plugin", "Error", f"Failed to start Zigpy ZNP: {str(e)}") self.pluginconf.pluginConf["ControllerInRawMode"] = True @@ -1111,15 +1107,17 @@ def _start_zigpy_deConz(self): from Classes.ZigpyTransport.Transport import ZigpyTransport - #self.pythonModuleVersion["zigpy"] = (zigpy.__version__) - #self.pythonModuleVersion["zigpy_deconz"] = (zigpy_deconz.__version__) check_python_modules_version( self ) self.pluginParameters["Zigpy"] = True - self.log.logging("Plugin", "Status","Z4D starting deConz") - self.ControllerLink= ZigpyTransport( - self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "deCONZ", Parameters["SerialPort"] - ) - self.ControllerLink.open_cie_connection() + self.log.logging("Plugin", "Status","Z4D starting deConz") + + try: + self.ControllerLink= ZigpyTransport( + self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "deCONZ", Parameters["SerialPort"] + ) + self.ControllerLink.open_cie_connection() + except Exception as e: + self.log.logging("Plugin", "Error", f"Failed to start Zigpy deConz: {str(e)}") self.pluginconf.pluginConf["ControllerInRawMode"] = True @@ -1131,8 +1129,6 @@ def _start_zigpy_EZSP(self): from Classes.ZigpyTransport.Transport import ZigpyTransport - #self.pythonModuleVersion["zigpy"] = (zigpy.__version__) - #self.pythonModuleVersion["zigpy_ezsp"] = (bellows.__version__) check_python_modules_version( self ) self.zigbee_communication = "zigpy" self.pluginParameters["Zigpy"] = True @@ -1146,10 +1142,13 @@ def _start_zigpy_EZSP(self): SerialPort = Parameters["SerialPort"] - self.ControllerLink= ZigpyTransport( - self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "ezsp", SerialPort - ) - self.ControllerLink.open_cie_connection() + try: + self.ControllerLink= ZigpyTransport( + self.ControllerData, self.pluginParameters, self.pluginconf,self.processFrame, self.zigpy_chk_upd_device, self.zigpy_get_device, self.zigpy_backup_available, self.restart_plugin, self.log, self.statistics, self.HardwareID, "ezsp", SerialPort + ) + self.ControllerLink.open_cie_connection() + except Exception as e: + self.log.logging("Plugin", "Error", f"Failed to start Zigpy EZSP: {str(e)}") self.pluginconf.pluginConf["ControllerInRawMode"] = True @@ -1567,7 +1566,51 @@ def debuging_information(self, mode): for info_name, info_value in debug_info.items(): self.log.logging("Plugin", mode, "%s: %s" % (info_name, info_value)) - +def _remove_device_entry(self, DeviceID, Unit, device_name): + """ + Removes an device and performs associated cleanup tasks if required + """ + NwkId = self.IEEE2NWK[DeviceID] + self.log.logging("Plugin", "Status", f"Removing Device {DeviceID} {device_name} in progress") + + # Check that we don't have any reference in plugin + fully_removed = removeDeviceInList(self, Devices, DeviceID, Unit) + + # Let's check that we still don't have a reference in Domoticz . This could happen when a Replace is done. + fully_removed = fully_removed and not is_device_ieee_in_domoticz_db(self, Devices, DeviceID) + + if fully_removed: + _cleanup_device(self, DeviceID, NwkId, device_name) + + self.log.logging("Plugin", "Debug", f"ListOfDevices: After REMOVE {self.ListOfDevices}") + load_list_of_domoticz_widget(self, Devices) + +def _cleanup_device(self, DeviceID, NwkId, device_name): + """ + Performs cleanup operations for a removed device. + """ + if self.groupmgt: + self.groupmgt.RemoveNwkIdFromAllGroups(NwkId) + + # Send leave request to the device + leaveRequest(self, ShortAddr=NwkId, IEEE=DeviceID) + + # Ensure removal from the coordinator + zigate_remove_device(self, str(self.ControllerIEEE), str(DeviceID)) + self.log.logging( + "Plugin", + "Status", + f"Request device {device_name} -> {DeviceID} to be removed from coordinator", + ) + +def _remove_group_entry(self, Unit, DeviceID): + """ + Removes a group from groups list. + """ + self.log.logging("Plugin", "Status", f"Request device {DeviceID} to be removed from Group(s)") + self.groupmgt.FullRemoveOfGroup(Unit, DeviceID) + + global _plugin # pylint: disable=global-variable-not-assigned _plugin = BasePlugin() diff --git a/requirements.txt b/requirements.txt index e1b8bd5b9..4ade2523c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,11 @@ zigpy_deconz bellows requests distro +zigpy_znp +zigpy_deconz +bellows +dnspython +pyserial +charset-normalizer +jsonschema +cryptography diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index b3ddfcdb1..000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,3 +0,0 @@ -sonar.issue.ignore.multicriteria=e1 -sonar.issue.ignore.multicriteria.e1.ruleKey=python:S5332 -sonar.issue.ignore.multicriteria.e1.resourceKey=**