From 9fba5eff6a650b60eb473590317893d79432bf5a Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 12:37:16 +0100
Subject: [PATCH 1/7] retreive python version and give it as a parameter for
 the upgrade scritp

---
 Classes/WebServer/rest_PluginUpgrade.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Classes/WebServer/rest_PluginUpgrade.py b/Classes/WebServer/rest_PluginUpgrade.py
index bdd16d390..25176a1dd 100644
--- a/Classes/WebServer/rest_PluginUpgrade.py
+++ b/Classes/WebServer/rest_PluginUpgrade.py
@@ -14,6 +14,7 @@
 import os
 import subprocess  # nosec
 from pathlib import Path
+import sys
 
 import distro
 import z4d_certified_devices
@@ -35,10 +36,14 @@ def rest_plugin_upgrade(self, verb, data, parameters):
     pluginFolder = Path(self.pluginParameters["HomeFolder"])
     upgrade_script = str( pluginFolder / PLUGIN_UPGRADE_SCRIPT)
 
+    # Identify the current Python version
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+    self.logging("Log", f"Current Python version: {python_version}")
+
     self.logging("Log", "Plugin Upgrade starting: %s" %(upgrade_script))
     
     process = subprocess.run( 
-        upgrade_script ,
+        f"{upgrade_script} {python_version}",
         cwd=self.pluginParameters["HomeFolder"],
         universal_newlines=True,
         text=True,

From de4f55ea06e28c13210457f04360430eab704d46 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 12:45:48 +0100
Subject: [PATCH 2/7] Print PYTHONPATH at pluguin start

---
 plugin.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/plugin.py b/plugin.py
index 0c245cae1..7ec53b997 100644
--- a/plugin.py
+++ b/plugin.py
@@ -310,6 +310,13 @@ def __init__(self):
 
     def onStart(self):
         Domoticz.Status( "Welcome to Zigbee for Domoticz (Z4D) plugin.")
+
+        # Print PYTHONPATH if set
+        pythonpath = os.getenv('PYTHONPATH')
+        if pythonpath:
+            Domoticz.Status(f"PYTHONPATH is set to: {pythonpath}")
+        else:
+            Domoticz.Status("PYTHONPATH is not set")
         
         _current_python_version_major = sys.version_info.major
         _current_python_version_minor = sys.version_info.minor

From 00b9c2bf49b8d138560ff7c5fc434d3952bea168 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 12:46:46 +0100
Subject: [PATCH 3/7] refactor and handle python venv

---
 Tools/plugin-auto-upgrade.sh | 194 +++++++++++++++++++++--------------
 1 file changed, 118 insertions(+), 76 deletions(-)

diff --git a/Tools/plugin-auto-upgrade.sh b/Tools/plugin-auto-upgrade.sh
index 2835c04d5..f4092fb09 100755
--- a/Tools/plugin-auto-upgrade.sh
+++ b/Tools/plugin-auto-upgrade.sh
@@ -7,85 +7,127 @@ exec 2>&1
 echo "Starting Zigbee for Domoticz plugin Upgrade process."
 echo "----------------------------------------------------"
 
-
-if [ -z ${HOME} ]; then
-    export HOME=$(pwd)
-fi
-
-env
-echo " "
-
-/usr/bin/id
-echo " "
-
-/usr/bin/whoami
-echo " "
-
-PIP_OPTIONS="--no-input install -r requirements.txt --ignore-requires-python --upgrade"
-
-if command -v lsb_release &> /dev/null; then
-    DISTRIB_ID=$(lsb_release -is)
-    DISTRIB_RELEASE=$(lsb_release -rs)
-    if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
-        PIP_OPTIONS="$PIP_OPTIONS --break-system-packages"
+# Function to set HOME environment variable if not set
+set_home() {
+    if [ -z ${HOME} ]; then
+        export HOME=$(pwd)
+    fi
+}
+
+# Function to print environment details
+print_env_details() {
+    env
+    echo " "
+    /usr/bin/id
+    echo " "
+    /usr/bin/whoami
+    echo " "
+}
+
+# Function to set PIP options based on the distribution
+set_pip_options() {
+    PIP_OPTIONS="--no-input install -r requirements.txt --ignore-requires-python --upgrade"
+    if command -v lsb_release &> /dev/null; then
+        DISTRIB_ID=$(lsb_release -is)
+        DISTRIB_RELEASE=$(lsb_release -rs)
+        if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
+            PIP_OPTIONS="$PIP_OPTIONS --break-system-packages"
+        fi
+    fi
+    echo "PIP Options: $PIP_OPTIONS"
+}
+
+# Function to check and activate virtual environment
+check_and_activate_venv() {
+    if [ -n "$PYTHONPATH" ]; then
+        echo "PYTHONPATH is set to: $PYTHONPATH"
+        VENV_PATH=$(echo $PYTHONPATH | cut -d':' -f1)
+        if [ -d "$VENV_PATH/bin" ]; then
+            VENV_ACTIVATED=true
+            if [ ! -f "$VENV_PATH/bin/$PYTHON_VERSION" ]; then
+                echo "pip is not installed in the virtual environment. Installing pip..."
+                $PYTHON_VERSION -m ensurepip
+                $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
+                $PYTHON_VERSION -m venv $VENV_PATH
+            fi
+        else
+            echo "Virtual environment path $VENV_PATH does not exist"
+            echo "pip is not installed in the virtual environment. Installing pip..."
+            $PYTHON_VERSION -m ensurepip
+            $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
+            $PYTHON_VERSION -m venv $VENV_PATH
+            VENV_ACTIVATED=true
+        fi
+    else
+        echo "PYTHONPATH is not set"
+        VENV_ACTIVATED=false
     fi
-fi
-echo "PIP Options: $PIP_OPTIONS"
-
-
-
-echo "Current version  : $(cat .hidden/VERSION)"
-echo "latest git commit: $(git log --pretty=oneline -1)"
-echo ""
-
-echo "(1) git config --global --add safe.directory"
-git config  --global --add safe.directory $(pwd)
-git config  --global --add safe.directory $(pwd)/external/zigpy
-git config  --global --add safe.directory $(pwd)/external/zigpy-znp
-git config  --global --add safe.directory $(pwd)/external/zigpy-zigate
-git config  --global --add safe.directory $(pwd)/external/zigpy-deconz
-git config  --global --add safe.directory $(pwd)/external/bellows
-
-echo " "
-echo "(2) updating Zigbee for Domoticz plugin"
-echo ""
-echo "Setup submodule.recurse $(git config --add submodule.recurse true)"
-echo ""
-git pull --recurse-submodules
-#git pull --recurse-submodules  && git submodule update --recursive
-ret="$?"
-if [ "$ret" != "0" ] ; then
-    echo "ERROR while running command 'git pull --recurse-submodules'."
-    echo "Git Status: $(git status)"
-    exit -1
-fi
-
-echo " "
-echo "(3) update python3 modules if needed"
-echo ""
-if [ "$(whoami)" == "root" ]; then
-    # Si l'utilisateur est root
-    python3 -m pip $PIP_OPTIONS
-else
-    # Si l'utilisateur n'est pas root, utilisez sudo
-    sudo python3 -m pip $PIP_OPTIONS
-fi
-ret="$?"
-if [ "$ret" != "0" ] ; then
-    echo "ERROR while running command 'sudo python3 -m pip --no-input install -r requirements.txt --ignore-requires-python --upgrade'."
-    echo "Is sudo available for this user without password ?"
-    exit -2
-fi
 
-echo " "
-echo "(4) git config --global --unset safe.directory"
-git config --global --unset-all safe.directory $(pwd)/external/bellows
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-deconz
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-zigate
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-znp
-git config --global --unset-all safe.directory $(pwd)/external/zigpy
-git config --global --unset-all safe.directory $(pwd)
+    if [ "$VENV_ACTIVATED" = true ]; then
+        echo "Using virtual environment at: $VENV_PATH"
+        source $VENV_PATH/bin/activate
+    fi
+}
+
+# Function to update git configuration
+update_git_config() {
+    echo "(1) git config --global --add safe.directory"
+    git config  --global --add safe.directory $(pwd)
+}
+
+# Function to update python modules
+update_python_modules() {
+    echo " "
+    echo "(2) update $PYTHON_VERSION modules if needed"
+    echo ""
+    if [ "$VENV_ACTIVATED" = true ]; then
+        $VENV_PATH/bin/python3 -m pip $PIP_OPTIONS -t $VENV_PATH
+    else
+        if [ "$(whoami)" == "root" ]; then
+            $PYTHON_VERSION -m pip $PIP_OPTIONS
+        else
+            sudo $PYTHON_VERSION -m pip $PIP_OPTIONS
+        fi
+    fi
+    ret="$?"
+    if [ "$ret" != "0" ] ; then
+        echo "ERROR while running command '$PYTHON_VERSION -m pip $PIP_OPTIONS'."
+        echo "Is sudo available for this user without password ?"
+        exit -2
+    fi
+}
+
+# Function to print current version and latest git commit
+print_version_info() {
+    echo "Current version  : $(cat .hidden/VERSION)"
+    echo "latest git commit: $(git log --pretty=oneline -1)"
+    echo ""
+}
+
+# Main script execution
+PYTHON_VERSION="python${1:-3}"
+PIP_VERSION="python${1:-3}"
+
+set_home
+print_env_details
+set_pip_options
+check_and_activate_venv
+print_version_info
+update_git_config
+update_python_modules
 
 echo " "
 echo "Plugin Upgrade process completed without errors."
 exit 0
+
+# Documentation:
+# This script automates the upgrade process for the Zigbee for Domoticz plugin.
+# It performs the following steps:
+# 1. Sets the HOME environment variable if not already set.
+# 2. Prints environment details for debugging purposes.
+# 3. Sets PIP options based on the distribution.
+# 4. Checks if PYTHONPATH is set and activates the virtual environment if available.
+# 5. Updates the git configuration to add the current directory as a safe directory.
+# 6. Updates Python modules using pip.
+# 7. Prints the current version and latest git commit of the plugin.
+# 8. Completes the upgrade process and exits.

From 4394ed7ed481b9c427b0efb2abbb9b77ed789c03 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 13:07:01 +0100
Subject: [PATCH 4/7] Adding a status message at startup, when the analytics
 infos are sent

---
 plugin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugin.py b/plugin.py
index 7ec53b997..ddcc61886 100644
--- a/plugin.py
+++ b/plugin.py
@@ -1350,6 +1350,7 @@ def zigateInit_Phase3(self):
         self.iaszonemgt.setZigateIEEE(self.ControllerIEEE)
     
     if self.internet_available and self.pluginconf.pluginConf["MatomoOptIn"]:
+        self.log.logging("Plugin", "Status", "Sending Analytics information. (disable the MatomoOptIn parameter to stop this)")
         matomo_plugin_started(self)
 
     if self.internet_available and self.pluginconf.pluginConf["MatomoOptIn"]:

From 1aa957a461022b99b3be72313634b787b5c0fd74 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 16:17:04 +0100
Subject: [PATCH 5/7] Update to handle debian 12

---
 Tools/plugin-auto-upgrade.sh | 65 ++++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 17 deletions(-)

diff --git a/Tools/plugin-auto-upgrade.sh b/Tools/plugin-auto-upgrade.sh
index f4092fb09..95c02a614 100755
--- a/Tools/plugin-auto-upgrade.sh
+++ b/Tools/plugin-auto-upgrade.sh
@@ -37,35 +37,64 @@ set_pip_options() {
     echo "PIP Options: $PIP_OPTIONS"
 }
 
+# Function to check if pip is installed in the virtual environment
+check_pip_in_venv() {
+    if [ ! -f "$VENV_PATH/bin/$PYTHON_VERSION" ]; then
+        echo "pip is not installed in the virtual environment. Installing pip..."
+        install_pip
+        $PYTHON_VERSION -m venv $VENV_PATH
+    fi
+}
+
+# Function to install pip
+install_pip() {
+    if command -v lsb_release &> /dev/null && [ "$(lsb_release -is)" = "Debian" ] || [ "$(lsb_release -is)" = "Ubuntu" ]; then
+        echo "We are expecting the user to properly install python3-pip package. if not yet done !!"
+    else
+        $PYTHON_VERSION -m ensurepip
+        $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
+    fi
+}
+
+# Function to activate virtual environment
+activate_venv() {
+    echo "Using virtual environment at: $VENV_PATH"
+    source $VENV_PATH/bin/activate
+}
+
 # Function to check and activate virtual environment
 check_and_activate_venv() {
     if [ -n "$PYTHONPATH" ]; then
         echo "PYTHONPATH is set to: $PYTHONPATH"
         VENV_PATH=$(echo $PYTHONPATH | cut -d':' -f1)
         if [ -d "$VENV_PATH/bin" ]; then
-            VENV_ACTIVATED=true
-            if [ ! -f "$VENV_PATH/bin/$PYTHON_VERSION" ]; then
-                echo "pip is not installed in the virtual environment. Installing pip..."
-                $PYTHON_VERSION -m ensurepip
-                $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
-                $PYTHON_VERSION -m venv $VENV_PATH
-            fi
+            check_pip_in_venv
         else
             echo "Virtual environment path $VENV_PATH does not exist"
             echo "pip is not installed in the virtual environment. Installing pip..."
-            $PYTHON_VERSION -m ensurepip
-            $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
+            install_pip
             $PYTHON_VERSION -m venv $VENV_PATH
-            VENV_ACTIVATED=true
         fi
+        VENV_ACTIVATED=true
+        activate_venv
     else
         echo "PYTHONPATH is not set"
         VENV_ACTIVATED=false
     fi
+}
 
-    if [ "$VENV_ACTIVATED" = true ]; then
-        echo "Using virtual environment at: $VENV_PATH"
-        source $VENV_PATH/bin/activate
+# Function to install python3-pip on Debian if necessary
+install_pip_on_debian() {
+    if command -v lsb_release &> /dev/null; then
+        DISTRIB_ID=$(lsb_release -is)
+        DISTRIB_RELEASE=$(lsb_release -rs)
+        if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
+            if ! command -v pip3 &> /dev/null; then
+                echo "pip3 is not installed. Installing python3-pip..."
+                sudo apt-get update
+                sudo apt-get install -y python3-pip
+            fi
+        fi
     fi
 }
 
@@ -111,6 +140,7 @@ PIP_VERSION="python${1:-3}"
 set_home
 print_env_details
 set_pip_options
+#install_pip_on_debian
 check_and_activate_venv
 print_version_info
 update_git_config
@@ -127,7 +157,8 @@ exit 0
 # 2. Prints environment details for debugging purposes.
 # 3. Sets PIP options based on the distribution.
 # 4. Checks if PYTHONPATH is set and activates the virtual environment if available.
-# 5. Updates the git configuration to add the current directory as a safe directory.
-# 6. Updates Python modules using pip.
-# 7. Prints the current version and latest git commit of the plugin.
-# 8. Completes the upgrade process and exits.
+# 5. Installs python3-pip on Debian if necessary.
+# 6. Updates the git configuration to add the current directory as a safe directory.
+# 7. Updates Python modules using pip.
+# 8. Prints the current version and latest git commit of the plugin.
+# 9. Completes the upgrade process and exits.

From 4ae4e87e4fe3da20d7c81a7e0fa2417f95dc99e2 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 16:47:41 +0100
Subject: [PATCH 6/7] refactoring the cleanup of the plugin repository when
 having git errors

---
 Tools/big-clean-git.sh       | 19 --------------
 Tools/clean-git.sh           |  6 -----
 Tools/cleaning-repository.sh | 50 ++++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 25 deletions(-)
 delete mode 100644 Tools/big-clean-git.sh
 delete mode 100644 Tools/clean-git.sh
 create mode 100644 Tools/cleaning-repository.sh

diff --git a/Tools/big-clean-git.sh b/Tools/big-clean-git.sh
deleted file mode 100644
index a42dd8624..000000000
--- a/Tools/big-clean-git.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-rm -rf .flake8 .github .gitmodules IMPORTANT.md CONTRIBUTING.md LICENSE.txt MANIFEST.in
-rm -rf Conf/Certified
-rm -rf Classes
-rm -rf Modules
-rm -rf Tools
-rm -rf Zigbee
-rm -rf Zigate-Firmware
-rm -rf bellows
-rm -rf zigpy
-rm -rf zigpy_znp
-rm -rf zigpy_deconz
-rm -rf external
-
-
-git config --add submodule.recurse true
-git reset --hard
diff --git a/Tools/clean-git.sh b/Tools/clean-git.sh
deleted file mode 100644
index eabd6055b..000000000
--- a/Tools/clean-git.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-#Cleans and resets a git repo and its submodules
-
-git reset --hard
-git submodule sync --recursive
-git submodule update --init --force --recursive
diff --git a/Tools/cleaning-repository.sh b/Tools/cleaning-repository.sh
new file mode 100644
index 000000000..a7c83bde8
--- /dev/null
+++ b/Tools/cleaning-repository.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# This script performs a cleanup in the local repository to allow a git pull without issues.
+
+# Ensure the script is run from the correct directory
+if [ ! -d ".git" ]; then
+    echo "This script must be run from the root of the git repository."
+    exit 1
+fi
+
+# Warning message
+echo "WARNING: This script will remove all local changes and reset the repository."
+echo "If you have made any local updates, they will be removed."
+read -p "Do you want to continue? (YES/no): " choice
+
+if [ "$choice" != "YES" ]; then
+    echo "Operation cancelled."
+    exit 0
+fi
+
+echo ""
+echo "Removing directories tracked by git, except Data, Conf, and OTAFirmware..."
+# Remove directories tracked by git, except Data and Conf
+for dir in $(git ls-tree -d --name-only HEAD); do
+    if [ "$dir" != "Data" ] && [ "$dir" != "Conf" ] && [ "$dir" != "OTAFirmware" ]; then
+        echo "Removing $dir..."
+        rm -rf "$dir"
+    fi
+done
+
+echo "Resetting the repository..."
+git reset --hard
+
+echo "Pulling the latest changes from the repository..."
+git pull
+
+echo ""
+echo "Cleanup and update process completed."
+
+echo ""
+echo "Repository status:"
+git status
+
+echo ""
+echo "Current branch:"
+git branch --show-current
+
+echo ""
+echo "Latest commits:"
+git log -n 5 --oneline
\ No newline at end of file

From 0e86a1472e3e42e1b4ae260c265cc6b90dd36be5 Mon Sep 17 00:00:00 2001
From: Patrick Pichon <patrick@pichon.me>
Date: Thu, 19 Dec 2024 16:53:17 +0100
Subject: [PATCH 7/7] Update and document

---
 Tools/printLOD.py | 90 +++++++++++++++++++++++++++++------------------
 1 file changed, 55 insertions(+), 35 deletions(-)

diff --git a/Tools/printLOD.py b/Tools/printLOD.py
index 6bb49798a..44233661d 100644
--- a/Tools/printLOD.py
+++ b/Tools/printLOD.py
@@ -1,37 +1,57 @@
-import os.path
-
-
-while 1:
-    print("Enter the DeviceList.txt filename: ")
-    filename=input()
-    if os.path.exists(filename):
-        break
-
-
-nb = 0
-with open( filename, 'r') as myfile2:
-    for line in myfile2:
-        if not line.strip() :
-            #Empty line
-            continue
-        (key, val) = line.split(":",1)
-        key = key.replace(" ","")
-        key = key.replace("'","")
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Implementation of Zigbee for Domoticz plugin.
+#
+# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee
+# (C) 2015-2024
+#
+# Initial authors: zaraki673 & pipiche38
+#
+# SPDX-License-Identifier:    GPL-3.0 license
 
-        dlVal=eval(val)
-        print("%-10s %s" %('NwkID', key))
-        for i, j in dlVal.items():
-            if 'Ep' == i:
-                # Ep {'01': {'0000': {}, 'ClusterType': {'576': 'ColorControl'}, '0003': {}, '0004': {}, '0005': {}, '0006': '00', '0008': {}, '0300': {}, '0b05': {}, '1000': {}}}
-                print("Ep")
-                j = eval(str(j))
-                for k,l in j.items():
-                    print("           %-10s %s" %(k,l))
-            else:
-                print("%-10s %s" %(i,j))
-
-        print("======")
-
-
-myfile2.close()
+import os.path
 
+def main():
+    """
+    Main function to prompt the user for a filename and process the DeviceList.txt file.
+    """
+    while True:
+        print("Enter the DeviceList.txt filename: ")
+        filename = input()
+        if os.path.exists(filename):
+            break
+
+    process_file(filename)
+
+def process_file(filename):
+    """
+    Process the given DeviceList.txt file and print its contents in a formatted manner.
+
+    Args:
+        filename (str): The name of the DeviceList.txt file to process.
+    """
+    with open(filename, 'r') as myfile2:
+        for line in myfile2:
+            if not line.strip():
+                # Empty line
+                continue
+            key, val = line.split(":", 1)
+            key = key.replace(" ", "").replace("'", "")
+
+            dlVal = eval(val)
+            print("%-10s %s" % ('NwkID', key))
+            for i, j in dlVal.items():
+                if i == 'Ep':
+                    # Ep {'01': {'0000': {}, 'ClusterType': {'576': 'ColorControl'}, '0003': {}, '0004': {}, '0005': {}, '0006': '00', '0008': {}, '0300': {}, '0b05': {}, '1000': {}}}
+                    print("Ep")
+                    j = eval(str(j))
+                    for k, l in j.items():
+                        print("           %-10s %s" % (k, l))
+                else:
+                    print("%-10s %s" % (i, j))
+
+            print("======")
+
+if __name__ == "__main__":
+    main()