From 10ab2eb92eff340af007e0301a2aa674b4813b26 Mon Sep 17 00:00:00 2001
From: Pipiche <8291674+pipiche38@users.noreply.github.com>
Date: Tue, 24 Dec 2024 10:33:45 +0100
Subject: [PATCH] Implement use of python virtual environment (#1816)

* retreive python version and give it as a parameter for the upgrade scritp
* Print PYTHONPATH at pluguin start
* refactor and handle python venv
* Adding a status message at startup, when the analytics infos are sent
* Update to handle debian 12
* refactoring the cleanup of the plugin repository when having git errors
---
 Classes/WebServer/rest_PluginUpgrade.py |   7 +-
 Tools/big-clean-git.sh                  |  19 --
 Tools/clean-git.sh                      |   6 -
 Tools/cleaning-repository.sh            |  50 ++++++
 Tools/plugin-auto-upgrade.sh            | 227 ++++++++++++++++--------
 Tools/printLOD.py                       |  90 ++++++----
 plugin.py                               |   8 +
 7 files changed, 269 insertions(+), 138 deletions(-)
 delete mode 100644 Tools/big-clean-git.sh
 delete mode 100644 Tools/clean-git.sh
 create mode 100644 Tools/cleaning-repository.sh

diff --git a/Classes/WebServer/rest_PluginUpgrade.py b/Classes/WebServer/rest_PluginUpgrade.py
index bdd16d390..25176a1dd 100644
--- a/Classes/WebServer/rest_PluginUpgrade.py
+++ b/Classes/WebServer/rest_PluginUpgrade.py
@@ -14,6 +14,7 @@
 import os
 import subprocess  # nosec
 from pathlib import Path
+import sys
 
 import distro
 import z4d_certified_devices
@@ -35,10 +36,14 @@ def rest_plugin_upgrade(self, verb, data, parameters):
     pluginFolder = Path(self.pluginParameters["HomeFolder"])
     upgrade_script = str( pluginFolder / PLUGIN_UPGRADE_SCRIPT)
 
+    # Identify the current Python version
+    python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
+    self.logging("Log", f"Current Python version: {python_version}")
+
     self.logging("Log", "Plugin Upgrade starting: %s" %(upgrade_script))
     
     process = subprocess.run( 
-        upgrade_script ,
+        f"{upgrade_script} {python_version}",
         cwd=self.pluginParameters["HomeFolder"],
         universal_newlines=True,
         text=True,
diff --git a/Tools/big-clean-git.sh b/Tools/big-clean-git.sh
deleted file mode 100644
index a42dd8624..000000000
--- a/Tools/big-clean-git.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-
-rm -rf .flake8 .github .gitmodules IMPORTANT.md CONTRIBUTING.md LICENSE.txt MANIFEST.in
-rm -rf Conf/Certified
-rm -rf Classes
-rm -rf Modules
-rm -rf Tools
-rm -rf Zigbee
-rm -rf Zigate-Firmware
-rm -rf bellows
-rm -rf zigpy
-rm -rf zigpy_znp
-rm -rf zigpy_deconz
-rm -rf external
-
-
-git config --add submodule.recurse true
-git reset --hard
diff --git a/Tools/clean-git.sh b/Tools/clean-git.sh
deleted file mode 100644
index eabd6055b..000000000
--- a/Tools/clean-git.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-#Cleans and resets a git repo and its submodules
-
-git reset --hard
-git submodule sync --recursive
-git submodule update --init --force --recursive
diff --git a/Tools/cleaning-repository.sh b/Tools/cleaning-repository.sh
new file mode 100644
index 000000000..a7c83bde8
--- /dev/null
+++ b/Tools/cleaning-repository.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# This script performs a cleanup in the local repository to allow a git pull without issues.
+
+# Ensure the script is run from the correct directory
+if [ ! -d ".git" ]; then
+    echo "This script must be run from the root of the git repository."
+    exit 1
+fi
+
+# Warning message
+echo "WARNING: This script will remove all local changes and reset the repository."
+echo "If you have made any local updates, they will be removed."
+read -p "Do you want to continue? (YES/no): " choice
+
+if [ "$choice" != "YES" ]; then
+    echo "Operation cancelled."
+    exit 0
+fi
+
+echo ""
+echo "Removing directories tracked by git, except Data, Conf, and OTAFirmware..."
+# Remove directories tracked by git, except Data and Conf
+for dir in $(git ls-tree -d --name-only HEAD); do
+    if [ "$dir" != "Data" ] && [ "$dir" != "Conf" ] && [ "$dir" != "OTAFirmware" ]; then
+        echo "Removing $dir..."
+        rm -rf "$dir"
+    fi
+done
+
+echo "Resetting the repository..."
+git reset --hard
+
+echo "Pulling the latest changes from the repository..."
+git pull
+
+echo ""
+echo "Cleanup and update process completed."
+
+echo ""
+echo "Repository status:"
+git status
+
+echo ""
+echo "Current branch:"
+git branch --show-current
+
+echo ""
+echo "Latest commits:"
+git log -n 5 --oneline
\ No newline at end of file
diff --git a/Tools/plugin-auto-upgrade.sh b/Tools/plugin-auto-upgrade.sh
index 2835c04d5..95c02a614 100755
--- a/Tools/plugin-auto-upgrade.sh
+++ b/Tools/plugin-auto-upgrade.sh
@@ -7,85 +7,158 @@ exec 2>&1
 echo "Starting Zigbee for Domoticz plugin Upgrade process."
 echo "----------------------------------------------------"
 
-
-if [ -z ${HOME} ]; then
-    export HOME=$(pwd)
-fi
-
-env
-echo " "
-
-/usr/bin/id
-echo " "
-
-/usr/bin/whoami
-echo " "
-
-PIP_OPTIONS="--no-input install -r requirements.txt --ignore-requires-python --upgrade"
-
-if command -v lsb_release &> /dev/null; then
-    DISTRIB_ID=$(lsb_release -is)
-    DISTRIB_RELEASE=$(lsb_release -rs)
-    if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
-        PIP_OPTIONS="$PIP_OPTIONS --break-system-packages"
+# Function to set HOME environment variable if not set
+set_home() {
+    if [ -z ${HOME} ]; then
+        export HOME=$(pwd)
     fi
-fi
-echo "PIP Options: $PIP_OPTIONS"
-
-
-
-echo "Current version  : $(cat .hidden/VERSION)"
-echo "latest git commit: $(git log --pretty=oneline -1)"
-echo ""
-
-echo "(1) git config --global --add safe.directory"
-git config  --global --add safe.directory $(pwd)
-git config  --global --add safe.directory $(pwd)/external/zigpy
-git config  --global --add safe.directory $(pwd)/external/zigpy-znp
-git config  --global --add safe.directory $(pwd)/external/zigpy-zigate
-git config  --global --add safe.directory $(pwd)/external/zigpy-deconz
-git config  --global --add safe.directory $(pwd)/external/bellows
-
-echo " "
-echo "(2) updating Zigbee for Domoticz plugin"
-echo ""
-echo "Setup submodule.recurse $(git config --add submodule.recurse true)"
-echo ""
-git pull --recurse-submodules
-#git pull --recurse-submodules  && git submodule update --recursive
-ret="$?"
-if [ "$ret" != "0" ] ; then
-    echo "ERROR while running command 'git pull --recurse-submodules'."
-    echo "Git Status: $(git status)"
-    exit -1
-fi
-
-echo " "
-echo "(3) update python3 modules if needed"
-echo ""
-if [ "$(whoami)" == "root" ]; then
-    # Si l'utilisateur est root
-    python3 -m pip $PIP_OPTIONS
-else
-    # Si l'utilisateur n'est pas root, utilisez sudo
-    sudo python3 -m pip $PIP_OPTIONS
-fi
-ret="$?"
-if [ "$ret" != "0" ] ; then
-    echo "ERROR while running command 'sudo python3 -m pip --no-input install -r requirements.txt --ignore-requires-python --upgrade'."
-    echo "Is sudo available for this user without password ?"
-    exit -2
-fi
-
-echo " "
-echo "(4) git config --global --unset safe.directory"
-git config --global --unset-all safe.directory $(pwd)/external/bellows
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-deconz
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-zigate
-git config --global --unset-all safe.directory $(pwd)/external/zigpy-znp
-git config --global --unset-all safe.directory $(pwd)/external/zigpy
-git config --global --unset-all safe.directory $(pwd)
+}
+
+# Function to print environment details
+print_env_details() {
+    env
+    echo " "
+    /usr/bin/id
+    echo " "
+    /usr/bin/whoami
+    echo " "
+}
+
+# Function to set PIP options based on the distribution
+set_pip_options() {
+    PIP_OPTIONS="--no-input install -r requirements.txt --ignore-requires-python --upgrade"
+    if command -v lsb_release &> /dev/null; then
+        DISTRIB_ID=$(lsb_release -is)
+        DISTRIB_RELEASE=$(lsb_release -rs)
+        if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
+            PIP_OPTIONS="$PIP_OPTIONS --break-system-packages"
+        fi
+    fi
+    echo "PIP Options: $PIP_OPTIONS"
+}
+
+# Function to check if pip is installed in the virtual environment
+check_pip_in_venv() {
+    if [ ! -f "$VENV_PATH/bin/$PYTHON_VERSION" ]; then
+        echo "pip is not installed in the virtual environment. Installing pip..."
+        install_pip
+        $PYTHON_VERSION -m venv $VENV_PATH
+    fi
+}
+
+# Function to install pip
+install_pip() {
+    if command -v lsb_release &> /dev/null && [ "$(lsb_release -is)" = "Debian" ] || [ "$(lsb_release -is)" = "Ubuntu" ]; then
+        echo "We are expecting the user to properly install python3-pip package. if not yet done !!"
+    else
+        $PYTHON_VERSION -m ensurepip
+        $PYTHON_VERSION -m pip install --upgrade pip virtualenv -t $VENV_PATH
+    fi
+}
+
+# Function to activate virtual environment
+activate_venv() {
+    echo "Using virtual environment at: $VENV_PATH"
+    source $VENV_PATH/bin/activate
+}
+
+# Function to check and activate virtual environment
+check_and_activate_venv() {
+    if [ -n "$PYTHONPATH" ]; then
+        echo "PYTHONPATH is set to: $PYTHONPATH"
+        VENV_PATH=$(echo $PYTHONPATH | cut -d':' -f1)
+        if [ -d "$VENV_PATH/bin" ]; then
+            check_pip_in_venv
+        else
+            echo "Virtual environment path $VENV_PATH does not exist"
+            echo "pip is not installed in the virtual environment. Installing pip..."
+            install_pip
+            $PYTHON_VERSION -m venv $VENV_PATH
+        fi
+        VENV_ACTIVATED=true
+        activate_venv
+    else
+        echo "PYTHONPATH is not set"
+        VENV_ACTIVATED=false
+    fi
+}
+
+# Function to install python3-pip on Debian if necessary
+install_pip_on_debian() {
+    if command -v lsb_release &> /dev/null; then
+        DISTRIB_ID=$(lsb_release -is)
+        DISTRIB_RELEASE=$(lsb_release -rs)
+        if [ "$DISTRIB_ID" = "Debian" ] && [ "$DISTRIB_RELEASE" = "12" ]; then
+            if ! command -v pip3 &> /dev/null; then
+                echo "pip3 is not installed. Installing python3-pip..."
+                sudo apt-get update
+                sudo apt-get install -y python3-pip
+            fi
+        fi
+    fi
+}
+
+# Function to update git configuration
+update_git_config() {
+    echo "(1) git config --global --add safe.directory"
+    git config  --global --add safe.directory $(pwd)
+}
+
+# Function to update python modules
+update_python_modules() {
+    echo " "
+    echo "(2) update $PYTHON_VERSION modules if needed"
+    echo ""
+    if [ "$VENV_ACTIVATED" = true ]; then
+        $VENV_PATH/bin/python3 -m pip $PIP_OPTIONS -t $VENV_PATH
+    else
+        if [ "$(whoami)" == "root" ]; then
+            $PYTHON_VERSION -m pip $PIP_OPTIONS
+        else
+            sudo $PYTHON_VERSION -m pip $PIP_OPTIONS
+        fi
+    fi
+    ret="$?"
+    if [ "$ret" != "0" ] ; then
+        echo "ERROR while running command '$PYTHON_VERSION -m pip $PIP_OPTIONS'."
+        echo "Is sudo available for this user without password ?"
+        exit -2
+    fi
+}
+
+# Function to print current version and latest git commit
+print_version_info() {
+    echo "Current version  : $(cat .hidden/VERSION)"
+    echo "latest git commit: $(git log --pretty=oneline -1)"
+    echo ""
+}
+
+# Main script execution
+PYTHON_VERSION="python${1:-3}"
+PIP_VERSION="python${1:-3}"
+
+set_home
+print_env_details
+set_pip_options
+#install_pip_on_debian
+check_and_activate_venv
+print_version_info
+update_git_config
+update_python_modules
 
 echo " "
 echo "Plugin Upgrade process completed without errors."
 exit 0
+
+# Documentation:
+# This script automates the upgrade process for the Zigbee for Domoticz plugin.
+# It performs the following steps:
+# 1. Sets the HOME environment variable if not already set.
+# 2. Prints environment details for debugging purposes.
+# 3. Sets PIP options based on the distribution.
+# 4. Checks if PYTHONPATH is set and activates the virtual environment if available.
+# 5. Installs python3-pip on Debian if necessary.
+# 6. Updates the git configuration to add the current directory as a safe directory.
+# 7. Updates Python modules using pip.
+# 8. Prints the current version and latest git commit of the plugin.
+# 9. Completes the upgrade process and exits.
diff --git a/Tools/printLOD.py b/Tools/printLOD.py
index 6bb49798a..44233661d 100644
--- a/Tools/printLOD.py
+++ b/Tools/printLOD.py
@@ -1,37 +1,57 @@
-import os.path
-
-
-while 1:
-    print("Enter the DeviceList.txt filename: ")
-    filename=input()
-    if os.path.exists(filename):
-        break
-
-
-nb = 0
-with open( filename, 'r') as myfile2:
-    for line in myfile2:
-        if not line.strip() :
-            #Empty line
-            continue
-        (key, val) = line.split(":",1)
-        key = key.replace(" ","")
-        key = key.replace("'","")
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Implementation of Zigbee for Domoticz plugin.
+#
+# This file is part of Zigbee for Domoticz plugin. https://github.com/zigbeefordomoticz/Domoticz-Zigbee
+# (C) 2015-2024
+#
+# Initial authors: zaraki673 & pipiche38
+#
+# SPDX-License-Identifier:    GPL-3.0 license
 
-        dlVal=eval(val)
-        print("%-10s %s" %('NwkID', key))
-        for i, j in dlVal.items():
-            if 'Ep' == i:
-                # Ep {'01': {'0000': {}, 'ClusterType': {'576': 'ColorControl'}, '0003': {}, '0004': {}, '0005': {}, '0006': '00', '0008': {}, '0300': {}, '0b05': {}, '1000': {}}}
-                print("Ep")
-                j = eval(str(j))
-                for k,l in j.items():
-                    print("           %-10s %s" %(k,l))
-            else:
-                print("%-10s %s" %(i,j))
-
-        print("======")
-
-
-myfile2.close()
+import os.path
 
+def main():
+    """
+    Main function to prompt the user for a filename and process the DeviceList.txt file.
+    """
+    while True:
+        print("Enter the DeviceList.txt filename: ")
+        filename = input()
+        if os.path.exists(filename):
+            break
+
+    process_file(filename)
+
+def process_file(filename):
+    """
+    Process the given DeviceList.txt file and print its contents in a formatted manner.
+
+    Args:
+        filename (str): The name of the DeviceList.txt file to process.
+    """
+    with open(filename, 'r') as myfile2:
+        for line in myfile2:
+            if not line.strip():
+                # Empty line
+                continue
+            key, val = line.split(":", 1)
+            key = key.replace(" ", "").replace("'", "")
+
+            dlVal = eval(val)
+            print("%-10s %s" % ('NwkID', key))
+            for i, j in dlVal.items():
+                if i == 'Ep':
+                    # Ep {'01': {'0000': {}, 'ClusterType': {'576': 'ColorControl'}, '0003': {}, '0004': {}, '0005': {}, '0006': '00', '0008': {}, '0300': {}, '0b05': {}, '1000': {}}}
+                    print("Ep")
+                    j = eval(str(j))
+                    for k, l in j.items():
+                        print("           %-10s %s" % (k, l))
+                else:
+                    print("%-10s %s" % (i, j))
+
+            print("======")
+
+if __name__ == "__main__":
+    main()
diff --git a/plugin.py b/plugin.py
index 0c245cae1..ddcc61886 100644
--- a/plugin.py
+++ b/plugin.py
@@ -310,6 +310,13 @@ def __init__(self):
 
     def onStart(self):
         Domoticz.Status( "Welcome to Zigbee for Domoticz (Z4D) plugin.")
+
+        # Print PYTHONPATH if set
+        pythonpath = os.getenv('PYTHONPATH')
+        if pythonpath:
+            Domoticz.Status(f"PYTHONPATH is set to: {pythonpath}")
+        else:
+            Domoticz.Status("PYTHONPATH is not set")
         
         _current_python_version_major = sys.version_info.major
         _current_python_version_minor = sys.version_info.minor
@@ -1343,6 +1350,7 @@ def zigateInit_Phase3(self):
         self.iaszonemgt.setZigateIEEE(self.ControllerIEEE)
     
     if self.internet_available and self.pluginconf.pluginConf["MatomoOptIn"]:
+        self.log.logging("Plugin", "Status", "Sending Analytics information. (disable the MatomoOptIn parameter to stop this)")
         matomo_plugin_started(self)
 
     if self.internet_available and self.pluginconf.pluginConf["MatomoOptIn"]: