fix ci, only esp firmwre static analysis fails because it is work in

progress

.github/workflows/ci.yaml

@@ -74,6 +74,54 @@ jobs:
         run: |
           docker-compose -f docker-compose.yml -f docker-compose.ci.yml down
 
+  system-tests:
+    name: Run System Tests
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Install Docker Compose
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y docker-compose
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull Docker Image
+        run: |
+          docker pull ghcr.io/robertgawron/hardwaredatalogger:${{ github.sha }}
+          docker tag ghcr.io/robertgawron/hardwaredatalogger:${{ github.sha }} hardwaredatalogger:latest
+
+      - name: Run Docker Compose
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d
+
+      - name: Run System Tests
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.ci.yml exec -T dev bash -c "cd /workspace/build/ && cmake .. && make -j24 && cd /workspace/Test/System && pytest test_*.py -s  --html=/workspace/build/BuildArtifacts/SystemTests.html"
+
+      - name: Upload System Tests Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: system-tests-results
+          path: DevOps/BuildArtifacts/SystemTests.html
+
+      - name: Clean up Docker containers
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.ci.yml down
+        
+
   code-coverage:
     name: Run Code Coverage
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -11,14 +11,12 @@ set(USE_X86_HEADERS TRUE)  # Set this to TRUE for X86, FALSE for ARM
 add_subdirectory(Software/STM32F103RBTx/Application)
 add_subdirectory(Simulation/FirmwarePCSimulator/STM32F103RBTx)
 add_subdirectory(Simulation/FirmwarePCSimulator/ESP8266)
-
-
 add_subdirectory(Test/Unit)
 
-include(Software/CStaticAnalysis.cmake)
-include(Software/ClassDiagram.cmake)
-include(Software/DocsCoverage.cmake)
-
-include(Simulation/FirmwarePCSimulator/PythonStaticAnalysis.cmake)
+# Include the DevOps scripts (static code analysis, etc.)
+include(DevOps/Cmake/CStaticAnalysis.cmake)
+include(DevOps/Cmake/PythonStaticAnalysis.cmake)
+include(DevOps/Cmake/DocsCoverage.cmake)
+include(DevOps/Cmake/ClassDiagram.cmake)
 
 

Software/CStaticAnalysis.cmake → DevOps/Cmake/CStaticAnalysis.cmake

@@ -17,7 +17,7 @@ set(CODECHECKER_SOURCE_DIRS
 # Define the output directories for CodeChecker analysis and reports
 set(CODECHECKER_ANALYZE_DIR ${CMAKE_BINARY_DIR}/BuildArtifacts/StaticAnalysisIntermediary)
 set(CODECHECKER_REPORT_DIR ${CMAKE_BINARY_DIR}/BuildArtifacts/StaticAnalysis)
-set(CODECHECKER_SKIP_FILE ${CMAKE_SOURCE_DIR}/Software/CodeCheckerSkipList)
+set(CODECHECKER_SKIP_FILE ${CMAKE_SOURCE_DIR}/DevOps/Scripts/CodeCheckerSkipList)
 
 
 # Ensure the output directories exist

Software/DocsCoverage.cmake → DevOps/Cmake/DocsCoverage.cmake

@@ -12,8 +12,8 @@ if(DOXYGEN_EXECUTABLE)
     file(MAKE_DIRECTORY "${DOXYGEN_OUTPUT_DIR}")
 
     # Set the paths for the Doxyfile template and the final generated Doxyfile
-    set(DOXYFILE_IN "${CMAKE_CURRENT_LIST_DIR}/Doxyfile.in")
-    set(DOXYFILE_OUT "${CMAKE_CURRENT_LIST_DIR}/Doxyfile")
+    set(DOXYFILE_IN "${CMAKE_SOURCE_DIR}/DevOps/Scripts/Doxyfile.in")
+    set(DOXYFILE_OUT "${CMAKE_SOURCE_DIR}/DevOps/Scripts/Doxyfile")
 
     # Create a template Doxyfile with CMake variables
     configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY)

Simulation/FirmwarePCSimulator/PythonStaticAnalysis.cmake → DevOps/Cmake/PythonStaticAnalysis.cmake

@@ -1,7 +1,7 @@
 
 # Define the output directories for Python static analysis and reports
-set(PYTHON_ANALYZE_DIR ${CMAKE_BINARY_DIR}/BuildArtifacts/PythonStaticAnalysis)
-set(PYTHON_REPORT_DIR ${CMAKE_BINARY_DIR}/BuildArtifacts)
+set(PYTHON_ANALYZE_DIR ${CMAKE_SOURCE_DIR}/build/BuildArtifacts/PythonStaticAnalysis)
+set(PYTHON_REPORT_DIR ${CMAKE_SOURCE_DIR}/DevOps/BuildArtifacts)
 
 # Echo the variables for debugging
 message(STATUS "PYTHON_ANALYZE_DIR: ${PYTHON_ANALYZE_DIR}")
@@ -11,10 +11,14 @@ message(STATUS "PYTHON_REPORT_DIR: ${PYTHON_REPORT_DIR}")
 file(MAKE_DIRECTORY ${PYTHON_ANALYZE_DIR})
 file(MAKE_DIRECTORY ${PYTHON_REPORT_DIR})
 
+# Define a variable for the file path
+set(PYTHON_STATIC_ANALYSIS_SCRIPT "${CMAKE_SOURCE_DIR}/DevOps/Scripts/PythonStaticAnalysis.sh")
+
 # Add a custom target for Python static analysis
 add_custom_target(pystatic
     COMMAND bash -c " \
-        source /workspace/Simulation/FirmwarePCSimulator/PythonStaticAnalysis.sh"
+        dos2unix ${PYTHON_STATIC_ANALYSIS_SCRIPT} && source ${PYTHON_STATIC_ANALYSIS_SCRIPT}"
     COMMENT "Running Python static analysis with Prospector and generating reports..."
     VERBATIM
 )
+

DevOps/Dockerfile

@@ -104,6 +104,7 @@ RUN apt-get update && apt-get install -y \
     libfontconfig1 \
     libfreetype6 \
     lcov \
+    dos2unix \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 

DevOps/Scripts/.prospector.yaml

@@ -0,0 +1,21 @@
+strictness: veryhigh
+
+pydocstyle:
+  disable:
+    - D212
+    - D211
+    - D205
+    - D400
+    - D415
+    - D202
+    - D201
+  options:
+    max-line-length: 120
+
+pycodestyle:
+  options:
+    max-line-length: 105
+
+pylint:
+  options:
+    max-line-length: 105

DevOps/Scripts/CodeCheckerSkipList

@@ -0,0 +1,10 @@
+-*/Middlewares/*
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Arduino.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Client.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/ESP8266WiFi.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/HardwareSerial.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Print.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Printable.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/PubSubClient.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/stdlib_noniso.h
+-*/Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/WString.h

DevOps/Scripts/PythonStaticAnalysis.sh

@@ -0,0 +1,40 @@
+# Activate the virtual environment
+source /workspace/venv/bin/activate
+
+# Set up directories for artifacts
+OUTPUT_DIR="/workspace/build/BuildArtifacts/PythonStaticAnalysis"
+mkdir -p "$OUTPUT_DIR"
+
+# Define input and output file paths
+SYSTEM_JSON="$OUTPUT_DIR/SystemTests.json"
+SYSTEM_HTML="$OUTPUT_DIR/SystemTests.html"
+SIMULATOR_JSON="$OUTPUT_DIR/FirmwarePCSimulator.json"
+SIMULATOR_HTML="$OUTPUT_DIR/FirmwarePCSimulator.html"
+
+# Initialize exit code tracker
+EXIT_CODE=0
+
+# Run Prospector analysis
+prospector --profile /workspace/DevOps/Scripts/.prospector.yaml --strictness veryhigh --doc-warnings --output-format xunit /workspace/Test/System/ > "$SYSTEM_JSON" || EXIT_CODE=1
+prospector --profile /workspace/DevOps/Scripts/.prospector.yaml --strictness veryhigh --doc-warnings --output-format xunit /workspace/Simulation/FirmwarePCSimulator/ > "$SIMULATOR_JSON" || EXIT_CODE=1
+
+# vjunit is broken in original version, so we need to fix it
+find /workspace/venv/ -type f -name "vjunit.py" -exec sed -i 's/children = testcase\.getchildren()/children = list(testcase)/g' {} +
+
+# Convert JSON reports to HTML using prospector-html
+vjunit -f "$SYSTEM_JSON" -o "$SYSTEM_HTML" || echo "Warning: HTML conversion failed for SystemTests"
+vjunit -f "$SIMULATOR_JSON" -o "$SIMULATOR_HTML" || echo "Warning: HTML conversion failed for FirmwarePCSimulator"
+
+# Expand all tabs in the HTML reports
+sed -i "/\$('.ui.accordion').accordion();/c\\
+\$('.ui.accordion').accordion();\\
+\$('.ui.accordion .title').addClass('active');\\
+\$('.ui.accordion .content').addClass('active');" "$SYSTEM_HTML"
+
+sed -i "/\$('.ui.accordion').accordion();/c\\
+\$('.ui.accordion').accordion();\\
+\$('.ui.accordion .title').addClass('active');\\
+\$('.ui.accordion .content').addClass('active');" "$SIMULATOR_HTML"
+
+# Exit with the appropriate code
+exit $EXIT_CODE

Documentation/Manuals/SetupDockerContainers.md

@@ -36,3 +36,7 @@ When finished, shut down the Docker container:
 docker-compose down --remove-orphans
 ```
 
+Oneliner to build and start:
+
+docker-compose build dev && docker-compose up -d --remove-orphans && docker-compose exec dev bash
+

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Arduino.h

@@ -19,7 +19,7 @@
 #ifndef Arduino_h
 #define Arduino_h
 
-// Whole file was modified and maybe it was a bad decision.
+// codechecker_suppress [all] excluded from CodeChecker analysis because i need compatibility with the Arduino code
 
 #include <cstdint>
 

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Client.h

@@ -19,6 +19,9 @@
 
 #ifndef client_h
 #define client_h
+
+// codechecker_suppress [all] excluded from CodeChecker analysis because i need compatibility with the Arduino code
+
 // #include "Print.h"
 #include "Client.h"
 // #include "Stream.h"

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/ESP8266WiFi.h

@@ -36,7 +36,7 @@ extern "C" {
 #include "ESP8266WiFiScan.h"
 #include "ESP8266WiFiGeneric.h"
 */
-#include "WiFiClient.h"
+// #include "WiFiClient.h"
 
 /*#include "WiFiServer.h"
 #include "WiFiServerSecure.h"

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/Print.h

@@ -20,6 +20,8 @@
 #ifndef Print_h
 #define Print_h
 
+// codechecker_suppress [all] excluded from CodeChecker analysis because i need compatibility with the Arduino code
+
 #include <stdint.h>
 #include <stddef.h>
 

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/PubSubClient.h

@@ -7,6 +7,8 @@
 #ifndef PubSubClient_h
 #define PubSubClient_h
 
+// codechecker_suppress [all] excluded from CodeChecker analysis because i need compatibility with the Arduino code
+
 #include <Arduino.h>
 #include "Client.h"
 #include "Print.h" // not included directly in the original code, base class for PubSubClient

Simulation/FirmwarePCSimulator/ESP8266/Mocks/Inc/WString.h

@@ -23,11 +23,14 @@
 #define String_class_h
 #ifdef __cplusplus
 
-//====================== hck =================================
+//====================== hack =================================
+
+// codechecker_suppress [all] excluded from CodeChecker analysis because i need compatibility with the Arduino code
+
 /*
 #include <pgmspace.h>
 */
-//====================== hck =================================
+//====================== hack =================================
 
 #include <cstdlib>
 #include <cstdint>