Merge branch 'develop' into chore/text/set-is-graded-false-student-at…

…hena-request

.github/PULL_REQUEST_TEMPLATE.md

@@ -111,7 +111,7 @@ Prerequisites:
 - [ ] Test 2
 
 ### Test Coverage
-<!-- Please add the test coverages for all changed files modified in this PR here. You can use `supporting_script/generate_code_cov_table/generate_code_cov_table.py` to automatically generate the coverage table from the corresponding artefacts of your branch (follow the ReadMe for setup details). -->
+<!-- Please add the test coverages for all changed files modified in this PR here. You can use `supporting_script/code-coverage/generate_code_cov_table/generate_code_cov_table.py` to automatically generate the coverage table from the corresponding artefacts of your branch (follow the ReadMe for setup details). -->
 <!-- Alternatively you can execute the tests locally (see build.gradle and package.json) or look into the corresponding artefacts. -->
 <!-- The line coverage must be above 90% for changes files, and you must use extensive and useful assertions for server tests and expect statements for client tests. -->
 <!-- Note: Confirm in the last column that you have implemented extensive assertions for server tests and expect statements for client tests. -->

.github/dependabot.yml

@@ -57,7 +57,7 @@ updates:
 
   # Check for version updates for Python dependencies (coverage)
   - package-ecosystem: "pip"
-    directory: "/supporting_scripts/generate_code_cov_table"
+    directory: "/supporting_scripts/code-coverage/generate_code_cov_table"
     schedule:
       interval: "weekly"
     reviewers:

.github/workflows/test.yml

@@ -60,7 +60,32 @@ jobs:
           ${{ env.java }}
         cache: 'gradle'
     - name: Java Tests
-      run: set -o pipefail && ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log
+      run: |
+        set -o pipefail
+
+        DEFAULT_BRANCH="${{ github.event.repository.default_branch }}"
+        CURRENT_BRANCH="${{ github.ref_name }}"
+
+        if [[ "$DEFAULT_BRANCH" != "$CURRENT_BRANCH" ]]; then
+          # Explicitly fetch as the clone action only clones the current branch
+          git fetch origin "$DEFAULT_BRANCH"
+
+          chmod +x ./supporting_scripts/get_changed_modules.sh
+          CHANGED_MODULES=$(./supporting_scripts/get_changed_modules.sh "origin/$DEFAULT_BRANCH")
+
+          # Restrict executed tests to changed modules if there is diff between this and the base branch
+          if [ -n "${CHANGED_MODULES}" ]; then
+            IFS=,
+            TEST_MODULE_TAGS=$(echo "-DincludeModules=${CHANGED_MODULES[*]}")
+
+            echo "Executing tests for modules: $CHANGED_MODULES"
+            ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification "$TEST_MODULE_TAGS" | tee tests.log
+            exit 0
+          fi
+        fi
+
+        echo "Executing all tests"
+        ./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification | tee tests.log
     - name: Print failed tests
       if: failure()
       run: grep "Test >.* FAILED\$" tests.log || echo "No failed tests."
@@ -92,8 +117,13 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: Coverage Report Server Tests
-        path: build/reports/jacoco/test/html/
-
+        path: build/reports/jacoco/test/html
+    - name: Append Per-Module Coverage to Job Summary
+      if: success() || failure()
+      run: |
+        AGGREGATED_REPORT_FILE=./module_coverage_report.md
+        python3 ./supporting_scripts/code-coverage/per_module_cov_report/parse_module_coverage.py build/reports/jacoco $AGGREGATED_REPORT_FILE
+        cat $AGGREGATED_REPORT_FILE > $GITHUB_STEP_SUMMARY
 
   server-tests-mysql:
       needs: [ server-tests ]

.gitignore

@@ -220,3 +220,4 @@ data-exports/
 # Supporting scripts config
 ##############################
 /supporting_scripts/**/*.ini
+/legal

build.gradle

@@ -2,7 +2,9 @@ buildscript {
     dependencies {
         classpath "com.diffplug.spotless:spotless-plugin-gradle:${spotless_plugin_version}"
         // This is required so that the latest version of the liquibase gradle plugin works
-        classpath "org.liquibase:liquibase-core:${liquibase_version}"
+        classpath("org.liquibase:liquibase-core:${liquibase_version}") {
+            exclude group: 'com.opencsv', module: 'opencsv'
+        }
     }
 }
 
@@ -19,8 +21,9 @@ plugins {
     id "com.google.cloud.tools.jib" version "3.4.4"
     id "com.gorylenko.gradle-git-properties" version "2.4.2"
     id "io.spring.dependency-management" version "1.1.7"
+    id "nebula.lint" version "20.5.5"
     id "org.liquibase.gradle" version "${liquibase_plugin_version}"
-    id "org.owasp.dependencycheck" version "11.1.1"
+    id "org.owasp.dependencycheck" version "12.0.0"
     id "org.springframework.boot" version "${spring_boot_version}"
 }
 
@@ -94,13 +97,13 @@ repositories {
 }
 
 configurations.configureEach {
-//    exclude group: "org.dom4j", module: "dom4j"
+    exclude group: "org.dom4j", module: "dom4j"
     exclude group: "org.xmlpull", module: "pull-parser"
     exclude group: "jaxen", module: "jaxen"
     exclude group: "xmlpull", module: "xpp3"
     exclude group: "xsdlib", module: "xsdlib"
     exclude group: "javax.xml.stream", module: "stax-api"
-//    exclude group: "javax.xml.bind", module: "jaxb-api"
+    exclude group: "javax.xml.bind", module: "jaxb-api"
 
     exclude group: "org.junit.vintage", module: "junit-vintage-engine"
     exclude group: "com.vaadin.external.google", module: "android-json"
@@ -123,27 +126,39 @@ configurations.configureEach {
     exclude group: "org.apache.lucene", module: "lucene-core"
     exclude group: "org.apache.lucene", module: "lucene-analyzers-common"
     exclude group: "com.google.protobuf", module: "protobuf-java"
+
+    exclude group: "org.jasypt", module: "jasypt"
+
+    // required by eureka client, but not used in this project
+    exclude group: "com.thoughtworks.xstream", module: "xstream"
+    // required by JPlag, but not used in this project
+    exclude group: "xerces", module: "xercesImpl"
+    // required by JPlag, but not used in this project
+    exclude group: "xalan", module: "xalan"
+    // required by JPlag, but not used in this project
+    exclude group: "xalan", module: "serializer"
+
+    exclude group: "org.springframework.boot", module: "spring-boot-starter-cache"
+    exclude group: "io.micrometer", module: "micrometer-registry-prometheus"
+    exclude group: "net.logstash.logback", module: "logstash-logback-encoder"
+
+    exclude group: "javax.cache", module: "cache-api"
+    exclude group: "javax.transaction", module: "javax.transaction-api"
+
+    // JPlag depends on those, but they are not really needed
+    exclude group: "org.jgrapht", module: "jgrapht-core"
+    exclude group: "org.apfloat", module: "apfloat"
+
+    // only support one csv parser org.apache.commons:commons-csv
+//    exclude group: "com.opencsv", module: "opencsv"
 }
 
 dependencies {
 
-    // Note: jenkins-client is not well maintained and includes dependencies to libraries with critical security issues (e.g. CVE-2020-10683 for [email protected])
-    //    implementation "com.offbytwo.jenkins:jenkins-client:0.3.8"
-    implementation files("libs/jenkins-client-0.4.1.jar")
-    // The following 4 dependencies are explicitly integrated as transitive dependencies of jenkins-client-0.4.0.jar
+    // Required by Spring cloud
     implementation "org.apache.httpcomponents.client5:httpclient5:5.4.1"
-    implementation "org.apache.httpcomponents.core5:httpcore5:5.3.1"
+    implementation "org.apache.httpcomponents.core5:httpcore5:5.3.2"
     implementation "org.apache.httpcomponents:httpmime:4.5.14"
-    implementation("org.dom4j:dom4j:2.1.4") {
-        // Note: avoid org.xml.sax.SAXNotRecognizedException: unrecognized feature http://xml.org/sax/features/external-general-entities
-        // also see https://github.com/dom4j/dom4j/issues/99
-        exclude module: "pull-parser"
-        exclude module: "jaxen"
-        exclude module: "xpp3"
-        exclude module: "xsdlib"
-        exclude module: "stax-api"
-        exclude module: "jaxb-api"
-    }
 
     implementation "org.gitlab4j:gitlab4j-api:6.0.0-rc.8"
 
@@ -172,8 +187,10 @@ dependencies {
         }
     }
 
+    // Sentry depends on this
     implementation "org.apache.logging.log4j:log4j-to-slf4j:2.24.3"
 
+    // Used for LTI (e.g. Moodle --> Artemis)
     implementation "uk.ac.ox.ctl:spring-security-lti13:0.3.0"
 
     // https://search.maven.org/artifact/org.eclipse.jgit/org.eclipse.jgit
@@ -189,7 +206,6 @@ dependencies {
 
     // https://mvnrepository.com/artifact/net.sourceforge.plantuml/plantuml
     implementation "net.sourceforge.plantuml:plantuml:1.2024.8"
-    implementation "org.jasypt:jasypt:1.9.3"
     implementation "me.xdrop:fuzzywuzzy:1.4.0"
     implementation("org.yaml:snakeyaml") {
         version {
@@ -205,19 +221,12 @@ dependencies {
     // NOTE: the following six dependencies use the newer versions explicitly to avoid other dependencies to use older versions
     implementation "ch.qos.logback:logback-classic:${logback_version}"
     implementation "ch.qos.logback:logback-core:${logback_version}"
-    // required by eureka client
-    implementation "com.thoughtworks.xstream:xstream:1.4.21"
-    // required by JPlag, should NOT be used in other places
-    implementation "xerces:xercesImpl:2.12.2"
-    // required by JPlag, should NOT be used in other places
-    implementation "xalan:xalan:2.7.3"
-    // required by JPlag, should NOT be used in other places
-    implementation "xalan:serializer:2.7.3"
+
     // required by Saml2, should NOT be used in other places
     implementation "org.apache.santuario:xmlsec:4.0.3"
 
     implementation "org.jsoup:jsoup:1.18.3"
-    implementation "commons-codec:commons-codec:1.17.1"   // needed for spring security saml2
+    implementation "commons-codec:commons-codec:1.17.2"   // needed for spring security saml2
 
     // use the latest version to avoid security vulnerabilities
     implementation "org.springframework:spring-webmvc:${spring_framework_version}"
@@ -231,14 +240,7 @@ dependencies {
     // use newest version of commons-compress to avoid security issues through outdated dependencies
     implementation "org.apache.commons:commons-compress:1.27.1"
 
-
-    // import JHipster dependencies BOM
-    implementation platform("tech.jhipster:jhipster-dependencies:${jhipster_dependencies_version}")
-
     implementation "tech.jhipster:jhipster-framework:${jhipster_dependencies_version}"
-    implementation "org.springframework.boot:spring-boot-starter-cache:${spring_boot_version}"
-    implementation "io.micrometer:micrometer-registry-prometheus:1.14.2"
-    implementation "net.logstash.logback:logstash-logback-encoder:8.0"
 
     // Defines low-level streaming API, and includes JSON-specific implementations
     implementation "com.fasterxml.jackson.core:jackson-core:${fasterxml_version}"
@@ -256,21 +258,24 @@ dependencies {
     implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${fasterxml_version}"
 
 
+    // Required for synchronization between nodes and build agents (LocalCI)
     implementation "com.hazelcast:hazelcast:${hazelcast_version}"
     implementation "com.hazelcast:hazelcast-spring:${hazelcast_version}"
-    implementation "com.hazelcast:hazelcast-hibernate53:5.2.0"
+    // Required for Hibernate multi node caching
+    runtimeOnly "com.hazelcast:hazelcast-hibernate53:5.2.0"
 
-    implementation "javax.cache:cache-api:1.1.1"
     implementation "org.hibernate.orm:hibernate-core:${hibernate_version}"
 
+    // Required for jdbc connection pooling to databases
     implementation "com.zaxxer:HikariCP:6.2.1"
 
+    // Required for several dependencies
     implementation "org.apache.commons:commons-text:1.13.0"
     implementation "org.apache.commons:commons-math3:3.6.1"
 
-    implementation "javax.transaction:javax.transaction-api:1.3"
-
-    implementation "org.liquibase:liquibase-core:${liquibase_version}"
+    implementation("org.liquibase:liquibase-core:${liquibase_version}") {
+        exclude group: 'com.opencsv', module: 'opencsv'
+    }
 
     implementation "org.springframework.boot:spring-boot-starter-validation:${spring_boot_version}"
     implementation "org.springframework.boot:spring-boot-loader-tools:${spring_boot_version}"
@@ -289,15 +294,16 @@ dependencies {
     implementation "org.springframework.boot:spring-boot-starter-oauth2-client:${spring_boot_version}"
 
     implementation "org.springframework.ldap:spring-ldap-core:3.2.10"
-    implementation "org.springframework.data:spring-data-ldap:3.4.1"
+    implementation "org.springframework.data:spring-data-ldap:3.4.2"
 
     implementation "org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:${spring_cloud_version}"
     implementation "org.springframework.cloud:spring-cloud-starter-config:${spring_cloud_version}"
     implementation "org.springframework.cloud:spring-cloud-commons:${spring_cloud_version}"
 
-    implementation "io.netty:netty-all:4.1.116.Final"
-    implementation "io.projectreactor.netty:reactor-netty:1.2.1"
+    // required by the Websocket Broker Connection in WebsocketConfiguration (due to multi node setup support)
+    implementation "io.projectreactor.netty:reactor-netty:1.2.2"
     implementation "org.springframework:spring-messaging:${spring_framework_version}"
+    // required for the connection to Hermes (push notifications)
     implementation "org.springframework.retry:spring-retry:2.0.11"
 
     implementation "org.springframework.security:spring-security-config:${spring_security_version}"
@@ -324,34 +330,38 @@ dependencies {
     runtimeOnly "io.jsonwebtoken:jjwt-impl:${jwt_version}"
     runtimeOnly "io.jsonwebtoken:jjwt-jackson:${jwt_version}"
 
-    implementation "org.bouncycastle:bcpkix-jdk18on:1.79"
-    implementation "org.bouncycastle:bcprov-jdk18on:1.79"
+    // required by sshd-git
+    implementation "org.bouncycastle:bcpkix-jdk18on:1.80"
+    implementation "org.bouncycastle:bcprov-jdk18on:1.80"
 
     implementation "com.mysql:mysql-connector-j:${mysql_version}"
-    implementation "org.postgresql:postgresql:42.7.4"
+    implementation "org.postgresql:postgresql:42.7.5"
 
     implementation "org.zalando:problem-spring-web:0.29.1"
     implementation "org.zalando:jackson-datatype-problem:0.27.1"
+    // Required by JPlag
     implementation "com.ibm.icu:icu4j-charset:76.1"
+    // Required by exam session service
     implementation "com.github.seancfoley:ipaddress:5.5.1"
+
+    // used for testing and Java Template Upgrade Service
     implementation "org.apache.maven:maven-model:3.9.9"
     implementation "org.apache.pdfbox:pdfbox:3.0.3"
-    implementation "org.apache.commons:commons-csv:1.12.0"
+    implementation "org.apache.commons:commons-csv:1.13.0"
     implementation "org.commonmark:commonmark:0.24.0"
     implementation "commons-fileupload:commons-fileupload:1.5"
     implementation "net.lingala.zip4j:zip4j:2.11.5"
 
-    implementation "org.jgrapht:jgrapht-core:1.5.2"
-    // use the latest version explicitly to avoid security vulnerabilities (currently Artemis and JPlag rely on jgrapht 1.5.2 which relies on apfloat)
-    implementation "org.apfloat:apfloat:1.14.0"
-
     // use newest version of guava to avoid security issues through outdated dependencies
     implementation "com.google.guava:guava:33.4.0-jre"
     implementation "com.sun.activation:jakarta.activation:2.0.1"
 
     // use newest version of gson to avoid security issues through outdated dependencies
     implementation "com.google.code.gson:gson:2.11.0"
 
+    // explicitly only use this at runtime, because developers should prefer org.apache.commons:csv
+    runtimeOnly "com.opencsv:opencsv:5.10"
+
 
     implementation "com.google.errorprone:error_prone_annotations:2.36.0"
 
@@ -409,7 +419,6 @@ dependencies {
     testImplementation "org.apache.maven.shared:maven-invoker:3.3.0"
     testImplementation "org.gradle:gradle-tooling-api:8.12"
     testImplementation "org.apache.maven.surefire:surefire-report-parser:3.5.2"
-    testImplementation "com.opencsv:opencsv:5.9"
     testImplementation "io.zonky.test:embedded-database-spring-test:2.6.0"
 
     testImplementation "com.tngtech.archunit:archunit:1.3.0"
@@ -420,7 +429,7 @@ dependencies {
     testImplementation "com.h2database:h2:2.2.224"
 
     // Lightweight JSON library needed for the internals of the MockRestServiceServer
-    testImplementation "org.json:json:20241224"
+    testImplementation "org.json:json:20250107"
 
     // NOTE: make sure this corresponds to the version used for JUnit in the testImplementation
     testRuntimeOnly "org.junit.platform:junit-platform-launcher:${junit_platform_version}"
@@ -457,6 +466,12 @@ checkstyle {
     maxErrors = 0
 }
 
+gradleLint {
+    rules  = ['dependency-parentheses']
+//    criticalRules = ['unused-dependency'] // <-- this will fail the build in the event of a violation
+}
+
+
 def isNonStable = { String version ->
     def stableKeyword = ["RELEASE", "FINAL", "GA"].any { it -> version.toUpperCase().contains(it) }
     def regex = /^[0-9,.v-]+(-r)?$/
@@ -479,8 +494,9 @@ tasks.named("dependencyUpdates").configure {
 // 2) Execute tests with coverage report:           ./gradlew test jacocoTestReport -x webapp
 // 2a) Execute tests without coverage report:       ./gradlew test -x webapp
 // 2b) Run a single test:                           ./gradlew test --tests ExamIntegrationTest -x webapp or ./gradlew test --tests ExamIntegrationTest.testGetExamScore -x webapp
-// 2c) Execute tests with Postgres container:       SPRING_PROFILES_INCLUDE=postgres ./gradlew test -x webapp
-// 2d) Execute tests with MySQL container:          SPRING_PROFILES_INCLUDE=mysql ./gradlew test -x webapp
+// 2c) Run tests for modules:                       ./gradlew test -DincludeModules=athena,atlas -x webapp (executes all tests in directories ./src/main/test/.../athena and ./src/main/test/.../atlas) + ArchitectureTests
+// 2d) Execute tests with Postgres container:       SPRING_PROFILES_INCLUDE=postgres ./gradlew test -x webapp
+// 2e) Execute tests with MySQL container:          SPRING_PROFILES_INCLUDE=mysql ./gradlew test -x webapp
 // 3) Verify code coverage (after tests):           ./gradlew jacocoTestCoverageVerification
 // 4) Check Java code format:                       ./gradlew spotlessCheck -x webapp
 // 5) Apply Java code formatter:                    ./gradlew spotlessApply -x webapp
@@ -491,3 +507,4 @@ tasks.named("dependencyUpdates").configure {
 // 10) Clear Liquibase checksums:                   ./gradlew liquibaseClearChecksums
 // 11) Create changelog between Java and DB         ./gradlew liquibaseDiffChangeLog (make sure to set the correct username and password in liquibase.gradle)
 // 12) Generate initial schema from DB              ./gradlew liquibaseGenerateChangelog (make sure to set the correct username and password in liquibase.gradle)
+// 13) Find unused dependencies                     ./gradlew lintGradle -x webapp

check.sh

@@ -0,0 +1,14 @@
+set -o pipefail
+
+chmod +x ./changed-modules.sh
+CHANGED_MODULES=$(./changed-modules.sh)
+
+if [ -n "${CHANGED_MODULES}" ]; then
+  # Always execute ArchitectureTests
+  CHANGED_MODULES+=("ArchitectureTest")
+
+  IFS=,
+  TEST_MODULE_TAGS=$(echo "-DtestTags=${CHANGED_MODULES[*]}")
+fi
+
+echo "./gradlew --console=plain test jacocoTestReport -x webapp jacocoTestCoverageVerification $TEST_MODULE_TAGS | tee tests.log"