Merge remote-tracking branch 'upstream/main' into SOLR-16876

.git-blame-ignore-revs

@@ -30,3 +30,6 @@ f08f7bb3ef90381078e427f0d164a9f13afe070c
 7f7033e24169652303b7f1feb382b3d77320f9ec
 c99af207c761ec34812ef1cc3054eb2804b7448b
 
+# SOLR-17026 [google java formatter upgrade]
+8b9954317690428b13e1c6c4ed1dbdb743ab389b
+

.github/workflows/bin-solr-test.yml

@@ -15,6 +15,9 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
     steps:
     # Setup
     - uses: actions/checkout@v2

.github/workflows/docker-test.yml

@@ -20,6 +20,7 @@ jobs:
     env:
       SOLR_DOCKER_IMAGE_REPO: github-pr/solr
       SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}}
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
 
     steps:
     # Setup

.github/workflows/gradle-precommit.yml

@@ -11,6 +11,9 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
     steps:
     # Setup
     - uses: actions/checkout@v2

.github/workflows/solrj-test.yml

@@ -14,6 +14,9 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    env:
+      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+
     steps:
     # Setup
     - uses: actions/checkout@v2

.github/workflows/tests-via-crave.yml

@@ -9,13 +9,18 @@ jobs:
   test:
     name: Run Solr Tests using Crave.io resources
 
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
 
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Get the Crave binary
-      run: curl -s https://raw.githubusercontent.com/accupara/crave/master/get_crave.sh | bash -s --
+    - name: Destroy previous clone
+      run: crave clone destroy -y /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER} || echo "Clone did not exist"
+      continue-on-error: true
+    - name: Crave clone sources
+      run: crave clone create --projectID 39 /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}
+    - name: Checkout the correct branch
+      run: git -C /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}/solr checkout ${GITHUB_HEAD_REF} || echo "${GITHUB_HEAD_REF} not present on solr repo, staying at main"
+      continue-on-error: true
     - name: Initialize, build, test
-      run: ./crave run --clean
+      run: crave run --clean
+    - name: Delete Clone
+      run: crave clone destroy -y /crave-devspaces/pipeline/runs/${GITHUB_RUN_ID}_${GITHUB_RUN_NUMBER}

build.gradle

@@ -19,15 +19,15 @@ import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
 plugins {
-  id "base"
-  id "com.palantir.consistent-versions" version "2.11.0"
-  id "org.owasp.dependencycheck" version "8.0.1"
-  id 'ca.cutterslade.analyze' version "1.9.0"
+  id 'base'
+  id 'com.palantir.consistent-versions' version '2.16.0'
+  id 'org.owasp.dependencycheck' version '8.4.0'
+  id 'ca.cutterslade.analyze' version '1.9.1'
   id 'de.thetaphi.forbiddenapis' version '3.6' apply false
-  id "de.undercouch.download" version "5.2.0" apply false
-  id "net.ltgt.errorprone" version "3.0.1" apply false
-  id 'com.diffplug.spotless' version "6.5.2" apply false
-  id 'com.github.node-gradle.node' version '3.4.0' apply false
+  id 'de.undercouch.download' version '5.5.0' apply false
+  id 'net.ltgt.errorprone' version '3.1.0' apply false
+  id 'com.diffplug.spotless' version '6.5.2' apply false
+  id 'com.github.node-gradle.node' version '7.0.1' apply false
 }
 
 apply from: file('gradle/globals.gradle')

buildSrc/scriptDepVersions.gradle

@@ -21,11 +21,11 @@
 
 ext {
   scriptDepVersions = [
-      "apache-rat": "0.14",
-      "commons-codec": "1.15",
-      "ecj": "3.30.0",
+      "apache-rat": "0.15",
+      "commons-codec": "1.16.0",
+      "ecj": "3.33.0",
       "javacc": "7.0.12",
-      "jgit": "5.13.1.202206130422-r",
-      "flexmark": "0.64.0",
+      "jgit": "6.7.0.202309050840-r",
+      "flexmark": "0.64.8",
   ]
 }

buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -61,8 +61,8 @@ public static void main(String[] args) {
 
   public static void checkVersion() {
     int major = Runtime.getRuntime().version().feature();
-    if (major < 11 || major > 19) {
-      throw new IllegalStateException("java version must be between 11 and 19, your version: " + major);
+    if (major < 11 || major > 21) {
+      throw new IllegalStateException("java version must be between 11 and 21, your version: " + major);
     }
   }
 

dev-tools/doap/solr.rdf

@@ -68,6 +68,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+         <Version>
+           <name>solr-9.4.0</name>
+           <created>2023-10-14</created>
+           <revision>9.4.0</revision>
+         </Version>
+    </release>
     <release>
          <Version>
            <name>solr-9.3.0</name>

dev-tools/scripts/addVersion.py

@@ -44,7 +44,7 @@ def edit(buffer, match, line):
   print('done' if changed else 'uptodate')
 
 def update_solrversion_class(new_version):
-  filename = 'solr/core/src/java/org/apache/solr/util/SolrVersion.java'
+  filename = 'solr/api/src/java/org/apache/solr/client/api/util/SolrVersion.java'
   print('  changing version to %s...' % new_version.dot, end='', flush=True)
   constant_prefix = 'public static final String LATEST_STRING = "(.*?)"'
   matcher = re.compile(constant_prefix)
@@ -104,7 +104,7 @@ def edit(buffer, match, line):
 
 def check_solr_version_class_tests():
   print('  checking solr version tests...', end='', flush=True)
-  run('./gradlew -p solr/core test --tests TestSolrVersion')
+  run('./gradlew -p solr/api test --tests TestSolrVersion')
   print('ok')
 
 def check_lucene_match_version_tests():

dev-tools/scripts/buildAndPushRelease.py

@@ -128,7 +128,7 @@ def prepare(root, version, pause_before_sign, mf_username, gpg_key_id, gpg_passw
   if dev_mode:
     cmd += ' -Pvalidation.git.failOnModified=false'
   if gpg_key_id is None:
-    cmd += ' -Psign=false -x signJarsPublication'  # Disable signing if no key provided to script
+    cmd += ' -Psign=false'  # Disable signing if no key provided to script
   else:
     cmd += ' -Psign --max-workers 2'
     if sign_gradle:

dev-tools/scripts/cloud.sh

@@ -280,10 +280,11 @@ copyTarball() {
       curl -o "$RC_FILE" "$SMOKE_RC_URL"
       pushd
     else
-      if [[ ! -f $(ls "$VCS_WORK"/solr/packaging/build/distributions/solr-*.tgz) ]]; then
+      TARBALL=$(find "$VCS_WORK" -regex '.*/solr-.*\.tgz' | grep -v slim)
+      if [[ ! -f "$TARBALL" ]]; then
         echo "No solr tarball found try again with -r"; popd; exit 10;
       fi
-      cp "$VCS_WORK"/solr/packaging/build/distributions/solr-*.tgz ${CLUSTER_WD}
+      cp "$TARBALL" "${CLUSTER_WD}"
     fi
     pushd # back into cluster wd to unpack
     tar xzvf solr-*.tgz
@@ -315,7 +316,6 @@ start(){
     # Need a fresh root in zookeeper...
     "${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd makepath "/solr_${SAFE_DEST}";
     "${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd put "/solr_${SAFE_DEST}" "created by cloud.sh"; # so we can test for existence next time
-    "${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd putfile "/solr_${SAFE_DEST}/solr.xml" "${SOLR}/server/solr/solr.xml";
   fi
 
   ACTUAL_NUM_NODES=$(ls -1 -d ${CLUSTER_WD}/n* | wc -l )

dev-tools/scripts/releaseWizard.yaml

@@ -205,7 +205,7 @@ groups:
   title: Prerequisites
   description: |
     Releasing software requires thorough understanding of the process and careful execution,
-    as it is easy to make mistakes. It also requires an environtment and tools such as gpg
+    as it is easy to make mistakes. It also requires an environment and tools such as gpg
     correctly setup. This section makes sure you're in good shape for the job!
   todos:
   - !Todo
@@ -526,7 +526,7 @@ groups:
     - major
     - minor
     links:
-    - https://cwiki.apache.org/confluence/display/LUCENEJAVA/JenkinsReleaseBuilds
+    - https://cwiki.apache.org/confluence/display/SOLR/JenkinsReleaseBuilds+-+Solr
   - !Todo
     id: inform_devs
     title: Inform Devs of the new Release Branch
@@ -544,7 +544,7 @@ groups:
 
       NOTICE:
 
-      Branch {{ release_branch }} has been cut and versions updated to {{ release_version_major }}.{{ release_version_minor + 1 }} on stable branch.
+      Branch {{ release_branch }} has been cut and versions updated to {{ release_version_major }}.{{ release_version_minor + 1 }} on the stable branch.
 
       Please observe the normal rules:
 
@@ -636,7 +636,7 @@ groups:
       Ask on dev@ for input. Ideally the timing of this request mostly coincides with the
       release branch creation. It's a good idea to remind the devs of this later in the release too.
 
-      NOTE: Do not add every single JIRA issue, but distill the Release note into important changes!
+      NOTE: Do not add every single JIRA issue, but distill the release notes into important changes!
     links:
     - https://cwiki.apache.org/confluence/display/SOLR/Release+Notes
   - !Todo
@@ -660,7 +660,7 @@ groups:
     description: |-
       To make sure a new release is not announced before the reference guide is complete, add a new
       Blocker JIRA issue for version {{ release_version }} with subject
-      "Complete Major changes and Upgrade Notes in RefGudie for {{ release_version }}".
+      "Complete Major changes and Upgrade Notes in RefGuide for {{ release_version }}".
       Note that this does not block the RC and voting process, but will block announcing the release.
     types:
     - major
@@ -1588,6 +1588,7 @@ groups:
       Fill in the same date that you used for the release in previous steps.
       Do not use a product name prefix for the version, as this is the main release of the Solr PMC.
       Just use the version of this release: {{ release_version }}
+      Note: you have to be a member of the Solr PMC to be able to complete this step.
     links:
       - https://reporter.apache.org/addrelease.html?solr
 - !TodoGroup

gradle/ge.gradle

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def isCIBuild = System.getenv().keySet().find { it ==~ /(?i)((JENKINS|HUDSON)(_\w+)?|CI)/ } != null
+
+gradleEnterprise {
+    server = "https://ge.apache.org"
+    buildScan {
+        capture { taskInputFiles = true }
+        uploadInBackground = !isCIBuild
+        publishAlways()
+        publishIfAuthenticated()
+        obfuscation {
+            ipAddresses { addresses -> addresses.collect { address -> "0.0.0.0"} }
+        }
+    }
+}
+
+buildCache {
+    local {
+        enabled = !isCIBuild
+    }
+
+    remote(gradleEnterprise.buildCache) {
+        enabled = false
+    }
+}

gradle/hacks/turbocharge-jvm-opts.gradle

@@ -52,8 +52,8 @@ allprojects {
                 // are not part of up-to-date checks but these are internal JVM flags so we
                 // don't care.
                 //
-                // Pass VM options via -J only with a custom javaHome AND when java toolchains are not used.
-                if (task.options.forkOptions.javaHome != null && task.javaCompiler.getOrNull() == null) {
+                // Pass VM options via -J when a custom javaHome is used and we're in fork mode.
+                if (task.options.fork && task.options.forkOptions.javaHome != null) {
                     return vmOpts.collect {"-J" + it}
                 } else {
                     return vmOpts

gradle/maven/defaults-maven.gradle

@@ -177,7 +177,7 @@ configure(subprojects.findAll { it.path in rootProject.published }) { prj ->
         if (project(":solr:distribution").ext.useGpgForSigning) {
           useGpgCmd()
         }
-        required { !version.endsWith("SNAPSHOT") }
+        required { project(":solr:distribution").ext.withSignedArtifacts }
         sign publishing.publications.jars
       }
     }

gradle/node.gradle

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-configure([project(":solr:packaging"), project(":solr:solr-ref-guide")]) {
+configure([project(":solr:packaging"), project(":solr:solr-ref-guide"), project(":solr:webapp")]) {
     apply plugin: "com.github.node-gradle.node"
 
     ext {

gradle/testing/defaults-tests.gradle

@@ -89,7 +89,7 @@ allprojects {
 
     tasks.withType(Test) {
       ext {
-        testOutputsDir = file("${reports.junitXml.destination}/outputs")
+        testOutputsDir = file("${reports.junitXml.outputLocation.get()}/outputs")
       }
 
       // LUCENE-9660: Make it possible to always rerun tests, even if they're incrementally up-to-date.

gradle/validation/check-environment.gradle

@@ -22,7 +22,7 @@ import org.gradle.util.GradleVersion
 
 configure(rootProject) {
   ext {
-    expectedGradleVersion = '7.6'
+    expectedGradleVersion = '8.4'
   }
 
   wrapper {

gradle/validation/error-prone.gradle

@@ -17,8 +17,8 @@
 
 def skipReason
 
-if (rootProject.usesAltJvm && rootProject.runtimeJavaVersion > JavaVersion.VERSION_15) {
-  skipReason = "won't work with JDK ${rootProject.runtimeJavaVersion} if used as alternative java toolchain"
+if (rootProject.usesAltJvm) {
+  skipReason = "won't work with alternative java toolchain"
 }
 
 if (!propertyOrDefault("validation.errorprone", isCIBuild).asBoolean()) {
@@ -37,7 +37,7 @@ if (skipReason) {
 
 allprojects { prj ->
   plugins.withType(JavaPlugin) {
-    // LUCENE-9650: Errorprone on master/gradle does not work with JDK-16+ when running as plugin
+    // LUCENE-9650: Errorprone on master/gradle does not work when running as plugin
     // inside a forked Javac process. Javac running inside Gradle works, because we have
     // additional module system opens in place.
     // This is a hack to keep the dependency (so that palantir's version check doesn't complain)
@@ -305,6 +305,7 @@ allprojects { prj ->
             // '-Xep:DoNotMockAutoValue:OFF', // we don't use autovalue
             // '-Xep:DoNotUseRuleChain:OFF', // todo could be fixed but not easy
             // '-Xep:DoubleCheckedLocking:OFF', // todo check if useful or comment why not
+            '-Xep:DuplicateDateFormatField:WARN',
             '-Xep:EmptyBlockTag:WARN',
             // '-Xep:EmptyCatch:OFF', // todo check if useful or comment why not - might be handled by ECJ?
             // '-Xep:EmptySetMultibindingContributions:OFF', // we don't use this annotation
@@ -420,6 +421,7 @@ allprojects { prj ->
             // '-Xep:NonAtomicVolatileUpdate:OFF', // todo check if useful or comment why not
             '-Xep:NonCanonicalType:WARN',
             '-Xep:NonOverridingEquals:WARN',
+            //'-Xep:NonFinalStaticField:WARN', // todo there are problems that should be fixed
             '-Xep:NotJavadoc:WARN',
             '-Xep:NullOptional:WARN',
             // '-Xep:NullableConstructor:OFF', // we don't use this annotation
@@ -464,7 +466,10 @@ allprojects { prj ->
             // '-Xep:StreamResourceLeak:OFF', // todo check if useful or comment why not
             '-Xep:StreamToIterable:WARN',
             // '-Xep:StringCaseLocaleUsage:OFF', // we have forbiddenapis for that
+            '-Xep:StringCharset:WARN',
+            '-Xep:StringFormatWithLiteral:WARN',
             // '-Xep:StringSplitter:OFF', // todo check if useful or comment why not - might be able to use forbidden-apis for this?
+            '-Xep:SuperEqualsIsObjectEquals:WARN',
             // '-Xep:SwigMemoryLeak:OFF', // we don't use swig
             // '-Xep:SynchronizeOnNonFinalField:OFF', // todo check if useful or comment why not
             // '-Xep:ThreadJoinLoop:OFF', // todo check if useful or comment why not

gradle/validation/rat-sources.gradle

@@ -60,6 +60,7 @@ allprojects {
 
             // Exclude Eclipse
             exclude ".metadata"
+            exclude ".settings"
 
             // Include selected patterns from any source folders. We could make this
             // relative to source sets but it seems to be of little value - all our source sets

gradle/validation/spotless.gradle

@@ -36,7 +36,7 @@ configure(project(":solr").subprojects) { prj ->
         // it.licenseHeaderFile(file("${resources}/asl-header.txt"), '^(\\s*package)')
         it.setLineEndings(Enum.valueOf(rootProject.buildscript.classLoader.loadClass("com.diffplug.spotless.LineEnding"), "UNIX"))
         it.endWithNewline()
-        it.googleJavaFormat('1.15.0')
+        it.googleJavaFormat('1.18.1')
 
         it.custom('Refuse wildcard imports', { line ->
           // Wildcard imports can't be resolved by spotless itself.