Replace ANTLR with hand-rolled parser

.gitignore

@@ -5,6 +5,7 @@
 .gradle/
 build/
 generated/
+testgenerated/
 
 # IntelliJ
 .idea/

bench/bench.gradle.kts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,8 +25,6 @@ val graal: Configuration by configurations.creating
 @Suppress("UnstableApiUsage")
 dependencies {
   jmh(projects.pklCore)
-  // necessary because antlr4-runtime is declared as implementation dependency in pkl-core.gradle
-  jmh(libs.antlrRuntime)
   truffle(libs.truffleApi)
   graal(libs.graalCompiler)
 }

bench/gradle.lockfile

@@ -1,7 +1,6 @@
 # This is a Gradle generated file for dependency locking.
 # Manual edits can break the build and are not advised.
 # This file is expected to be part of source control.
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.sf.jopt-simple:jopt-simple:5.0.4=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath
 org.apache.commons:commons-math3:3.6.1=jmh,jmhCompileClasspath,jmhImplementationDependenciesMetadata,jmhRuntimeClasspath

bench/src/jmh/java/org/pkl/core/parser/ParserBenchmark.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

buildSrc/src/main/kotlin/pklFatJar.gradle.kts

@@ -39,7 +39,6 @@ val firstPartySourcesJarsConfiguration: Configuration =
 val relocations =
   mapOf(
     // pkl-core dependencies
-    "org.antlr.v4." to "org.pkl.thirdparty.antlr.v4.",
     "org.organicdesign.fp." to "org.pkl.thirdparty.paguro.",
     "org.snakeyaml.engine." to "org.pkl.thirdparty.snakeyaml.engine.",
     "org.msgpack." to "org.pkl.thirdparty.msgpack.",

docs/src/test/kotlin/DocSnippetTests.kt

@@ -16,15 +16,14 @@ import org.pkl.core.Loggers
 import org.pkl.core.SecurityManagers
 import org.pkl.core.StackFrameTransformers
 import org.pkl.core.module.ModuleKeyFactories
-import org.pkl.core.parser.LexParseException
-import org.pkl.core.parser.Parser
-import org.pkl.core.parser.antlr.PklParser
 import org.pkl.core.repl.ReplRequest
 import org.pkl.core.repl.ReplResponse
 import org.pkl.core.repl.ReplServer
 import org.pkl.core.util.IoUtils
-import org.antlr.v4.runtime.ParserRuleContext
 import org.pkl.core.http.HttpClient
+import org.pkl.core.parser.Parser
+import org.pkl.core.parser.ParserError
+import org.pkl.core.parser.cst.ClassProperty
 import org.pkl.core.resource.ResourceReaders
 import java.nio.file.Files
 import kotlin.io.path.isDirectory
@@ -303,7 +302,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
 
       override fun getType() = Type.TEST
 
-      private val parsed: ParserRuleContext by lazy {
+      private val parsed: org.pkl.core.parser.cst.Node by lazy {
         when (language) {
           "pkl" -> Parser().parseModule(code)
           "pkl-expr" -> Parser().parseExpressionInput(code)
@@ -318,7 +317,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
             if (expectError) {
               throw AssertionError("Expected a parse error, but got none.")
             }
-          } catch (e: LexParseException) {
+          } catch (e: ParserError) {
             if (!expectError) {
               throw AssertionError(e.message)
             }
@@ -335,7 +334,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
           )
         )
 
-        val properties = parsed.children.filterIsInstance<PklParser.ClassPropertyContext>()
+        val properties = parsed.children()?.filterIsInstance<ClassProperty>() ?: emptyList()
 
         val responses = mutableListOf<ReplResponse>()
 
@@ -344,7 +343,7 @@ class DocSnippetTestsEngine : HierarchicalTestEngine<DocSnippetTestsEngine.Execu
           responses.addAll(context.replServer.handleRequest(
             ReplRequest.Eval(
               "snippet",
-              prop.Identifier().text,
+              prop.name.value,
               false,
               true
             )

pkl-cli/gradle.lockfile

@@ -19,7 +19,6 @@ com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=test
 com.google.j2objc:j2objc-annotations:3.0.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.jayway.jsonpath:json-path:2.9.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.networknt:json-schema-validator:1.5.4=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=runtimeClasspath,testRuntimeClasspath
 commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 commons-io:commons-io:2.11.0=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath

pkl-cli/pkl-cli.gradle.kts

@@ -100,9 +100,6 @@ tasks.shadowJar {
   exclude("META-INF/maven/**")
   exclude("META-INF/upgrade/**")
 
-  // org.antlr.v4.runtime.misc.RuleDependencyProcessor
-  exclude("META-INF/services/javax.annotation.processing.Processor")
-
   exclude("module-info.*")
 }
 

pkl-cli/src/main/java/org/pkl/cli/svm/PolyglotContextImplTarget.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,15 +21,15 @@
 import com.oracle.svm.core.annotate.TargetClass;
 import com.oracle.svm.truffle.TruffleFeature;
 import java.util.Map;
+import org.pkl.core.ast.builder.AstBuilder;
 
 /**
  * Workaround to prevent the native-image build error "Detected a started Thread in the image
  * heap.". The cause of this error is the use of {@link org.graalvm.polyglot.Context} in the
  * (intentionally) statically reachable class {@link org.pkl.core.runtime.StdLibModule}.
  *
- * <p>A cleaner solution would be to have a separate {@link org.pkl.core.ast.builder.AstBuilder} for
- * stdlib modules that produces a fully initialized module object without executing any Truffle
- * nodes.
+ * <p>A cleaner solution would be to have a separate {@link AstBuilder} for stdlib modules that
+ * produces a fully initialized module object without executing any Truffle nodes.
  *
  * <p>This class is automatically discovered by native-image; no registration is required.
  */

pkl-codegen-java/gradle.lockfile

@@ -4,7 +4,6 @@
 com.github.ajalt.clikt:clikt-jvm:3.5.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.ajalt.clikt:clikt:3.5.4=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.palantir.javapoet:javapoet:0.6.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=runtimeClasspath,testRuntimeClasspath
 io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata

pkl-codegen-kotlin/gradle.lockfile

@@ -4,7 +4,6 @@
 com.github.ajalt.clikt:clikt-jvm:3.5.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.ajalt.clikt:clikt:3.5.4=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.squareup:kotlinpoet:1.6.0=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=runtimeClasspath,testRuntimeClasspath
 io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata

pkl-commons-cli/gradle.lockfile

@@ -3,7 +3,6 @@
 # This file is expected to be part of source control.
 com.github.ajalt.clikt:clikt-jvm:3.5.4=compileClasspath,runtimeClasspath,testCompileClasspath,testRuntimeClasspath
 com.github.ajalt.clikt:clikt:3.5.4=apiDependenciesMetadata,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=runtimeClasspath,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata
 org.assertj:assertj-core:3.27.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath

pkl-config-java/gradle.lockfile

@@ -4,7 +4,6 @@
 com.github.ajalt.clikt:clikt-jvm:3.5.4=pklCodegenJava
 com.github.ajalt.clikt:clikt:3.5.4=pklCodegenJava
 com.palantir.javapoet:javapoet:0.6.0=pklCodegenJava
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=pklCodegenJava,runtimeClasspath,testRuntimeClasspath
 io.leangen.geantyref:geantyref:1.3.16=compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 javax.inject:javax.inject:1=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath

pkl-config-kotlin/gradle.lockfile

@@ -4,7 +4,6 @@
 com.github.ajalt.clikt:clikt-jvm:3.5.4=pklCodegenKotlin
 com.github.ajalt.clikt:clikt:3.5.4=pklCodegenKotlin
 com.squareup:kotlinpoet:1.6.0=pklCodegenKotlin
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=pklCodegenKotlin,pklConfigJava,runtimeClasspath,testRuntimeClasspath
 io.leangen.geantyref:geantyref:1.3.16=pklConfigJava,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeOnlyDependenciesMetadata

pkl-core/gradle.lockfile

@@ -4,7 +4,7 @@
 com.google.code.findbugs:jsr305:3.0.2=compileClasspath,compileOnlyDependenciesMetadata
 com.palantir.javapoet:javapoet:0.6.0=generatorCompileClasspath,generatorImplementationDependenciesMetadata,generatorRuntimeClasspath
 com.tunnelvisionlabs:antlr4-annotations:4.9.0=antlr
-com.tunnelvisionlabs:antlr4-runtime:4.9.0=antlr,compileClasspath,implementationDependenciesMetadata,runtimeClasspath,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
+com.tunnelvisionlabs:antlr4-runtime:4.9.0=antlr,testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 com.tunnelvisionlabs:antlr4:4.9.0=antlr
 net.bytebuddy:byte-buddy:1.15.11=testCompileClasspath,testImplementationDependenciesMetadata,testRuntimeClasspath
 org.abego.treelayout:org.abego.treelayout.core:1.0.1=antlr

pkl-core/pkl-core.gradle.kts

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,15 +27,16 @@ plugins {
 
 val generatorSourceSet = sourceSets.register("generator")
 
-sourceSets { main { java { srcDir(file("generated/antlr")) } } }
+sourceSets { test { java { srcDir(file("testgenerated/antlr")) } } }
 
 idea {
   module {
     // mark src/main/antlr as source dir
     // mark generated/antlr as generated source dir
     // mark generated/truffle as generated source dir
-    sourceDirs = sourceDirs + files("src/main/antlr", "generated/antlr", "generated/truffle")
-    generatedSourceDirs = generatedSourceDirs + files("generated/antlr", "generated/truffle")
+    sourceDirs = sourceDirs + files("generated/truffle")
+    generatedSourceDirs = generatedSourceDirs + files("testgenerated/antlr", "generated/truffle")
+    testSources.from(files("src/test/antlr", "testgenerated/antlr"))
   }
 }
 
@@ -56,7 +57,6 @@ dependencies {
   // pkl-core implements pkl-executor's ExecutorSpi, but the SPI doesn't ship with pkl-core
   compileOnly(projects.pklExecutor)
 
-  implementation(libs.antlrRuntime)
   implementation(libs.msgpack)
   implementation(libs.truffleApi)
   implementation(libs.graalSdk)
@@ -65,6 +65,7 @@ dependencies {
 
   implementation(libs.snakeYaml)
 
+  testImplementation(libs.antlrRuntime)
   testImplementation(projects.pklCommonsTest)
 
   add("generatorImplementation", libs.javaPoet)
@@ -98,12 +99,14 @@ tasks.generateGrammarSource {
   // generate only visitor
   arguments = arguments + listOf("-visitor", "-no-listener")
 
+  source = fileTree("src/test/antlr") { include("*.g4") }
+
   // Due to https://github.com/antlr/antlr4/issues/2260,
   // we can't put .g4 files into src/main/antlr/org/pkl/core/parser/antlr.
   // Instead, we put .g4 files into src/main/antlr, adapt output dir below,
   // and use @header directives in .g4 files (instead of setting `-package` argument here)
   // and task makeIntelliJAntlrPluginHappy to fix up the IDE story.
-  outputDirectory = file("generated/antlr/org/pkl/core/parser/antlr")
+  outputDirectory = file("testgenerated/antlr/org/pkl/core/parser/antlr")
 }
 
 tasks.compileJava { dependsOn(tasks.generateGrammarSource) }
@@ -119,8 +122,8 @@ tasks.named("generateGeneratorGrammarSource") { enabled = false }
 val makeIntelliJAntlrPluginHappy by
   tasks.registering(Copy::class) {
     dependsOn(tasks.generateGrammarSource)
-    into("src/main/antlr")
-    from("generated/antlr/org/pkl/core/parser/antlr") { include("PklLexer.tokens") }
+    into("test/antlr")
+    from("testgenerated/antlr/org/pkl/core/parser/antlr") { include("PklLexer.tokens") }
   }
 
 tasks.processResources {
@@ -141,7 +144,7 @@ tasks.processResources {
         mapOf(
           "version" to buildInfo.pklVersion,
           "commitId" to buildInfo.commitId,
-          "stdlibModules" to stdlibModules.joinToString(",")
+          "stdlibModules" to stdlibModules.joinToString(","),
         )
     )
   }

pkl-core/src/main/java/org/pkl/core/PcfRenderer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 Apple Inc. and the Pkl project authors. All rights reserved.
+ * Copyright © 2024-2025 Apple Inc. and the Pkl project authors. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.