diff --git a/Cargo.lock b/Cargo.lock
index 0a298c1616..1ec82fb722 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1665,6 +1665,10 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "leo-linter"
+version = "2.1.0"
+
[[package]]
name = "leo-package"
version = "2.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index ac931df286..637012a462 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ members = [
"leo/package",
"tests/test-framework",
"utils/disassembler",
+ "utils/linter",
"utils/retriever"
]
diff --git a/compiler/ast/src/cst/functions/mod.rs b/compiler/ast/src/cst/functions/mod.rs
new file mode 100644
index 0000000000..716a4c9e41
--- /dev/null
+++ b/compiler/ast/src/cst/functions/mod.rs
@@ -0,0 +1,133 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ,
+ /// Is this function a transition, inlined, or a regular function?.
+ pub variant: Variant,
+ /// The function identifier, e.g., `foo` in `function foo(...) { ... }`.
+ pub identifier: Identifier,
+ /// The function's input parameters.
+ pub input: Vec,
+ /// The function's output type.
+ pub output_type: Type,
+ /// The body of the function.
+ pub block: Block,
+ /// The entire span of the function definition.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+}
+
+impl PartialEq for Function {
+ fn eq(&self, other: &Self) -> bool {
+ self.identifier == other.identifier
+ }
+}
+
+impl Eq for Function {}
+
+impl Function {
+ /// Initialize a new function.
+ #[allow(clippy::too_many_arguments)]
+ pub fn new(
+ annotations: Vec,
+ variant: Variant,
+ identifier: Identifier,
+ input: Vec,
+ block: Block,
+ span: Span,
+ id: NodeID,
+ ) -> Self {
+ let output_type = match output.len() {
+ 0 => Type::Unit,
+ 1 => output[0].type_.clone(),
+ _ => Type::Tuple(TupleType::new(output.iter().map(|o| o.type_.clone()).collect())),
+ };
+
+ Function { annotations, variant, identifier, input, output, output_type, block, span, id }
+ }
+
+ /// Returns function name.
+ pub fn name(&self) -> Symbol {
+ self.identifier.name
+ }
+
+ ///
+ /// Private formatting method used for optimizing [fmt::Debug] and [fmt::Display] implementations.
+ ///
+ fn format(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self.variant {
+ Variant::Inline => write!(f, "inline ")?,
+ Variant::Function | Variant::AsyncFunction => write!(f, "function ")?,
+ Variant::Transition | Variant::AsyncTransition => write!(f, "transition ")?,
+ }
+ write!(f, "{}", self.identifier)?;
+
+ let parameters = self.input.iter().map(|x| x.to_string()).collect::>().join(",");
+ let returns = match self.output.len() {
+ 0 => "()".to_string(),
+ 1 => self.output[0].to_string(),
+ _ => self.output.iter().map(|x| x.to_string()).collect::>().join(","),
+ };
+ write!(f, "({parameters}) -> {returns} {}", self.block)?;
+
+ Ok(())
+ }
+}
+
+impl From for Function {
+ fn from(function: FunctionStub) -> Self {
+ Self {
+ annotations: function.annotations,
+ variant: function.variant,
+ identifier: function.identifier,
+ input: function.input,
+ output: function.output,
+ output_type: function.output_type,
+ block: Block::default(),
+ span: function.span,
+ id: function.id,
+ }
+ }
+}
+
+impl fmt::Debug for Function {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ self.format(f)
+ }
+}
+
+impl fmt::Display for Function {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ self.format(f)
+ }
+}
+
+crate::simple_node_impl!(Function);
diff --git a/compiler/ast/src/cst/mapping/mod.rs b/compiler/ast/src/cst/mapping/mod.rs
new file mode 100644
index 0000000000..65f8c8d22a
--- /dev/null
+++ b/compiler/ast/src/cst/mapping/mod.rs
@@ -0,0 +1,59 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see (mapping: &MappingCore) -> Self {
+ Self {
+ identifier: Identifier::from(mapping.name()),
+ key_type: Type::from_snarkvm(mapping.key().plaintext_type(), None),
+ value_type: Type::from_snarkvm(mapping.value().plaintext_type(), None),
+ span: Default::default(),
+ id: Default::default(),
+ comment: Comment::None
+ }
+ }
+}
+impl fmt::Display for Mapping {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "mapping {}: {} => {}; ", self.identifier, self.key_type, self.value_type)?;
+ self.comment.fmt(f)?;
+ Ok(())
+ }
+}
\ No newline at end of file
diff --git a/compiler/ast/src/cst/mod.rs b/compiler/ast/src/cst/mod.rs
new file mode 100644
index 0000000000..eb001b13bd
--- /dev/null
+++ b/compiler/ast/src/cst/mod.rs
@@ -0,0 +1,171 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see {
+ Ok(serde_json::to_string_pretty(&self.cst).map_err(|e| AstError::failed_to_convert_ast_to_json_string(&e))?)
+ }
+
+ // Converts the ast into a JSON value.
+ // Note that there is no corresponding `from_json_value` function
+ // since we modify JSON values leaving them unable to be converted
+ // back into Programs.
+ pub fn to_json_value(&self) -> Result {
+ Ok(serde_json::to_value(&self.cst).map_err(|e| AstError::failed_to_convert_ast_to_json_value(&e))?)
+ }
+
+ /// Serializes the ast into a JSON file.
+ pub fn to_json_file(&self, mut path: std::path::PathBuf, file_name: &str) -> Result<()> {
+ path.push(file_name);
+ let file = std::fs::File::create(&path).map_err(|e| AstError::failed_to_create_ast_json_file(&path, &e))?;
+ let writer = std::io::BufWriter::new(file);
+ Ok(serde_json::to_writer_pretty(writer, &self.cst)
+ .map_err(|e| AstError::failed_to_write_ast_to_json_file(&path, &e))?)
+ }
+
+ /// Serializes the ast into a JSON value and removes keys from object mappings before writing to a file.
+ pub fn to_json_file_without_keys(
+ &self,
+ mut path: std::path::PathBuf,
+ file_name: &str,
+ excluded_keys: &[&str],
+ ) -> Result<()> {
+ path.push(file_name);
+ let file = std::fs::File::create(&path).map_err(|e| AstError::failed_to_create_ast_json_file(&path, &e))?;
+ let writer = std::io::BufWriter::new(file);
+
+ let mut value = self.to_json_value().unwrap();
+ for key in excluded_keys {
+ value = remove_key_from_json(value, key);
+ }
+ value = normalize_json_value(value);
+
+ Ok(serde_json::to_writer_pretty(writer, &value)
+ .map_err(|e| AstError::failed_to_write_ast_to_json_file(&path, &e))?)
+ }
+
+ /// Deserializes the JSON string into a ast.
+ pub fn from_json_string(json: &str) -> Result {
+ let cst: Program = serde_json::from_str(json).map_err(|e| AstError::failed_to_read_json_string_to_ast(&e))?;
+ Ok(Self { cst })
+ }
+
+ /// Deserializes the JSON string into a ast from a file.
+ pub fn from_json_file(path: std::path::PathBuf) -> Result {
+ let data = std::fs::read_to_string(&path).map_err(|e| AstError::failed_to_read_json_file(&path, &e))?;
+ Self::from_json_string(&data)
+ }
+}
+
+impl AsRef for Cst {
+ fn as_ref(&self) -> &Program {
+ &self.cst
+ }
+}
+
+
+/// Helper function to recursively filter keys from AST JSON
+pub fn remove_key_from_json(value: serde_json::Value, key: &str) -> serde_json::Value {
+ match value {
+ serde_json::Value::Object(map) => serde_json::Value::Object(
+ map.into_iter().filter(|(k, _)| k != key).map(|(k, v)| (k, remove_key_from_json(v, key))).collect(),
+ ),
+ serde_json::Value::Array(values) => {
+ serde_json::Value::Array(values.into_iter().map(|v| remove_key_from_json(v, key)).collect())
+ }
+ _ => value,
+ }
+}
+
+/// Helper function to normalize AST JSON into a form compatible with tgc.
+/// This function will traverse the original JSON value and produce a new
+/// one under the following rules:
+/// 1. Remove empty object mappings from JSON arrays
+/// 2. If there are two elements in a JSON array and one is an empty object
+/// mapping and the other is not, then lift up the one that isn't
+pub fn normalize_json_value(value: serde_json::Value) -> serde_json::Value {
+ match value {
+ serde_json::Value::Array(vec) => {
+ let orig_length = vec.len();
+ let mut new_vec: Vec = vec
+ .into_iter()
+ .filter(|v| !matches!(v, serde_json::Value::Object(map) if map.is_empty()))
+ .map(normalize_json_value)
+ .collect();
+
+ if orig_length == 2 && new_vec.len() == 1 {
+ new_vec.pop().unwrap()
+ } else {
+ serde_json::Value::Array(new_vec)
+ }
+ }
+ serde_json::Value::Object(map) => {
+ serde_json::Value::Object(map.into_iter().map(|(k, v)| (k, normalize_json_value(v))).collect())
+ }
+ _ => value,
+ }
+}
diff --git a/compiler/ast/src/cst/program/comment.rs b/compiler/ast/src/cst/program/comment.rs
new file mode 100644
index 0000000000..eb9d0a946f
--- /dev/null
+++ b/compiler/ast/src/cst/program/comment.rs
@@ -0,0 +1,39 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see , Comment)>,
+ /// A map from program stub names to program stub scopes.
+ pub stubs: IndexMap,
+ /// A map from program names to program scopes.
+ pub program_scopes: IndexMap)>,
+
+}
+
+impl fmt::Display for Program {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ for (id, import) in self.imports.iter() {
+ for comment in &import.2 {
+ comment.fmt(f)?;
+ }
+ if import.3 == Comment::None {
+ writeln!(f, "import {id}.aleo;")?;
+ }else {
+ write!(f, "import {id}.aleo; ")?;
+ }
+ import.3.fmt(f)?;
+ }
+ for (_, stub) in self.stubs.iter() {
+ stub.0.fmt(f)?;
+ writeln!(f,)?;
+ }
+ for (_, program_scope) in self.program_scopes.iter() {
+ for comment in &program_scope.1 {
+ comment.fmt(f)?;
+ }
+ program_scope.0.fmt(f)?;
+ }
+ Ok(())
+ }
+}
+
+impl Default for Program {
+ /// Constructs an empty program node.
+ fn default() -> Self {
+ Self { imports: IndexMap::new(), stubs: IndexMap::new(), program_scopes: IndexMap::new() }
+ }
+}
diff --git a/compiler/ast/src/cst/program/program_scope.rs b/compiler/ast/src/cst/program/program_scope.rs
new file mode 100644
index 0000000000..5a60aefd71
--- /dev/null
+++ b/compiler/ast/src/cst/program/program_scope.rs
@@ -0,0 +1,84 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ))>,
+ /// A vector of struct definitions.
+ pub structs: Vec<(Symbol, (Composite, Vec))>,
+ /// A vector of mapping definitions.
+ pub mappings: Vec<(Symbol, (Mapping, Vec))>,
+ /// A vector of function definitions.
+ pub functions: Vec<(Symbol, (Function, Vec))>,
+ /// The span associated with the program scope.
+ pub span: Span,
+ ///A comment
+ pub comment: Comment,
+
+}
+
+impl fmt::Display for ProgramScope {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ if self.comment == Comment::None {
+ writeln!(f, "program {} {{", self.program_id)?;
+ }else {
+ write!(f, "program {} {{ ", self.program_id)?;
+ self.comment.fmt(f)?;
+ }
+ for (_, const_) in self.consts.iter() {
+ for comment in &const_.1 {
+ write!(f, " ")?;
+ comment.fmt(f)?;
+ }
+ writeln!(f, " {}", const_.0)?;
+ }
+ for (_, struct_) in self.structs.iter() {
+ for comment in &struct_.1 {
+ write!(f, " ")?;
+ comment.fmt(f)?;
+ }
+ writeln!(f, " {}", struct_.0)?;
+ }
+ for (_, mapping) in self.mappings.iter() {
+ for comment in &mapping.1 {
+ write!(f, " ")?;
+ comment.fmt(f)?;
+ }
+ writeln!(f, " {}", mapping.0)?;
+ }
+ for (_, function) in self.functions.iter() {
+ for comment in &function.1 {
+ write!(f, " ")?;
+ comment.fmt(f)?;
+ }
+ writeln!(f, " {}", function.0)?;
+ }
+ writeln!(f, "}}")?;
+ Ok(())
+ }
+}
diff --git a/compiler/ast/src/cst/statement/assert.rs b/compiler/ast/src/cst/statement/assert.rs
new file mode 100644
index 0000000000..c282e8d837
--- /dev/null
+++ b/compiler/ast/src/cst/statement/assert.rs
@@ -0,0 +1,57 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see )`, `assert_eq()` or `assert_neq()`.
+#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Debug)]
+pub struct AssertStatement {
+ /// The variant of the assert statement.
+ pub variant: AssertVariant,
+ /// The span, excluding the semicolon.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+ /// The comment
+ pub comment: Comment
+}
+
+impl fmt::Display for AssertStatement {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self.variant {
+ AssertVariant::Assert(ref expr) => write!(f, "assert({expr}); {}", self.comment),
+ AssertVariant::AssertEq(ref expr1, ref expr2) => write!(f, "assert_eq({expr1}, {expr2}); {}", self.comment),
+ AssertVariant::AssertNeq(ref expr1, ref expr2) => write!(f, "assert_neq({expr1}, {expr2}); {}", self.comment),
+ }
+ }
+}
+
+crate::simple_node_impl!(AssertStatement);
diff --git a/compiler/ast/src/cst/statement/assign.rs b/compiler/ast/src/cst/statement/assign.rs
new file mode 100644
index 0000000000..b3805a9146
--- /dev/null
+++ b/compiler/ast/src/cst/statement/assign.rs
@@ -0,0 +1,46 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see )>,
+ /// The span from `{` to `}`.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+}
+
+impl fmt::Display for Block {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ writeln!(f, "{{")?;
+ if self.statements.is_empty() {
+ writeln!(f, "\t")?;
+ } else {
+ self.statements.iter().try_for_each(|statement| {
+ for comment in &statement.1 {
+ write!(f, "\t\t")?;
+ comment.fmt(f)?;
+ }
+ writeln!(f, "\t\t{}", statement.0)
+ })?;
+ }
+ write!(f, "\t}}")
+ }
+}
+
+crate::simple_node_impl!(Block);
diff --git a/compiler/ast/src/cst/statement/conditional.rs b/compiler/ast/src/cst/statement/conditional.rs
new file mode 100644
index 0000000000..070c22e550
--- /dev/null
+++ b/compiler/ast/src/cst/statement/conditional.rs
@@ -0,0 +1,48 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see >,
+ /// The span from `if` to `next` or to `block`.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+}
+
+impl fmt::Display for ConditionalStatement {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ write!(f, "if ({}) {}", self.condition, self.then)?;
+ match self.otherwise.as_ref() {
+ Some(n_or_e) => write!(f, " else {n_or_e}"),
+ None => write!(f, ""),
+ }
+ }
+}
+
+crate::simple_node_impl!(ConditionalStatement);
diff --git a/compiler/ast/src/cst/statement/console/console_function.rs b/compiler/ast/src/cst/statement/console/console_function.rs
new file mode 100644
index 0000000000..2ded743cbe
--- /dev/null
+++ b/compiler/ast/src/cst/statement/console/console_function.rs
@@ -0,0 +1,40 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see >,
+ /// The end of the iteration, possibly `inclusive`.
+ pub stop: Expression,
+ /// The concrete value of `stop`.
+ #[serde(skip)]
+ pub stop_value: RefCell>,
+ /// Whether `stop` is inclusive or not.
+ /// Signified with `=` when parsing.
+ pub inclusive: bool,
+ /// The block to run on each iteration.
+ pub block: Block,
+ /// The span from `for` to `block`.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+}
+
+impl fmt::Display for IterationStatement {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ let eq = if self.inclusive { "=" } else { "" };
+ write!(f, "for {} in {}..{eq}{} {}", self.variable, self.start, self.stop, self.block)
+ }
+}
+
+crate::simple_node_impl!(IterationStatement);
diff --git a/compiler/ast/src/cst/statement/mod.rs b/compiler/ast/src/cst/statement/mod.rs
new file mode 100644
index 0000000000..dd5cf63461
--- /dev/null
+++ b/compiler/ast/src/cst/statement/mod.rs
@@ -0,0 +1,166 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ),
+ /// A block statement.
+ Block(Block),
+ /// An `if` statement.
+ Conditional(ConditionalStatement),
+ /// A console logging statement.
+ Console(ConsoleStatement),
+ /// A binding from identifier to constant value.
+ Const(ConstDeclaration),
+ /// A binding or set of bindings / variables to declare.
+ Definition(DefinitionStatement),
+ /// An expression statement
+ Expression(ExpressionStatement),
+ /// A `for` statement.
+ Iteration(Box),
+ /// A return statement `return expr;`.
+ Return(ReturnStatement),
+}
+
+impl Statement {
+ /// Returns a dummy statement made from an empty block `{}`.
+ pub fn dummy(span: Span, id: NodeID) -> Self {
+ Self::Block(Block { statements: Vec::new(), span, id })
+ }
+}
+
+impl fmt::Display for Statement {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ Statement::Assert(x) => x.fmt(f),
+ Statement::Assign(x) => x.fmt(f),
+ Statement::Block(x) => x.fmt(f),
+ Statement::Conditional(x) => x.fmt(f),
+ Statement::Console(x) => x.fmt(f),
+ Statement::Const(x) => x.fmt(f),
+ Statement::Definition(x) => x.fmt(f),
+ Statement::Expression(x) => x.fmt(f),
+ Statement::Iteration(x) => x.fmt(f),
+ Statement::Return(x) => x.fmt(f),
+ }
+ }
+}
+
+impl Node for Statement {
+ fn span(&self) -> Span {
+ use Statement::*;
+ match self {
+ Assert(n) => n.span(),
+ Assign(n) => n.span(),
+ Block(n) => n.span(),
+ Conditional(n) => n.span(),
+ Console(n) => n.span(),
+ Const(n) => n.span(),
+ Definition(n) => n.span(),
+ Expression(n) => n.span(),
+ Iteration(n) => n.span(),
+ Return(n) => n.span(),
+ }
+ }
+
+ fn set_span(&mut self, span: Span) {
+ use Statement::*;
+ match self {
+ Assert(n) => n.set_span(span),
+ Assign(n) => n.set_span(span),
+ Block(n) => n.set_span(span),
+ Conditional(n) => n.set_span(span),
+ Console(n) => n.set_span(span),
+ Const(n) => n.set_span(span),
+ Definition(n) => n.set_span(span),
+ Expression(n) => n.set_span(span),
+ Iteration(n) => n.set_span(span),
+ Return(n) => n.set_span(span),
+ }
+ }
+
+ fn id(&self) -> NodeID {
+ use Statement::*;
+ match self {
+ Assert(n) => n.id(),
+ Assign(n) => n.id(),
+ Block(n) => n.id(),
+ Conditional(n) => n.id(),
+ Console(n) => n.id(),
+ Const(n) => n.id(),
+ Definition(n) => n.id(),
+ Expression(n) => n.id(),
+ Iteration(n) => n.id(),
+ Return(n) => n.id(),
+ }
+ }
+
+ fn set_id(&mut self, id: NodeID) {
+ use Statement::*;
+ match self {
+ Assert(n) => n.set_id(id),
+ Assign(n) => n.set_id(id),
+ Block(n) => n.set_id(id),
+ Conditional(n) => n.set_id(id),
+ Console(n) => n.set_id(id),
+ Const(n) => n.set_id(id),
+ Definition(n) => n.set_id(id),
+ Expression(n) => n.set_id(id),
+ Iteration(n) => n.set_id(id),
+ Return(n) => n.set_id(id),
+ }
+ }
+}
diff --git a/compiler/ast/src/cst/statement/return_.rs b/compiler/ast/src/cst/statement/return_.rs
new file mode 100644
index 0000000000..6ebad0d33e
--- /dev/null
+++ b/compiler/ast/src/cst/statement/return_.rs
@@ -0,0 +1,43 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see )>,
+ /// The external program the struct is defined in.
+ pub external: Option,
+ /// Was this a `record Foo { ... }`?
+ /// If so, it wasn't a composite.
+ pub is_record: bool,
+ /// The entire span of the composite definition.
+ pub span: Span,
+ /// The ID of the node.
+ pub id: NodeID,
+ /// The comment
+ pub comment: Comment,
+}
+
+impl PartialEq for Composite {
+ fn eq(&self, other: &Self) -> bool {
+ self.identifier == other.identifier
+ }
+}
+
+impl Eq for Composite {}
+
+impl fmt::Debug for Composite {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ ::fmt(self, f)
+ }
+}
+
+impl fmt::Display for Composite {
+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+ f.write_str(if self.is_record { "record" } else { "struct" })?;
+ write!(f, " {} {{ ", self.identifier)?;
+ self.comment.fmt(f)?;
+ for field in self.members.iter() {
+ for comment in &field.1 {
+ write!(f, " ")?;
+ comment.fmt(f)?;
+ }
+ write!(f, " {}", field.0)?;
+ }
+ if let Some(memeber) = self.members.last() {
+ if memeber.0.comment == Comment::None {
+ writeln!(f, "")?;
+ }else {
+ write!(f, "")?;
+ }
+ }
+ write!(f, " }}")
+ }
+}
\ No newline at end of file
diff --git a/compiler/ast/src/lib.rs b/compiler/ast/src/lib.rs
index 2d079a93c5..52dda8c414 100644
--- a/compiler/ast/src/lib.rs
+++ b/compiler/ast/src/lib.rs
@@ -60,6 +60,8 @@ pub mod value;
pub mod stub;
pub use self::stub::*;
+pub mod cst;
+
pub use self::value::*;
pub use common::node::*;
diff --git a/compiler/parser/src/cst/mod.rs b/compiler/parser/src/cst/mod.rs
new file mode 100644
index 0000000000..851864829f
--- /dev/null
+++ b/compiler/parser/src/cst/mod.rs
@@ -0,0 +1,53 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see (
+ handler: &Handler,
+ node_builder: &NodeBuilder,
+ source: &str,
+ start_pos: BytePos,
+) -> Result {
+ Ok(Cst::new(parse_::(handler, node_builder, source, start_pos)?))
+}
\ No newline at end of file
diff --git a/compiler/parser/src/cst/parser/context.rs b/compiler/parser/src/cst/parser/context.rs
new file mode 100644
index 0000000000..3d94394271
--- /dev/null
+++ b/compiler/parser/src/cst/parser/context.rs
@@ -0,0 +1,340 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ,
+ /// The current token, i.e., if `p.tokens = ['3', *, '4']`,
+ /// then after a `p.bump()`, we'll have `p.token = '3'`.
+ pub(crate) token: SpannedToken,
+ /// The previous token, i.e., if `p.tokens = ['3', *, '4']`,
+ /// then after two `p.bump()`s, we'll have `p.token = '*'` and `p.prev_token = '3'`.
+ pub(crate) prev_token: SpannedToken,
+ /// True if parsing an expression for if and loop statements -- means struct inits are not legal.
+ pub(crate) disallow_struct_construction: bool,
+ /// The name of the program being parsed.
+ pub(crate) program_name: Option,
+ // Allows the parser to be generic over the network.
+ phantom: PhantomData,
+}
+
+/// Dummy span used to appease borrow checker.
+const DUMMY_EOF: SpannedToken = SpannedToken { token: Token::Eof, span: Span::dummy() };
+
+impl<'a, N: Network> ParserContext<'a, N> {
+ /// Returns a new [`ParserContext`] type given a vector of tokens.
+ pub fn new(handler: &'a Handler, node_builder: &'a NodeBuilder, mut tokens: Vec) -> Self {
+ // For performance we reverse so that we get cheap `.pop()`s.
+ tokens.reverse();
+
+ let token = SpannedToken::dummy();
+ let mut p = Self {
+ handler,
+ node_builder,
+ disallow_struct_construction: false,
+ prev_token: token.clone(),
+ token,
+ tokens,
+ program_name: None,
+ phantom: Default::default(),
+ };
+ p.bump();
+ p
+ }
+
+ /// Advances the parser cursor by one token.
+ ///
+ /// So e.g., if we had `previous = A`, `current = B`, and `tokens = [C, D, E]`,
+ /// then after `p.bump()`, the state will be `previous = B`, `current = C`, and `tokens = [D, E]`.
+ pub(crate) fn bump(&mut self) {
+ // Probably a bug (infinite loop), as the previous token was already EOF.
+ if let Token::Eof = self.prev_token.token {
+ panic!("attempted to bump the parser past EOF (may be stuck in a loop)");
+ }
+
+ // Extract next token, or `Eof` if there was none.
+ let next_token = self.tokens.pop().unwrap_or(SpannedToken { token: Token::Eof, span: self.token.span });
+
+ // Set the new token.
+ self.prev_token = mem::replace(&mut self.token, next_token);
+ }
+
+ /// Checks whether the current token is `tok`.
+ pub(super) fn check(&self, tok: &Token) -> bool {
+ &self.token.token == tok
+ }
+
+ /// Checks whether the current token is a `Token::Integer(_)`.
+ pub(super) fn check_int(&self) -> bool {
+ matches!(&self.token.token, Token::Integer(_))
+ }
+
+ /// Returns `true` if the next token is equal to the given token.
+ /// Advances the parser to the next token.
+ pub(super) fn eat(&mut self, token: &Token) -> bool {
+ self.check(token).then(|| self.bump()).is_some()
+ }
+
+ /// Look-ahead `dist` tokens of `self.token` and get access to that token there.
+ /// When `dist == 0` then the current token is looked at.
+ pub(super) fn look_ahead<'s, R>(&'s self, dist: usize, looker: impl FnOnce(&'s SpannedToken) -> R) -> R {
+ if dist == 0 {
+ return looker(&self.token);
+ }
+
+ let idx = match self.tokens.len().checked_sub(dist) {
+ None => return looker(&DUMMY_EOF),
+ Some(idx) => idx,
+ };
+
+ looker(self.tokens.get(idx).unwrap_or(&DUMMY_EOF))
+ }
+
+ /// Emit the error `err`.
+ pub(super) fn emit_err(&self, err: ParserError) {
+ self.handler.emit_err(err);
+ }
+
+ /// Emit the warning `warning`.
+ pub(super) fn emit_warning(&self, warning: ParserWarning) {
+ self.handler.emit_warning(warning.into());
+ }
+
+ /// Returns true if the next token exists.
+ pub(crate) fn has_next(&self) -> bool {
+ !matches!(self.token.token, Token::Eof)
+ }
+
+ /// At the previous token, return and make an identifier with `name`.
+ fn mk_ident_prev(&self, name: Symbol) -> Identifier {
+ let span = self.prev_token.span;
+ Identifier { name, span, id: self.node_builder.next_id() }
+ }
+
+ /// Eats the next token if it is an identifier and returns it.
+ pub(super) fn eat_identifier(&mut self) -> Option {
+ if let Token::Identifier(name) = self.token.token {
+ self.bump();
+ return Some(self.mk_ident_prev(name));
+ }
+ None
+ }
+
+ /// Expects an [`Identifier`], or errors.
+ pub(super) fn expect_identifier(&mut self) -> Result {
+ self.eat_identifier()
+ .ok_or_else(|| ParserError::unexpected_str(&self.token.token, "identifier", self.token.span).into())
+ }
+
+ ///
+ /// Removes the next token if it is a [`Token::Integer(_)`] and returns it, or [None] if
+ /// the next token is not a [`Token::Integer(_)`] or if the next token does not exist.
+ ///
+ pub fn eat_whole_number(&mut self) -> Result<(NonNegativeNumber, Span)> {
+ if let Token::Integer(value) = &self.token.token {
+ let value = value.clone();
+ self.bump();
+ // Reject value if the length is over 2 and the first character is 0
+ if (value.len() > 1 && value.starts_with('0')) || value.contains('_') {
+ return Err(ParserError::tuple_index_must_be_whole_number(&self.token.token, self.token.span).into());
+ }
+
+ Ok((NonNegativeNumber::from(value), self.prev_token.span))
+ } else {
+ Err(ParserError::unexpected(&self.token.token, "integer literal", self.token.span).into())
+ }
+ }
+
+ /// Eats any of the given `tokens`, returning `true` if anything was eaten.
+ pub(super) fn eat_any(&mut self, tokens: &[Token]) -> bool {
+ tokens.iter().any(|x| self.check(x)).then(|| self.bump()).is_some()
+ }
+
+ /// Returns an unexpected error at the current token.
+ pub(super) fn unexpected(&self, expected: impl Display) -> Result {
+ Err(ParserError::unexpected(&self.token.token, expected, self.token.span).into())
+ }
+
+ /// Eats the expected `token`, or errors.
+ pub(super) fn expect(&mut self, token: &Token) -> Result {
+ if self.eat(token) { Ok(self.prev_token.span) } else { self.unexpected(token) }
+ }
+
+ /// Eats one of the expected `tokens`, or errors.
+ pub(super) fn expect_any(&mut self, tokens: &[Token]) -> Result {
+ if self.eat_any(tokens) {
+ Ok(self.prev_token.span)
+ } else {
+ self.unexpected(tokens.iter().map(|x| format!("'{x}'")).collect::>().join(", "))
+ }
+ }
+
+ /// Parses a list of `T`s using `inner`
+ /// The opening and closing delimiters are specified in `delimiter`,
+ /// and elements in the list are optionally separated by `sep`.
+ /// When `(list, true)` is returned, `sep` was a terminator.
+ pub(super) fn parse_list_statement(
+ &mut self,
+ delimiter: Delimiter,
+ sep: Option,
+ mut inner: impl FnMut(&mut Self) -> Result>,
+ ) -> Result<(Vec<(Statement, Vec)>, bool, Span)> {
+ let (open, close) = delimiter.open_close_pair();
+ let mut list: Vec<(Statement, Vec)> = Vec::new();
+ let mut trailing = false;
+ let mut comments: Vec = vec![];
+ // Parse opening delimiter.
+ let open_span = self.expect(&open)?;
+
+ while !self.check(&close) {
+ let token = self.token.token.clone();
+ if matches!(delimiter, Delimiter::Parenthesis) {
+ if self.is_comment(token.clone()) || self.is_inline_comment(token.clone()) {
+ self.bump();
+ continue;
+ }
+ } else if matches!(delimiter, Delimiter::Brace) {
+ if self.is_comment(token.clone()) {
+ comments.push(self.get_comment(token.clone()));
+ self.bump();
+ continue;
+ } else if self.is_inline_comment(token.clone()) {
+ let comment = self.get_comment(token.clone());
+ if let Some(value) = list.last_mut() {
+ match value.0 {
+ Statement::Const(ref mut _const) => {
+ _const.comment = comment.clone()
+ },
+ Statement::Assert(ref mut _assert) => {
+ _assert.comment = comment.clone()
+ },
+ Statement::Assign(ref mut _assign) => {
+ _assign.comment = comment.clone()
+ },
+ Statement::Console(ref mut _console) => {
+ _console.comment = comment.clone()
+ },
+ Statement::Definition(ref mut _def) => {
+ _def.comment = comment.clone()
+ },
+ Statement::Expression(ref mut _expression) => {
+ _expression.comment = comment.clone()
+ },
+ Statement::Return(ref mut _ret) => {
+ _ret.comment = comment.clone()
+ },
+ _ => {
+
+ }
+ }
+ }
+ self.bump();
+ continue;
+ }
+ }
+ // Parse the element. We allow inner parser recovery through the `Option`.
+ if let Some(elem) = inner(self)? {
+ list.push((elem, comments.clone()));
+ comments.clear()
+ }
+ // Parse the separator, if any.
+ if sep.as_ref().filter(|sep| !self.eat(sep)).is_some() {
+ trailing = false;
+ break;
+ }
+
+ trailing = true;
+ }
+ // Parse closing delimiter.
+ let span = open_span + self.expect(&close)?;
+
+ Ok((list, trailing, span))
+ }
+
+ pub(super) fn parse_list(
+ &mut self,
+ delimiter: Delimiter,
+ sep: Option,
+ mut inner: impl FnMut(&mut Self) -> Result>,
+ ) -> Result<(Vec, bool, Span)> {
+ let (open, close) = delimiter.open_close_pair();
+ let mut list = Vec::new();
+ let mut trailing = false;
+ // Parse opening delimiter.
+ let open_span = self.expect(&open)?;
+
+ while !self.check(&close) {
+ let token = self.token.token.clone();
+ if matches!(delimiter, Delimiter::Parenthesis) {
+ if self.is_comment(token.clone()) || self.is_inline_comment(token.clone()) {
+ self.bump();
+ continue;
+ }
+ }
+ // Parse the element. We allow inner parser recovery through the `Option`.
+ if let Some(elem) = inner(self)? {
+ list.push(elem);
+ }
+ // Parse the separator, if any.
+ if sep.as_ref().filter(|sep| !self.eat(sep)).is_some() {
+ trailing = false;
+ break;
+ }
+
+ trailing = true;
+ }
+
+ // Parse closing delimiter.
+ let span = open_span + self.expect(&close)?;
+
+ Ok((list, trailing, span))
+ }
+ /// Parse a list separated by `,` and delimited by parens.
+ pub(super) fn parse_paren_comma_list(
+ &mut self,
+ f: impl FnMut(&mut Self) -> Result>,
+ ) -> Result<(Vec, bool, Span)> {
+ self.parse_list(Delimiter::Parenthesis, Some(Token::Comma), f)
+ }
+
+ /// Parse a list separated by `,` and delimited by brackets.
+ pub(super) fn parse_bracket_comma_list(
+ &mut self,
+ f: impl FnMut(&mut Self) -> Result>,
+ ) -> Result<(Vec, bool, Span)> {
+ self.parse_list(Delimiter::Bracket, Some(Token::Comma), f)
+ }
+
+ /// Returns true if the current token is `(`.
+ pub(super) fn peek_is_left_par(&self) -> bool {
+ matches!(self.token.token, Token::LeftParen)
+ }
+}
diff --git a/compiler/parser/src/cst/parser/expression.rs b/compiler/parser/src/cst/parser/expression.rs
new file mode 100644
index 0000000000..f398536e8f
--- /dev/null
+++ b/compiler/parser/src/cst/parser/expression.rs
@@ -0,0 +1,841 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ParserContext<'_, N> {
+ /// Returns an [`Expression`] AST node if the next token is an expression.
+ /// Includes struct init expressions.
+ pub(crate) fn parse_expression(&mut self) -> Result {
+ // Store current parser state.
+ let prior_fuzzy_state = self.disallow_struct_construction;
+
+ // Allow struct init expressions.
+ self.disallow_struct_construction = false;
+
+ // Parse expression.
+ let result = self.parse_conditional_expression();
+
+ // Restore prior parser state.
+ self.disallow_struct_construction = prior_fuzzy_state;
+
+ result
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent
+ /// a ternary expression. May or may not include struct init expressions.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_boolean_or_expression`].
+ pub(super) fn parse_conditional_expression(&mut self) -> Result {
+ // Try to parse the next expression. Try BinaryOperation::Or.
+ let mut expr = self.parse_boolean_or_expression()?;
+
+ // Parse the rest of the ternary expression.
+ if self.eat(&Token::Question) {
+ let if_true = self.parse_expression()?;
+ self.expect(&Token::Colon)?;
+ let if_false = self.parse_expression()?;
+ expr = Expression::Ternary(TernaryExpression {
+ span: expr.span() + if_false.span(),
+ condition: Box::new(expr),
+ if_true: Box::new(if_true),
+ if_false: Box::new(if_false),
+ id: self.node_builder.next_id(),
+ });
+ }
+ Ok(expr)
+ }
+
+ /// Constructs a binary expression `left op right`.
+ fn bin_expr(node_builder: &NodeBuilder, left: Expression, right: Expression, op: BinaryOperation) -> Expression {
+ Expression::Binary(BinaryExpression {
+ span: left.span() + right.span(),
+ op,
+ left: Box::new(left),
+ right: Box::new(right),
+ id: node_builder.next_id(),
+ })
+ }
+
+ /// Parses a left-associative binary expression ` token ` using `f` for left/right.
+ /// The `token` is translated to `op` in the AST.
+ fn parse_bin_expr(
+ &mut self,
+ tokens: &[Token],
+ mut f: impl FnMut(&mut Self) -> Result,
+ ) -> Result {
+ let mut expr = f(self)?;
+ while let Some(op) = self.eat_bin_op(tokens) {
+ expr = Self::bin_expr(self.node_builder, expr, f(self)?, op);
+ }
+ Ok(expr)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent
+ /// a binary OR expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_boolean_and_expression`].
+ fn parse_boolean_or_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::Or], Self::parse_boolean_and_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary AND expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_equality_expression`].
+ fn parse_boolean_and_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::And], Self::parse_equality_expression)
+ }
+
+ /// Eats one of binary operators matching any in `tokens`.
+ fn eat_bin_op(&mut self, tokens: &[Token]) -> Option {
+ self.eat_any(tokens).then(|| match &self.prev_token.token {
+ Token::Eq => BinaryOperation::Eq,
+ Token::NotEq => BinaryOperation::Neq,
+ Token::Lt => BinaryOperation::Lt,
+ Token::LtEq => BinaryOperation::Lte,
+ Token::Gt => BinaryOperation::Gt,
+ Token::GtEq => BinaryOperation::Gte,
+ Token::Add => BinaryOperation::Add,
+ Token::Sub => BinaryOperation::Sub,
+ Token::Mul => BinaryOperation::Mul,
+ Token::Div => BinaryOperation::Div,
+ Token::Rem => BinaryOperation::Rem,
+ Token::Or => BinaryOperation::Or,
+ Token::And => BinaryOperation::And,
+ Token::BitOr => BinaryOperation::BitwiseOr,
+ Token::BitAnd => BinaryOperation::BitwiseAnd,
+ Token::Pow => BinaryOperation::Pow,
+ Token::Shl => BinaryOperation::Shl,
+ Token::Shr => BinaryOperation::Shr,
+ Token::BitXor => BinaryOperation::Xor,
+ _ => unreachable!("`eat_bin_op` shouldn't produce this"),
+ })
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary relational expression: less than, less than or equals, greater than, greater than or equals.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_additive_expression`].
+ fn parse_ordering_expression(&mut self) -> Result {
+ let mut expr = self.parse_bitwise_exclusive_or_expression()?;
+ if let Some(op) = self.eat_bin_op(&[Token::Lt, Token::LtEq, Token::Gt, Token::GtEq]) {
+ let right = self.parse_bitwise_exclusive_or_expression()?;
+ expr = Self::bin_expr(self.node_builder, expr, right, op);
+ }
+ Ok(expr)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary equals or not equals expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_ordering_expression`].
+ fn parse_equality_expression(&mut self) -> Result {
+ let mut expr = self.parse_ordering_expression()?;
+ if let Some(op) = self.eat_bin_op(&[Token::Eq, Token::NotEq]) {
+ let right = self.parse_ordering_expression()?;
+ expr = Self::bin_expr(self.node_builder, expr, right, op);
+ }
+ Ok(expr)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// bitwise exclusive or expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_bitwise_inclusive_or_expression`].
+ fn parse_bitwise_exclusive_or_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::BitXor], Self::parse_bitwise_inclusive_or_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// bitwise inclusive or expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_bitwise_and_expression`].
+ fn parse_bitwise_inclusive_or_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::BitOr], Self::parse_bitwise_and_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// bitwise and expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_shift_expression`].
+ fn parse_bitwise_and_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::BitAnd], Self::parse_shift_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// shift left or a shift right expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_additive_expression`].
+ fn parse_shift_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::Shl, Token::Shr], Self::parse_additive_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary addition or subtraction expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_mul_div_pow_expression`].
+ fn parse_additive_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::Add, Token::Sub], Self::parse_multiplicative_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary multiplication, division, or a remainder expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_exponential_expression`].
+ fn parse_multiplicative_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::Mul, Token::Div, Token::Rem], Self::parse_exponential_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// binary exponentiation expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_cast_expression`].
+ fn parse_exponential_expression(&mut self) -> Result {
+ self.parse_bin_expr(&[Token::Pow], Self::parse_cast_expression)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// cast expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_unary_expression`].
+ fn parse_cast_expression(&mut self) -> Result {
+ let mut expr = self.parse_unary_expression()?;
+ if self.eat(&Token::As) {
+ let (type_, end_span) = self.parse_primitive_type()?;
+ let span = expr.span() + end_span;
+ expr = Expression::Cast(CastExpression {
+ expression: Box::new(expr),
+ type_,
+ span,
+ id: self.node_builder.next_id(),
+ });
+ }
+
+ Ok(expr)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// unary not, negate, or bitwise not expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_postfix_expression`].
+ pub(super) fn parse_unary_expression(&mut self) -> Result {
+ let mut ops = Vec::new();
+ while self.eat_any(&[Token::Not, Token::Sub]) {
+ let operation = match self.prev_token.token {
+ Token::Not => UnaryOperation::Not,
+ Token::Sub => UnaryOperation::Negate,
+ _ => unreachable!("parse_unary_expression_ shouldn't produce this"),
+ };
+ ops.push((operation, self.prev_token.span));
+ }
+
+ let mut inner = self.parse_postfix_expression()?;
+
+ // If the last operation is a negation and the inner expression is a literal, then construct a negative literal.
+ if let Some((UnaryOperation::Negate, _)) = ops.last() {
+ match inner {
+ Expression::Literal(Literal::Integer(integer_type, string, span, id)) => {
+ // Remove the negation from the operations.
+ // Note that this unwrap is safe because there is at least one operation in `ops`.
+ let (_, op_span) = ops.pop().unwrap();
+ // Construct a negative integer literal.
+ inner =
+ Expression::Literal(Literal::Integer(integer_type, format!("-{string}"), op_span + span, id));
+ }
+ Expression::Literal(Literal::Field(string, span, id)) => {
+ // Remove the negation from the operations.
+ // Note that
+ let (_, op_span) = ops.pop().unwrap();
+ // Construct a negative field literal.
+ inner = Expression::Literal(Literal::Field(format!("-{string}"), op_span + span, id));
+ }
+ Expression::Literal(Literal::Group(group_literal)) => {
+ // Remove the negation from the operations.
+ let (_, op_span) = ops.pop().unwrap();
+ // Construct a negative group literal.
+ // Note that we only handle the case where the group literal is a single integral value.
+ inner = Expression::Literal(Literal::Group(Box::new(match *group_literal {
+ GroupLiteral::Single(string, span, id) => {
+ GroupLiteral::Single(format!("-{string}"), op_span + span, id)
+ }
+ GroupLiteral::Tuple(tuple) => GroupLiteral::Tuple(tuple),
+ })));
+ }
+ Expression::Literal(Literal::Scalar(string, span, id)) => {
+ // Remove the negation from the operations.
+ let (_, op_span) = ops.pop().unwrap();
+ // Construct a negative scalar literal.
+ inner = Expression::Literal(Literal::Scalar(format!("-{string}"), op_span + span, id));
+ }
+ _ => (), // Do nothing.
+ }
+ }
+
+ // Apply the operations in reverse order, constructing a unary expression.
+ for (op, op_span) in ops.into_iter().rev() {
+ inner = Expression::Unary(UnaryExpression {
+ span: op_span + inner.span(),
+ op,
+ receiver: Box::new(inner),
+ id: self.node_builder.next_id(),
+ });
+ }
+
+ Ok(inner)
+ }
+
+ // TODO: Parse method call expressions directly and later put them into a canonical form.
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// method call expression.
+ fn parse_method_call_expression(&mut self, receiver: Expression, method: Identifier) -> Result {
+ // Parse the argument list.
+ let (mut args, _, span) = self.parse_expr_tuple()?;
+ let span = receiver.span() + span;
+
+ if let (true, Some(op)) = (args.is_empty(), UnaryOperation::from_symbol(method.name)) {
+ // Found an unary operator and the argument list is empty.
+ Ok(Expression::Unary(UnaryExpression {
+ span,
+ op,
+ receiver: Box::new(receiver),
+ id: self.node_builder.next_id(),
+ }))
+ } else if let (1, Some(op)) = (args.len(), BinaryOperation::from_symbol(method.name)) {
+ // Found a binary operator and the argument list contains a single argument.
+ Ok(Expression::Binary(BinaryExpression {
+ span,
+ op,
+ left: Box::new(receiver),
+ right: Box::new(args.swap_remove(0)),
+ id: self.node_builder.next_id(),
+ }))
+ } else if let (2, Some(CoreFunction::SignatureVerify)) =
+ (args.len(), CoreFunction::from_symbols(sym::signature, method.name))
+ {
+ Ok(Expression::Access(AccessExpression::AssociatedFunction(AssociatedFunction {
+ variant: Identifier::new(sym::signature, self.node_builder.next_id()),
+ name: method,
+ arguments: {
+ let mut arguments = vec![receiver];
+ arguments.extend(args);
+ arguments
+ },
+ span,
+ id: self.node_builder.next_id(),
+ })))
+ } else if let (0, Some(CoreFunction::FutureAwait)) =
+ (args.len(), CoreFunction::from_symbols(sym::Future, method.name))
+ {
+ Ok(Expression::Access(AccessExpression::AssociatedFunction(AssociatedFunction {
+ variant: Identifier::new(sym::Future, self.node_builder.next_id()),
+ name: method,
+ arguments: vec![receiver],
+ span,
+ id: self.node_builder.next_id(),
+ })))
+ } else {
+ // Attempt to parse the method call as a mapping operation.
+ match (args.len(), CoreFunction::from_symbols(sym::Mapping, method.name)) {
+ (1, Some(CoreFunction::MappingGet))
+ | (2, Some(CoreFunction::MappingGetOrUse))
+ | (2, Some(CoreFunction::MappingSet))
+ | (1, Some(CoreFunction::MappingRemove))
+ | (1, Some(CoreFunction::MappingContains)) => {
+ // Found an instance of `.get`, `.get_or_use`, `.set`, `.remove`, or `.contains`.
+ Ok(Expression::Access(AccessExpression::AssociatedFunction(AssociatedFunction {
+ variant: Identifier::new(sym::Mapping, self.node_builder.next_id()),
+ name: method,
+ arguments: {
+ let mut arguments = vec![receiver];
+ arguments.extend(args);
+ arguments
+ },
+ span,
+ id: self.node_builder.next_id(),
+ })))
+ }
+ _ => {
+ // Either an invalid unary/binary operator, or more arguments given.
+ self.emit_err(ParserError::invalid_method_call(receiver, method, args.len(), span));
+ Ok(Expression::Err(ErrExpression { span, id: self.node_builder.next_id() }))
+ }
+ }
+ }
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// static access expression.
+ fn parse_associated_access_expression(&mut self, module_name: Expression) -> Result {
+ // Ensure that the preceding expression is an identifier (a named type).
+ let variant = if let Expression::Identifier(ident) = module_name {
+ ident
+ } else {
+ return Err(ParserError::invalid_associated_access(&module_name, module_name.span()).into());
+ };
+
+ // Parse the constant or function name.
+ let member_name = self.expect_identifier()?;
+
+ // Check if there are arguments.
+ Ok(Expression::Access(if self.check(&Token::LeftParen) {
+ // Parse the arguments
+ let (args, _, end) = self.parse_expr_tuple()?;
+
+ // Return the associated function.
+ AccessExpression::AssociatedFunction(AssociatedFunction {
+ span: module_name.span() + end,
+ variant,
+ name: member_name,
+ arguments: args,
+ id: self.node_builder.next_id(),
+ })
+ } else {
+ // Return the associated constant.
+ AccessExpression::AssociatedConstant(AssociatedConstant {
+ span: module_name.span() + member_name.span(),
+ ty: Type::Identifier(variant),
+ name: member_name,
+ id: self.node_builder.next_id(),
+ })
+ }))
+ }
+
+ /// Parses a tuple of `Expression` AST nodes.
+ pub(crate) fn parse_expr_tuple(&mut self) -> Result<(Vec, bool, Span)> {
+ self.parse_paren_comma_list(|p| p.parse_expression().map(Some))
+ }
+
+ /// Parses an external function call `credits.aleo/transfer()` or locator `token.aleo/accounts`.
+ ///
+ /// In the ABNF grammar,
+ /// an external function call is one of the two kinds of free function calls,
+ /// namely the one that uses a locator to designate the function;
+ /// a locator is a kind of primary expression.
+ fn parse_external_resource(&mut self, expr: Expression, network_span: Span) -> Result {
+ // Parse `/`.
+ self.expect(&Token::Div)?;
+
+ // Parse name.
+ let name = self.expect_identifier()?;
+
+ // Ensure the preceding expression is a (program) identifier.
+ let program: Identifier = match expr {
+ Expression::Identifier(identifier) => identifier,
+ _ => unreachable!("Function called must be preceded by a program identifier."),
+ };
+
+ // Parsing a '{' means that user is trying to illegally define an external record.
+ if self.token.token == Token::LeftCurly {
+ return Err(ParserError::cannot_define_external_record(expr.span() + name.span()).into());
+ }
+
+ // If there is no parenthesis, then it is a locator.
+ if self.token.token != Token::LeftParen {
+ // Parse an external resource locator.
+ return Ok(Expression::Locator(LocatorExpression {
+ program: ProgramId {
+ name: program,
+ network: Identifier { name: sym::aleo, span: network_span, id: self.node_builder.next_id() },
+ },
+ name: name.name,
+ span: expr.span() + name.span(),
+ id: self.node_builder.next_id(),
+ }));
+ }
+
+ // Parse the function call.
+ let (arguments, _, span) = self.parse_paren_comma_list(|p| p.parse_expression().map(Some))?;
+
+ Ok(Expression::Call(CallExpression {
+ span: expr.span() + span,
+ function: Box::new(Expression::Identifier(name)),
+ program: Some(program.name),
+ arguments,
+ id: self.node_builder.next_id(),
+ }))
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent an
+ /// array access, struct member access, tuple access, or method call expression.
+ ///
+ /// Otherwise, tries to parse the next token using [`parse_primary_expression`].
+ /// Note that, as mentioned in [`parse_primary_expression`],
+ /// this function also completes the parsing of some primary expressions
+ /// (as defined in the ABNF grammar),
+ /// which [`parse_primary_expression`] only starts to parse.
+ fn parse_postfix_expression(&mut self) -> Result {
+ // We don't directly parse named types and identifiers in associated constants and functions
+ // here as the ABNF states. Rather, those named types and identifiers are parsed
+ // as primary expressions, and combined to form associated constants and functions here.
+ let mut expr = self.parse_primary_expression()?;
+ loop {
+ if self.eat(&Token::Dot) {
+ if self.check_int() {
+ // Eat a tuple member access.
+ let (index, span) = self.eat_whole_number()?;
+ expr = Expression::Access(AccessExpression::Tuple(TupleAccess {
+ tuple: Box::new(expr),
+ index,
+ span,
+ id: self.node_builder.next_id(),
+ }))
+ } else if self.eat(&Token::Leo) {
+ return Err(ParserError::only_aleo_external_calls(expr.span()).into());
+ } else if self.eat(&Token::Aleo) {
+ if self.token.token == Token::Div {
+ expr = self.parse_external_resource(expr, self.prev_token.span)?;
+ } else {
+ // Parse as address literal, e.g. `hello.aleo`.
+ if !matches!(expr, Expression::Identifier(_)) {
+ self.emit_err(ParserError::unexpected(expr.to_string(), "an identifier", expr.span()))
+ }
+
+ expr = Expression::Literal(Literal::Address(
+ format!("{}.aleo", expr),
+ expr.span(),
+ self.node_builder.next_id(),
+ ))
+ }
+ } else {
+ // Parse instances of `self.address`.
+ if let Expression::Identifier(id) = expr {
+ if id.name == sym::SelfLower && self.token.token == Token::Address {
+ let span = self.expect(&Token::Address)?;
+ // Convert `self.address` to the current program name. TODO: Move this conversion to canonicalization pass when the new pass is added.
+ // Note that the unwrap is safe as in order to get to this stage of parsing a program name must have already been parsed.
+ return Ok(Expression::Literal(Literal::Address(
+ format!("{}.aleo", self.program_name.unwrap()),
+ expr.span() + span,
+ self.node_builder.next_id(),
+ )));
+ }
+ }
+
+ // Parse identifier name.
+ let name = self.expect_identifier()?;
+
+ if self.check(&Token::LeftParen) {
+ // Eat a method call on a type
+ expr = self.parse_method_call_expression(expr, name)?
+ } else {
+ // Eat a struct member access.
+ expr = Expression::Access(AccessExpression::Member(MemberAccess {
+ span: expr.span() + name.span(),
+ inner: Box::new(expr),
+ name,
+ id: self.node_builder.next_id(),
+ }))
+ }
+ }
+ } else if self.eat(&Token::DoubleColon) {
+ // Eat a core associated constant or core associated function call.
+ expr = self.parse_associated_access_expression(expr)?;
+ } else if self.eat(&Token::LeftSquare) {
+ // Eat an array access.
+ let index = self.parse_expression()?;
+ // Eat the closing bracket.
+ let span = self.expect(&Token::RightSquare)?;
+ expr = Expression::Access(AccessExpression::Array(ArrayAccess {
+ span: expr.span() + span,
+ array: Box::new(expr),
+ index: Box::new(index),
+ id: self.node_builder.next_id(),
+ }))
+ } else if self.check(&Token::LeftParen) {
+ // Check that the expression is an identifier.
+ if !matches!(expr, Expression::Identifier(_)) {
+ self.emit_err(ParserError::unexpected(expr.to_string(), "an identifier", expr.span()))
+ }
+ // Parse a function call that's by itself.
+ let (arguments, _, span) = self.parse_paren_comma_list(|p| p.parse_expression().map(Some))?;
+ expr = Expression::Call(CallExpression {
+ span: expr.span() + span,
+ function: Box::new(expr),
+ program: self.program_name,
+ arguments,
+ id: self.node_builder.next_id(),
+ });
+ }
+ // Stop parsing the postfix expression unless a dot or square bracket follows.
+ if !(self.check(&Token::Dot) || self.check(&Token::LeftSquare)) {
+ break;
+ }
+ }
+ Ok(expr)
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent
+ /// a parenthesized expression or a unit expression
+ /// or a tuple initialization expression or an affine group literal.
+ fn parse_tuple_expression(&mut self) -> Result {
+ if let Some(gt) = self.eat_group_partial().transpose()? {
+ return Ok(Expression::Literal(Literal::Group(Box::new(GroupLiteral::Tuple(gt)))));
+ }
+
+ let (mut elements, trailing, span) = self.parse_expr_tuple()?;
+
+ match elements.len() {
+ // If the tuple expression is empty, return a `UnitExpression`.
+ 0 => Ok(Expression::Unit(UnitExpression { span, id: self.node_builder.next_id() })),
+ 1 => match trailing {
+ // If there is one element in the tuple but no trailing comma, e.g `(foo)`, return the element.
+ false => Ok(elements.swap_remove(0)),
+ // If there is one element in the tuple and a trailing comma, e.g `(foo,)`, emit an error since tuples must have at least two elements.
+ true => Err(ParserError::tuple_must_have_at_least_two_elements("expression", span).into()),
+ },
+ // Otherwise, return a tuple expression.
+ // Note: This is the only place where `TupleExpression` is constructed in the parser.
+ _ => Ok(Expression::Tuple(TupleExpression { elements, span, id: self.node_builder.next_id() })),
+ }
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent an array initialization expression.
+ fn parse_array_expression(&mut self) -> Result {
+ let (elements, _, span) = self.parse_bracket_comma_list(|p| p.parse_expression().map(Some))?;
+
+ match elements.is_empty() {
+ // If the array expression is empty, return an error.
+ true => Err(ParserError::array_must_have_at_least_one_element("expression", span).into()),
+ // Otherwise, return an array expression.
+ // Note: This is the only place where `ArrayExpression` is constructed in the parser.
+ false => Ok(Expression::Array(ArrayExpression { elements, span, id: self.node_builder.next_id() })),
+ }
+ }
+
+ /// Returns a reference to the next token if it is a [`GroupCoordinate`], or [None] if
+ /// the next token is not a [`GroupCoordinate`].
+ fn peek_group_coordinate(&self, dist: &mut usize) -> Option {
+ let (advanced, gc) = self.look_ahead(*dist, |t0| match &t0.token {
+ Token::Add => Some((1, GroupCoordinate::SignHigh)),
+ Token::Sub => self.look_ahead(*dist + 1, |t1| match &t1.token {
+ Token::Integer(value) => Some((2, GroupCoordinate::Number(format!("-{value}"), t1.span))),
+ _ => Some((1, GroupCoordinate::SignLow)),
+ }),
+ Token::Underscore => Some((1, GroupCoordinate::Inferred)),
+ Token::Integer(value) => Some((1, GroupCoordinate::Number(value.clone(), t0.span))),
+ _ => None,
+ })?;
+ *dist += advanced;
+ Some(gc)
+ }
+
+ /// Attempts to parse an affine group literal, if present.
+ /// If absent, returns [None].
+ fn eat_group_partial(&mut self) -> Option> {
+ assert!(self.check(&Token::LeftParen)); // `(`.
+
+ // Peek at first group coordinate.
+ let start_span = &self.token.span;
+ let mut dist = 1; // 0th is `(` so 1st is first group coordinate's start.
+ let first_gc = self.peek_group_coordinate(&mut dist)?;
+
+ let check_ahead = |d, token: &_| self.look_ahead(d, |t| (&t.token == token).then_some(t.span));
+
+ // Peek at `,`.
+ check_ahead(dist, &Token::Comma)?;
+ dist += 1; // Standing at `,` so advance one for next gc's start.
+
+ // Peek at second group coordinate.
+ let second_gc = self.peek_group_coordinate(&mut dist)?;
+
+ // Peek at `)`.
+ let right_paren_span = check_ahead(dist, &Token::RightParen)?;
+ dist += 1; // Standing at `)` so advance one for 'group'.
+
+ // Peek at `group`.
+ let end_span = check_ahead(dist, &Token::Group)?;
+ dist += 1; // Standing at `)` so advance one for 'group'.
+
+ let gt =
+ GroupTuple { span: start_span + &end_span, x: first_gc, y: second_gc, id: self.node_builder.next_id() };
+
+ // Eat everything so that this isn't just peeking.
+ for _ in 0..dist {
+ self.bump();
+ }
+
+ // Ensure that the ending `)` and `group` are treated as one token `)group` as in the ABNF grammar:
+ if let Err(e) = assert_no_whitespace(right_paren_span, end_span, &format!("({},{})", gt.x, gt.y), "group") {
+ return Some(Err(e));
+ }
+
+ Some(Ok(gt))
+ }
+
+ fn parse_struct_member(&mut self) -> Result {
+ let identifier = self.expect_identifier()?;
+
+ let (expression, span) = if self.eat(&Token::Colon) {
+ // Parse individual struct variable declarations.
+ let expression = self.parse_expression()?;
+ let span = identifier.span + expression.span();
+ (Some(expression), span)
+ } else {
+ (None, identifier.span)
+ };
+
+ Ok(StructVariableInitializer { identifier, expression, id: self.node_builder.next_id(), span })
+ }
+
+ /// Returns an [`Expression`] AST node if the next tokens represent a
+ /// struct initialization expression.
+ /// let foo = Foo { x: 1u8 };
+ pub fn parse_struct_init_expression(&mut self, identifier: Identifier) -> Result {
+ let (members, _, end) =
+ self.parse_list(Delimiter::Brace, Some(Token::Comma), |p| p.parse_struct_member().map(Some))?;
+
+ Ok(Expression::Struct(StructExpression {
+ span: identifier.span + end,
+ name: identifier,
+ members,
+ id: self.node_builder.next_id(),
+ }))
+ }
+
+ /// Returns an [`Expression`] AST node if the next token is a primary expression:
+ /// - Literals: field, group, unsigned integer, signed integer, boolean, address, string
+ /// - Aggregate type constructors: array, tuple, structs
+ /// - Identifiers: variables, keywords
+ ///
+ /// This function only parses some of the primary expressions defined in the ABNF grammar;
+ /// for the others, it parses their initial parts,
+ /// leaving it to the [self.parse_postfix_expression] function to complete the parsing.
+ /// For example, of the primary expression `u8::c`, this function only parses the `u8` part,
+ /// leaving it to [self.parse_postfix_expression] to parse the `::c` part.
+ /// So technically the expression returned by this function may not quite be
+ /// an expression as defined in the ABNF grammar,
+ /// but it is only a temporary expression that is combined into a larger one
+ /// by [self.parse_postfix_expression], yielding an actual expression according to the grammar.
+ ///
+ /// Returns an expression error if the token cannot be matched.
+ fn parse_primary_expression(&mut self) -> Result {
+ if let Token::LeftParen = self.token.token {
+ return self.parse_tuple_expression();
+ } else if let Token::LeftSquare = self.token.token {
+ return self.parse_array_expression();
+ }
+
+ let SpannedToken { token, span } = self.token.clone();
+ self.bump();
+
+ Ok(match token {
+ Token::Integer(value) => {
+ let suffix_span = self.token.span;
+ let full_span = span + suffix_span;
+ let assert_no_whitespace = |x| assert_no_whitespace(span, suffix_span, &value, x);
+ match self.eat_any(INT_TYPES).then_some(&self.prev_token.token) {
+ // Literal followed by `field`, e.g., `42field`.
+ Some(Token::Field) => {
+ assert_no_whitespace("field")?;
+ Expression::Literal(Literal::Field(value, full_span, self.node_builder.next_id()))
+ }
+ // Literal followed by `group`, e.g., `42group`.
+ Some(Token::Group) => {
+ assert_no_whitespace("group")?;
+ Expression::Literal(Literal::Group(Box::new(GroupLiteral::Single(
+ value,
+ full_span,
+ self.node_builder.next_id(),
+ ))))
+ }
+ // Literal followed by `scalar` e.g., `42scalar`.
+ Some(Token::Scalar) => {
+ assert_no_whitespace("scalar")?;
+ Expression::Literal(Literal::Scalar(value, full_span, self.node_builder.next_id()))
+ }
+ // Literal followed by other type suffix, e.g., `42u8`.
+ Some(suffix) => {
+ assert_no_whitespace(&suffix.to_string())?;
+ let int_ty = Self::token_to_int_type(suffix).expect("unknown int type token");
+ Expression::Literal(Literal::Integer(int_ty, value, full_span, self.node_builder.next_id()))
+ }
+ None => return Err(ParserError::implicit_values_not_allowed(value, span).into()),
+ }
+ }
+ Token::True => Expression::Literal(Literal::Boolean(true, span, self.node_builder.next_id())),
+ Token::False => Expression::Literal(Literal::Boolean(false, span, self.node_builder.next_id())),
+ Token::AddressLit(address_string) => {
+ if address_string.parse::>().is_err() {
+ self.emit_err(ParserError::invalid_address_lit(&address_string, span));
+ }
+ Expression::Literal(Literal::Address(address_string, span, self.node_builder.next_id()))
+ }
+ Token::StaticString(value) => {
+ Expression::Literal(Literal::String(value, span, self.node_builder.next_id()))
+ }
+ Token::Identifier(name) => {
+ let ident = Identifier { name, span, id: self.node_builder.next_id() };
+ if !self.disallow_struct_construction && self.check(&Token::LeftCurly) {
+ // Parse struct and records inits as struct expressions.
+ // Enforce struct or record type later at type checking.
+ self.parse_struct_init_expression(ident)?
+ } else {
+ Expression::Identifier(ident)
+ }
+ }
+ Token::SelfLower => {
+ Expression::Identifier(Identifier { name: sym::SelfLower, span, id: self.node_builder.next_id() })
+ }
+ Token::Block => {
+ Expression::Identifier(Identifier { name: sym::block, span, id: self.node_builder.next_id() })
+ }
+ Token::Future => {
+ Expression::Identifier(Identifier { name: sym::Future, span, id: self.node_builder.next_id() })
+ }
+ Token::Network => {
+ Expression::Identifier(Identifier { name: sym::network, span, id: self.node_builder.next_id() })
+ }
+ t if crate::cst::type_::TYPE_TOKENS.contains(&t) => Expression::Identifier(Identifier {
+ name: t.keyword_to_symbol().unwrap(),
+ span,
+ id: self.node_builder.next_id(),
+ }),
+ token => {
+ return Err(ParserError::unexpected_str(token, "expression", span).into());
+ }
+ })
+ }
+}
+
+fn assert_no_whitespace(left_span: Span, right_span: Span, left: &str, right: &str) -> Result<()> {
+ if left_span.hi != right_span.lo {
+ let error_span = Span::new(left_span.hi, right_span.lo); // The span between them.
+ return Err(ParserError::unexpected_whitespace(left, right, error_span).into());
+ }
+
+ Ok(())
+}
diff --git a/compiler/parser/src/cst/parser/file.rs b/compiler/parser/src/cst/parser/file.rs
new file mode 100644
index 0000000000..0ffae80367
--- /dev/null
+++ b/compiler/parser/src/cst/parser/file.rs
@@ -0,0 +1,509 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ParserContext<'_, N> {
+ /// Returns a [`Program`] AST if all tokens can be consumed and represent a valid Leo program.
+ pub fn parse_program(&mut self) -> Result {
+ let mut imports = IndexMap::new();
+ let mut program_scopes = IndexMap::new();
+ let mut comments = vec![];
+ // TODO: Remove restrictions on multiple program scopes
+ let mut parsed_program_scope = false;
+ let mut block = LeoBlock::None;
+ while self.has_next() {
+ match &self.token.token {
+ Token::Import => {
+ let (id, import) = self.parse_import()?;
+ imports.insert(id, (import.0, import.1, comments.clone(), Comment::None));
+ comments.clear();
+ block = LeoBlock::Import;
+ }
+ Token::Program => {
+ match parsed_program_scope {
+ // Only one program scope is allowed per file.
+ true => {
+ return Err(ParserError::only_one_program_scope_is_allowed(self.token.span).into());
+ }
+ false => {
+ parsed_program_scope = true;
+ let program_scope = self.parse_program_scope()?;
+ program_scopes.insert(program_scope.program_id.name.name,(program_scope, comments.clone()));
+ comments.clear();
+ block = LeoBlock::Program;
+ }
+ }
+ }
+ Token::CommentLine(string) => {
+ comments.push(Comment::CommentLine(string.clone()));
+ self.bump();
+ }
+ Token::CommentBlock(string) => {
+ comments.push(Comment::CommentBlock(string.clone()));
+ self.bump();
+ }
+ Token::_CommentLine(_) | Token::_CommentBlock(_) => {
+ if matches!(block, LeoBlock::Import) {
+ if let Some((key, value)) = imports.last() {
+ imports.insert(
+ key.clone(),
+ (value.0.clone(), value.1, value.2.clone(), self.get_comment(self.token.token.clone()))
+ );
+ }
+ }
+ self.bump();
+ }
+ _ => return Err(Self::unexpected_item(&self.token, &[Token::Import, Token::Program]).into()),
+ }
+ }
+
+ // Requires that at least one program scope is present.
+ if !parsed_program_scope {
+ return Err(ParserError::missing_program_scope(self.token.span).into());
+ }
+ Ok(Program { imports, stubs: IndexMap::new(), program_scopes})
+ }
+
+ fn unexpected_item(token: &SpannedToken, expected: &[Token]) -> ParserError {
+ ParserError::unexpected(
+ &token.token,
+ expected.iter().map(|x| format!("'{x}'")).collect::>().join(", "),
+ token.span,
+ )
+ }
+
+ // TODO: remove import resolution from parser.
+ /// Parses an import statement `import foo.leo;`.
+ pub(super) fn parse_import(&mut self) -> Result<(Symbol, (Program, Span))> {
+ // Parse `import`.
+ let start = self.expect(&Token::Import)?;
+
+ // Parse `foo`.
+ let import_name = self.expect_identifier()?;
+
+ // Parse `.`.
+ self.expect(&Token::Dot)?;
+
+ // Parse network, which currently must be `aleo`.
+ if !self.eat(&Token::Aleo) {
+ // Throw error for non-aleo networks.
+ return Err(ParserError::invalid_network(self.token.span).into());
+ }
+
+ let end = self.expect(&Token::Semicolon)?;
+
+ // Return the import name and the span.
+ Ok((import_name.name, (Program::default(), start + end)))
+ }
+
+ /// Parses a program scope `program foo.aleo { ... }`.
+ fn parse_program_scope(&mut self) -> Result {
+ // Parse `program` keyword.
+ let start = self.expect(&Token::Program)?;
+
+ // Parse the program name.
+ let name = self.expect_identifier()?;
+
+ // Set the program name in the context.
+ self.program_name = Some(name.name);
+
+ // Parse the `.`.
+ self.expect(&Token::Dot)?;
+
+ // Parse the program network, which must be `aleo`, otherwise throw parser error.
+ self.expect(&Token::Aleo).map_err(|_| ParserError::invalid_network(self.token.span))?;
+
+ // Construct the program id.
+ let program_id =
+ ProgramId { name, network: Identifier::new(Symbol::intern("aleo"), self.node_builder.next_id()) };
+
+ // Parse `{`.
+ self.expect(&Token::LeftCurly)?;
+
+ let mut header_comment = Comment::None;
+ let mut comments = vec![];
+
+ let mut block: ProgramBlock = ProgramBlock::None;
+ // Parse the body of the program scope.
+ let mut consts: Vec<(Symbol, (ConstDeclaration, Vec))> = Vec::new();
+ let (mut transitions, mut functions) = (Vec::new(), Vec::new());
+ let mut structs: Vec<(Symbol, (Composite, Vec))> = Vec::new();
+ let mut mappings: Vec<(Symbol, (Mapping, Vec))> = Vec::new();
+
+ while self.has_next() {
+ match &self.token.token {
+ Token::CommentLine(string) => {
+ comments.push(Comment::CommentLine(string.clone()));
+ self.bump();
+ }
+ Token::CommentBlock(string) => {
+ comments.push(Comment::CommentBlock(string.clone()));
+ self.bump();
+ }
+ Token::_CommentLine(_) | Token::_CommentBlock(_) => {
+ let comment = self.get_comment(self.token.token.clone());
+ if matches!(block, ProgramBlock::Const) {
+ if let Some((_, value)) = consts.last_mut() {
+ value.0.comment = comment;
+ }
+ }else if matches!(block, ProgramBlock::Mapping) {
+ if let Some((_, value)) = mappings.last_mut() {
+ value.0.comment = comment;
+ }
+ }else if matches!(block, ProgramBlock::Composite) {
+ if let Some((_, value)) = structs.last_mut() {
+ value.0.comment = comment;
+ }
+ }else if header_comment == Comment::None{
+ header_comment = comment.clone();
+ }
+ self.bump();
+ }
+
+ Token::Const => {
+ let declaration = self.parse_const_declaration_statement()?;
+ consts.push((Symbol::intern(&declaration.place.to_string()), (declaration, comments.clone())));
+ comments.clear();
+ block = ProgramBlock::Const;
+ }
+ Token::Struct | Token::Record => {
+ let (id, struct_) = self.parse_struct()?;
+ structs.push((id, (struct_, comments.clone())));
+ comments.clear();
+ block = ProgramBlock::Composite;
+ }
+ Token::Mapping => {
+ let (id, mapping) = self.parse_mapping()?;
+ mappings.push((id, (mapping, comments.clone())));
+ comments.clear();
+ block = ProgramBlock::Mapping;
+ }
+ Token::At | Token::Async | Token::Function | Token::Transition | Token::Inline => {
+ let (id, function) = self.parse_function()?;
+
+ // Partition into transitions and functions so that we don't have to sort later.
+ if function.variant.is_transition() {
+ transitions.push((id, (function, comments.clone())));
+ } else {
+ functions.push((id, (function, comments.clone())));
+ }
+ }
+ Token::RightCurly => break,
+ _ => {
+ return Err(Self::unexpected_item(&self.token, &[
+ Token::Const,
+ Token::Struct,
+ Token::Record,
+ Token::Mapping,
+ Token::At,
+ Token::Async,
+ Token::Function,
+ Token::Transition,
+ Token::Inline,
+ Token::RightCurly,
+ ])
+ .into());
+ }
+ }
+ }
+
+ // Parse `}`.
+ let end = self.expect(&Token::RightCurly)?;
+
+ Ok(ProgramScope {
+ program_id,
+ consts,
+ // functions: vec![],
+ functions: [transitions, functions].concat(),
+ structs,
+ mappings,
+ span: start + end,
+ comment: header_comment.clone()
+ })
+ }
+
+ /// Returns a [`Vec`] AST node if the next tokens represent a struct member.
+ fn parse_struct_members(&mut self) -> Result<(Vec<(Member, Vec)>, Span, Comment)> {
+ let mut members: Vec<(Member, Vec)> = Vec::new();
+ let mut header_comment = Comment::None;
+ let mut comments: Vec = vec![];
+ let (mut semi_colons, mut commas) = (false, false);
+ while !self.check(&Token::RightCurly) {
+ if self.is_inline_comment(self.token.token.clone()) {
+ if header_comment == Comment::None {
+ header_comment = self.get_comment(self.token.token.clone());
+ }else {
+ if let Some(member) = members.last_mut() {
+ member.0.comment = self.get_comment(self.token.token.clone());
+ }
+ }
+ self.bump();
+ continue;
+ }
+ while self.is_comment(self.token.token.clone()) {
+ comments.push(self.get_comment(self.token.token.clone()));
+ self.bump();
+ }
+ let variable: Member = self.parse_member_variable_declaration()?;
+ if self.eat(&Token::Semicolon) {
+ if commas {
+ self.emit_err(ParserError::mixed_commas_and_semicolons(self.token.span));
+ }
+ semi_colons = true;
+ }
+ if self.eat(&Token::Comma) {
+ if semi_colons {
+ self.emit_err(ParserError::mixed_commas_and_semicolons(self.token.span));
+ }
+ commas = true;
+ }
+ members.push((variable, comments.clone()));
+ comments.clear();
+ }
+ let span = self.expect(&Token::RightCurly)?;
+ Ok((members, span, header_comment))
+ }
+
+ /// Parses `IDENT: TYPE`.
+ pub(super) fn parse_typed_ident(&mut self) -> Result<(Identifier, Type, Span)> {
+ let name = self.expect_identifier()?;
+ self.expect(&Token::Colon)?;
+ let (type_, span) = self.parse_type()?;
+
+ Ok((name, type_, name.span + span))
+ }
+
+ /// Returns a [`Member`] AST node if the next tokens represent a struct member variable.
+ fn parse_member_variable_declaration(&mut self) -> Result {
+ let mode = self.parse_mode()?;
+ let (identifier, type_, span) = self.parse_typed_ident()?;
+ Ok(Member { mode, identifier, type_, span, id: self.node_builder.next_id(), comment: Comment::None })
+ }
+
+ /// Parses a struct or record definition, e.g., `struct Foo { ... }` or `record Foo { ... }`.
+ pub(super) fn parse_struct(&mut self) -> Result<(Symbol, Composite)> {
+ let is_record = matches!(&self.token.token, Token::Record);
+ let start = self.expect_any(&[Token::Struct, Token::Record])?;
+ // Check if using external type
+ let file_type = self.look_ahead(1, |t| &t.token);
+ if self.token.token == Token::Dot && (file_type == &Token::Aleo) {
+ return Err(ParserError::cannot_declare_external_struct(self.token.span).into());
+ }
+
+ let struct_name = self.expect_identifier()?;
+
+ self.expect(&Token::LeftCurly)?;
+ let (members, end, comment) = self.parse_struct_members()?;
+ // Only provide a program name for records.
+ let external = if is_record { self.program_name } else { None };
+ Ok((struct_name.name, Composite {
+ identifier: struct_name,
+ members,
+ external,
+ is_record,
+ span: start + end,
+ id: self.node_builder.next_id(),
+ comment
+ }))
+ }
+
+ /// Parses a mapping declaration, e.g. `mapping balances: address => u128`.
+ pub(super) fn parse_mapping(&mut self) -> Result<(Symbol, Mapping)> {
+ let start = self.expect(&Token::Mapping)?;
+ let identifier = self.expect_identifier()?;
+ self.expect(&Token::Colon)?;
+ let (key_type, _) = self.parse_type()?;
+ self.expect(&Token::BigArrow)?;
+ let (value_type, _) = self.parse_type()?;
+ let end = self.expect(&Token::Semicolon)?;
+ Ok((identifier.name, Mapping {
+ identifier,
+ key_type,
+ value_type,
+ span: start + end,
+ id: self.node_builder.next_id(),
+ comment: Comment::None
+ }))
+ }
+
+ // TODO: Return a span associated with the mode.
+ /// Returns a [`ParamMode`] AST node if the next tokens represent a function parameter mode.
+ pub(super) fn parse_mode(&mut self) -> Result {
+ let private = self.eat(&Token::Private).then_some(self.prev_token.span);
+ let public = self.eat(&Token::Public).then_some(self.prev_token.span);
+ let constant = self.eat(&Token::Constant).then_some(self.prev_token.span);
+
+ match (private, public, constant) {
+ (None, None, None) => Ok(Mode::None),
+ (Some(_), None, None) => Ok(Mode::Private),
+ (None, Some(_), None) => Ok(Mode::Public),
+ (None, None, Some(_)) => Ok(Mode::Constant),
+ _ => {
+ let mut spans = [private, public, constant].into_iter().flatten();
+
+ // There must exist at least one mode, since the none case is handled above.
+ let starting_span = spans.next().unwrap();
+ // Sum the spans.
+ let summed_span = spans.fold(starting_span, |span, next| span + next);
+ // Emit an error.
+ Err(ParserError::inputs_multiple_variable_modes_specified(summed_span).into())
+ }
+ }
+ }
+
+ /// Returns an [`Input`] AST node if the next tokens represent a function input.
+ fn parse_input(&mut self) -> Result {
+ let mode = self.parse_mode()?;
+ let (type_, span) = self.parse_type()?;
+ Ok(Output { mode, type_, span, id: self.node_builder.next_id() })
+ }
+
+ /// Returns an [`Annotation`] AST node if the next tokens represent an annotation.
+ fn parse_annotation(&mut self) -> Result {
+ // Parse the `@` symbol and identifier.
+ let start = self.expect(&Token::At)?;
+ let identifier = match self.token.token {
+ Token::Program => {
+ Identifier { name: sym::program, span: self.expect(&Token::Program)?, id: self.node_builder.next_id() }
+ }
+ _ => self.expect_identifier()?,
+ };
+ let span = start + identifier.span;
+
+ // TODO: Verify that this check is sound.
+ // Check that there is no whitespace or comments in between the `@` symbol and identifier.
+ match identifier.span.hi.0 - start.lo.0 > 1 + identifier.name.to_string().len() as u32 {
+ true => Err(ParserError::space_in_annotation(span).into()),
+ false => Ok(Annotation { identifier, span, id: self.node_builder.next_id() }),
+ }
+ }
+
+ /// Returns an [`(Identifier, Function)`] AST node if the next tokens represent a function name
+ /// and function definition.
+ fn parse_function(&mut self) -> Result<(Symbol, Function)> {
+ // TODO: Handle dangling annotations.
+ // Parse annotations, if they exist.
+ let mut annotations = Vec::new();
+ while self.look_ahead(0, |t| &t.token) == &Token::At {
+ annotations.push(self.parse_annotation()?)
+ }
+ // Parse a potential async signifier.
+ let (is_async, start_async) =
+ if self.token.token == Token::Async { (true, self.expect(&Token::Async)?) } else { (false, Span::dummy()) };
+ // Parse ` IDENT`, where `` is `function`, `transition`, or `inline`.
+ let (variant, start) = match self.token.token.clone() {
+ Token::Inline => (Variant::Inline, self.expect(&Token::Inline)?),
+ Token::Function => {
+ (if is_async { Variant::AsyncFunction } else { Variant::Function }, self.expect(&Token::Function)?)
+ }
+ Token::Transition => (
+ if is_async { Variant::AsyncTransition } else { Variant::Transition },
+ self.expect(&Token::Transition)?,
+ ),
+ _ => self.unexpected("'function', 'transition', or 'inline'")?,
+ };
+ let name = self.expect_identifier()?;
+
+ // Parse parameters.
+ let (inputs, ..) = self.parse_paren_comma_list(|p| p.parse_input().map(Some))?;
+
+ // Parse return type.
+ let output = match self.eat(&Token::Arrow) {
+ false => vec![],
+ true => {
+ self.disallow_struct_construction = true;
+ let output = match self.peek_is_left_par() {
+ true => self.parse_paren_comma_list(|p| p.parse_output().map(Some))?.0,
+ false => vec![self.parse_output()?],
+ };
+ self.disallow_struct_construction = false;
+ output
+ }
+ };
+
+ // Parse the function body, which must be a block,
+ // but we also allow no body and a semicolon instead (e.g. `fn foo(a:u8);`).
+ let (_has_empty_block, block) = match &self.token.token {
+ Token::LeftCurly => (false, self.parse_block()?),
+ Token::Semicolon => {
+ let semicolon = self.expect(&Token::Semicolon)?;
+ // TODO: make a distinction between empty block and no block (i.e. semicolon)
+ (true, Block { statements: Vec::new(), span: semicolon, id: self.node_builder.next_id() })
+ }
+ _ => self.unexpected("block or semicolon")?,
+ };
+
+ let span = if start_async == Span::dummy() { start + block.span } else { start_async + block.span };
+
+ Ok((
+ name.name,
+ Function::new(annotations, variant, name, inputs, output, block, span, self.node_builder.next_id()),
+ ))
+ }
+
+ pub fn get_comment(&mut self, token: Token) -> Comment{
+ match token {
+ Token::_CommentLine(string) => Comment::_CommentLine(string),
+ Token::_CommentBlock(string) => Comment::_CommentBlock(string),
+ Token::CommentLine(string) => Comment::CommentLine(string),
+ Token::CommentBlock(string) => Comment::CommentBlock(string),
+ _ => Comment::None
+ }
+ }
+ pub fn is_inline_comment(&mut self, token: Token) -> bool {
+ match token {
+ Token::_CommentLine(_) => true,
+ Token::_CommentBlock(_) => true,
+ _ => false
+ }
+ }
+ pub fn is_comment(&mut self, token: Token) -> bool {
+ match token {
+ Token::CommentLine(_) => true,
+ Token::CommentBlock(_) => true,
+ _ => false
+ }
+ }
+}
+
+use leo_span::{sym, Symbol};
\ No newline at end of file
diff --git a/compiler/parser/src/cst/parser/mod.rs b/compiler/parser/src/cst/parser/mod.rs
new file mode 100644
index 0000000000..16ed6992a5
--- /dev/null
+++ b/compiler/parser/src/cst/parser/mod.rs
@@ -0,0 +1,49 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see (
+ handler: &Handler,
+ node_builder: &NodeBuilder,
+ source: &str,
+ start_pos: BytePos,
+) -> Result {
+ let mut tokens = ParserContext::::new(handler, node_builder, crate::cst::tokenize(source, start_pos)?);
+
+ tokens.parse_program()
+}
\ No newline at end of file
diff --git a/compiler/parser/src/cst/parser/statement.rs b/compiler/parser/src/cst/parser/statement.rs
new file mode 100644
index 0000000000..847505b597
--- /dev/null
+++ b/compiler/parser/src/cst/parser/statement.rs
@@ -0,0 +1,348 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ParserContext<'_, N> {
+ /// Returns a [`Statement`] AST node if the next tokens represent a statement.
+ pub(crate) fn parse_statement(&mut self) -> Result {
+ match &self.token.token {
+ Token::Return => Ok(Statement::Return(self.parse_return_statement()?)),
+ Token::If => Ok(Statement::Conditional(self.parse_conditional_statement()?)),
+ Token::For => Ok(Statement::Iteration(Box::new(self.parse_loop_statement()?))),
+ Token::Assert | Token::AssertEq | Token::AssertNeq => Ok(self.parse_assert_statement()?),
+ Token::Let => Ok(Statement::Definition(self.parse_definition_statement()?)),
+ Token::Const => Ok(Statement::Const(self.parse_const_declaration_statement()?)),
+ Token::LeftCurly => Ok(Statement::Block(self.parse_block()?)),
+ Token::Console => Err(ParserError::console_statements_are_not_yet_supported(self.token.span).into()),
+ _ => Ok(self.parse_assign_statement()?),
+ }
+ }
+
+ /// Returns an [`AssertStatement`] AST node if the next tokens represent an assertion statement.
+ fn parse_assert_statement(&mut self) -> Result {
+ // Check which variant of the assert statement is being used.
+ // Note that `parse_assert_statement` is called only if the next token is an assertion token.
+ let is_assert = self.check(&Token::Assert);
+ let is_assert_eq = self.check(&Token::AssertEq);
+ let is_assert_neq = self.check(&Token::AssertNeq);
+ // Parse the span of the assertion statement.
+ let span = self.expect_any(&[Token::Assert, Token::AssertEq, Token::AssertNeq])?;
+ // Parse the left parenthesis token.
+ self.expect(&Token::LeftParen)?;
+ // Parse the variant.
+ let variant = match (is_assert, is_assert_eq, is_assert_neq) {
+ (true, false, false) => AssertVariant::Assert(self.parse_expression()?),
+ (false, true, false) => AssertVariant::AssertEq(self.parse_expression()?, {
+ self.expect(&Token::Comma)?;
+ self.parse_expression()?
+ }),
+ (false, false, true) => AssertVariant::AssertNeq(self.parse_expression()?, {
+ self.expect(&Token::Comma)?;
+ self.parse_expression()?
+ }),
+ _ => unreachable!("The call the `expect_any` ensures that only one of the three tokens is true."),
+ };
+ // Parse the right parenthesis token.
+ self.expect(&Token::RightParen)?;
+ // Parse the semicolon token.
+ self.expect(&Token::Semicolon)?;
+
+ // Return the assertion statement.
+ Ok(Statement::Assert(AssertStatement { variant, span, id: self.node_builder.next_id(), comment: Comment::None }))
+ }
+
+ /// Returns an [`AssignStatement`] AST node if the next tokens represent an assignment, otherwise expects an expression statement.
+ fn parse_assign_statement(&mut self) -> Result {
+ let place = self.parse_expression()?;
+
+ if self.eat_any(ASSIGN_TOKENS) {
+ // Determine the corresponding binary operation for each token, if it exists.
+ let operation = match &self.prev_token.token {
+ Token::Assign => None,
+ Token::AddAssign => Some(BinaryOperation::Add),
+ Token::SubAssign => Some(BinaryOperation::Sub),
+ Token::MulAssign => Some(BinaryOperation::Mul),
+ Token::DivAssign => Some(BinaryOperation::Div),
+ Token::RemAssign => Some(BinaryOperation::Rem),
+ Token::PowAssign => Some(BinaryOperation::Pow),
+ Token::OrAssign => Some(BinaryOperation::Or),
+ Token::AndAssign => Some(BinaryOperation::And),
+ Token::BitAndAssign => Some(BinaryOperation::BitwiseAnd),
+ Token::BitOrAssign => Some(BinaryOperation::BitwiseOr),
+ Token::BitXorAssign => Some(BinaryOperation::Xor),
+ Token::ShrAssign => Some(BinaryOperation::Shr),
+ Token::ShlAssign => Some(BinaryOperation::Shl),
+ _ => unreachable!("`parse_assign_statement` shouldn't produce this"),
+ };
+
+ let value = self.parse_expression()?;
+ self.expect(&Token::Semicolon)?;
+
+ // Construct the span for the statement.
+ let span = place.span() + value.span();
+
+ // Construct a copy of the lhs with a unique id.
+ let mut left = place.clone();
+ left.set_id(self.node_builder.next_id());
+
+ // Simplify complex assignments into simple assignments.
+ // For example, `x += 1` becomes `x = x + 1`, while simple assignments like `x = y` remain unchanged.
+ let value = match operation {
+ None => value,
+ Some(op) => Expression::Binary(BinaryExpression {
+ left: Box::new(left),
+ right: Box::new(value),
+ op,
+ span,
+ id: self.node_builder.next_id(),
+ }),
+ };
+
+ Ok(Statement::Assign(Box::new(AssignStatement { span, place, value, id: self.node_builder.next_id(), comment: Comment::None })))
+ } else {
+ // Check for `increment` and `decrement` statements. If found, emit a deprecation warning.
+ if let Expression::Call(call_expression) = &place {
+ match *call_expression.function {
+ Expression::Identifier(Identifier { name: sym::decrement, .. }) => {
+ self.emit_warning(ParserWarning::deprecated(
+ "decrement",
+ "Use `Mapping::{get, get_or_use, set, remove, contains}` for manipulating on-chain mappings.",
+ place.span(),
+ ));
+ }
+ Expression::Identifier(Identifier { name: sym::increment, .. }) => {
+ self.emit_warning(ParserWarning::deprecated(
+ "increment",
+ "Use `Mapping::{get, get_or_use, set, remove, contains}` for manipulating on-chain mappings.",
+ place.span(),
+ ));
+ }
+ _ => (),
+ }
+ }
+
+ // Parse the expression as a statement.
+ let end = self.expect(&Token::Semicolon)?;
+ Ok(Statement::Expression(ExpressionStatement {
+ span: place.span() + end,
+ expression: place,
+ id: self.node_builder.next_id(),
+ comment: Comment::None
+ }))
+ }
+ }
+
+ /// Returns a [`Block`] AST node if the next tokens represent a block of statements.
+ pub(super) fn parse_block(&mut self) -> Result {
+ self.parse_list_statement(Delimiter::Brace, None, |p| p.parse_statement().map(Some)).map(|(statements, _, span)| Block {
+ statements,
+ span,
+ id: self.node_builder.next_id(),
+ })
+ }
+
+ /// Returns a [`ReturnStatement`] AST node if the next tokens represent a return statement.
+ fn parse_return_statement(&mut self) -> Result {
+ let start = self.expect(&Token::Return)?;
+
+ let expression = match self.token.token {
+ // If the next token is a semicolon, implicitly return a unit expression, `()`.
+ Token::Semicolon => {
+ Expression::Unit(UnitExpression { span: self.token.span, id: self.node_builder.next_id() })
+ }
+ // Otherwise, attempt to parse an expression.
+ _ => self.parse_expression()?,
+ };
+ let end = self.expect(&Token::Semicolon)?;
+ let span = start + end;
+ Ok(ReturnStatement { span, expression, id: self.node_builder.next_id(), comment: Comment::None })
+ }
+
+ /// Returns a [`ConditionalStatement`] AST node if the next tokens represent a conditional statement.
+ fn parse_conditional_statement(&mut self) -> Result {
+ let start = self.expect(&Token::If)?;
+ self.disallow_struct_construction = true;
+ let expr = self.parse_conditional_expression()?;
+ self.disallow_struct_construction = false;
+ let body = self.parse_block()?;
+ let next = if self.eat(&Token::Else) {
+ let s = self.parse_statement()?;
+ if !matches!(s, Statement::Block(_) | Statement::Conditional(_)) {
+ self.emit_err(ParserError::unexpected_statement(&s, "Block or Conditional", s.span()));
+ }
+ Some(Box::new(s))
+ } else {
+ None
+ };
+
+ Ok(ConditionalStatement {
+ span: start + next.as_ref().map(|x| x.span()).unwrap_or(body.span),
+ condition: expr,
+ then: body,
+ otherwise: next,
+ id: self.node_builder.next_id(),
+ })
+ }
+
+ /// Returns an [`IterationStatement`] AST node if the next tokens represent an iteration statement.
+ fn parse_loop_statement(&mut self) -> Result {
+ let start_span = self.expect(&Token::For)?;
+ let ident = self.expect_identifier()?;
+ self.expect(&Token::Colon)?;
+ let type_ = self.parse_type()?;
+ self.expect(&Token::In)?;
+
+ // Parse iteration range.
+ let start = self.parse_expression()?;
+ self.expect(&Token::DotDot)?;
+ self.disallow_struct_construction = true;
+ let stop = self.parse_conditional_expression()?;
+ self.disallow_struct_construction = false;
+
+ let block = self.parse_block()?;
+
+ Ok(IterationStatement {
+ span: start_span + block.span,
+ variable: ident,
+ type_: type_.0,
+ start,
+ start_value: Default::default(),
+ stop,
+ stop_value: Default::default(),
+ inclusive: false,
+ block,
+ id: self.node_builder.next_id(),
+ })
+ }
+
+ /// Returns a [`ConsoleStatement`] AST node if the next tokens represent a console statement.
+ #[allow(dead_code)]
+ fn parse_console_statement(&mut self) -> Result {
+ let keyword = self.expect(&Token::Console)?;
+ self.expect(&Token::Dot)?;
+ let identifier = self.expect_identifier()?;
+ let (span, function) = match identifier.name {
+ sym::assert => {
+ self.expect(&Token::LeftParen)?;
+ let expr = self.parse_expression()?;
+ self.expect(&Token::RightParen)?;
+ (keyword + expr.span(), ConsoleFunction::Assert(expr))
+ }
+ sym::assert_eq => {
+ self.expect(&Token::LeftParen)?;
+ let left = self.parse_expression()?;
+ self.expect(&Token::Comma)?;
+ let right = self.parse_expression()?;
+ self.expect(&Token::RightParen)?;
+ (left.span() + right.span(), ConsoleFunction::AssertEq(left, right))
+ }
+ sym::assert_neq => {
+ self.expect(&Token::LeftParen)?;
+ let left = self.parse_expression()?;
+ self.expect(&Token::Comma)?;
+ let right = self.parse_expression()?;
+ self.expect(&Token::RightParen)?;
+ (left.span() + right.span(), ConsoleFunction::AssertNeq(left, right))
+ }
+ symbol => {
+ // Not sure what it is, assume it's `log`.
+ self.emit_err(ParserError::unexpected_ident(
+ symbol,
+ &["assert", "assert_eq", "assert_neq"],
+ identifier.span,
+ ));
+ (
+ Default::default(),
+ ConsoleFunction::Assert(Expression::Err(ErrExpression {
+ span: Default::default(),
+ id: self.node_builder.next_id(),
+ })),
+ )
+ }
+ };
+ self.expect(&Token::Semicolon)?;
+
+ Ok(ConsoleStatement { span: keyword + span, function, id: self.node_builder.next_id(), comment: Comment::None })
+ }
+
+ /// Returns a [`ConstDeclaration`] AST node if the next tokens represent a const declaration statement.
+ pub(super) fn parse_const_declaration_statement(&mut self) -> Result {
+ self.expect(&Token::Const)?;
+ let decl_span = self.prev_token.span;
+
+ // Parse variable name and type.
+ let (place, type_, _) = self.parse_typed_ident()?;
+
+ self.expect(&Token::Assign)?;
+ let value = self.parse_expression()?;
+ self.expect(&Token::Semicolon)?;
+
+ Ok(ConstDeclaration { span: decl_span + value.span(), place, type_, value, id: self.node_builder.next_id(), comment: Comment::None })
+ }
+
+ /// Returns a [`DefinitionStatement`] AST node if the next tokens represent a definition statement.
+ pub(super) fn parse_definition_statement(&mut self) -> Result {
+ self.expect(&Token::Let)?;
+ let decl_span = self.prev_token.span;
+ let decl_type = match &self.prev_token.token {
+ Token::Let => DeclarationType::Let,
+ // Note: Reserving for `constant` declarations.
+ _ => unreachable!("parse_definition_statement_ shouldn't produce this"),
+ };
+
+ // Parse variable name and type.
+ let place = self.parse_expression()?;
+ self.expect(&Token::Colon)?;
+ let type_ = self.parse_type()?.0;
+
+ self.expect(&Token::Assign)?;
+ let value = self.parse_expression()?;
+ self.expect(&Token::Semicolon)?;
+
+ Ok(DefinitionStatement {
+ span: decl_span + value.span(),
+ declaration_type: decl_type,
+ place,
+ type_,
+ value,
+ id: self.node_builder.next_id(),
+ comment: Comment::None
+ })
+ }
+}
diff --git a/compiler/parser/src/cst/parser/type_.rs b/compiler/parser/src/cst/parser/type_.rs
new file mode 100644
index 0000000000..85b89f2591
--- /dev/null
+++ b/compiler/parser/src/cst/parser/type_.rs
@@ -0,0 +1,159 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see ParserContext<'_, N> {
+ /// Returns a [`IntegerType`] AST node if the given token is a supported integer type, or [`None`].
+ pub(super) fn token_to_int_type(token: &Token) -> Option {
+ Some(match token {
+ Token::I8 => IntegerType::I8,
+ Token::I16 => IntegerType::I16,
+ Token::I32 => IntegerType::I32,
+ Token::I64 => IntegerType::I64,
+ Token::I128 => IntegerType::I128,
+ Token::U8 => IntegerType::U8,
+ Token::U16 => IntegerType::U16,
+ Token::U32 => IntegerType::U32,
+ Token::U64 => IntegerType::U64,
+ Token::U128 => IntegerType::U128,
+ _ => return None,
+ })
+ }
+
+ /// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a primitive type.
+ /// Also returns the span of the parsed token.
+ ///
+ /// These correspond to what the ABNF grammar calls 'named primitive types';
+ /// the 'primitive types' according to the ABNF grammar include also the unit type.
+ pub fn parse_primitive_type(&mut self) -> Result<(Type, Span)> {
+ let span = self.expect_any(TYPE_TOKENS)?;
+ Ok((
+ match &self.prev_token.token {
+ Token::Address => Type::Address,
+ Token::Bool => Type::Boolean,
+ Token::Field => Type::Field,
+ Token::Group => Type::Group,
+ Token::Scalar => Type::Scalar,
+ Token::Signature => Type::Signature,
+ Token::String => Type::String,
+ x => Type::Integer(Self::token_to_int_type(x).expect("invalid int type")),
+ },
+ span,
+ ))
+ }
+
+ /// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type.
+ /// Also returns the span of the parsed token.
+ pub fn parse_type(&mut self) -> Result<(Type, Span)> {
+ if let Some(ident) = self.eat_identifier() {
+ // Check if using external type
+ let file_type = self.look_ahead(1, |t| &t.token);
+ if self.token.token == Token::Dot && (file_type == &Token::Aleo) {
+ // Only allow `.aleo` as the network identifier
+ if file_type == &Token::Leo {
+ return Err(ParserError::invalid_network(self.token.span).into());
+ }
+
+ // Parse `.aleo/`
+ self.expect(&Token::Dot)?;
+ self.expect(&Token::Aleo)?;
+ self.expect(&Token::Div)?;
+
+ // Parse the record name
+ if let Some(record_name) = self.eat_identifier() {
+ // Return the external type
+ return Ok((
+ Type::Composite(CompositeType { id: record_name, program: Some(ident.name) }),
+ ident.span + record_name.span,
+ ));
+ } else {
+ return Err(ParserError::invalid_external_type(self.token.span).into());
+ }
+ }
+
+ Ok((Type::Composite(CompositeType { id: ident, program: self.program_name }), ident.span))
+ } else if self.token.token == Token::LeftSquare {
+ // Parse the left bracket.
+ self.expect(&Token::LeftSquare)?;
+ // Parse the element type.
+ let (element_type, _) = self.parse_type()?;
+ // Parse the semi-colon.
+ self.expect(&Token::Semicolon)?;
+ // Parse the length.
+ let (length, _) = self.eat_whole_number()?;
+ // Parse the right bracket.
+ self.expect(&Token::RightSquare)?;
+ // Return the array type.
+ Ok((Type::Array(ArrayType::new(element_type, length)), self.prev_token.span))
+ } else if self.token.token == Token::LeftParen {
+ let (types, _, span) = self.parse_paren_comma_list(|p| p.parse_type().map(Some))?;
+ match types.len() {
+ // If the parenthetical block is empty, e.g. `()` or `( )`, it should be parsed into `Unit` types.
+ 0 => Ok((Type::Unit, span)),
+ // If the parenthetical block contains a single type, e.g. `(u8)`, emit an error, since tuples must have at least two elements.
+ 1 => Err(ParserError::tuple_must_have_at_least_two_elements("type", span).into()),
+ // Otherwise, parse it into a `Tuple` type.
+ // Note: This is the only place where `Tuple` type is constructed in the parser.
+ _ => Ok((Type::Tuple(TupleType::new(types.into_iter().map(|t| t.0).collect())), span)),
+ }
+ } else if self.token.token == Token::Future {
+ // Parse the `Future` token.
+ let span = self.expect(&Token::Future)?;
+ // Parse the explicit future type, e.g. `Future`, `Future)>` etc.
+ if self.token.token == Token::Lt {
+ // Expect the sequence `<`, `Fn`.
+ self.expect(&Token::Lt)?;
+ self.expect(&Token::Fn)?;
+ // Parse the parenthesized list of function arguments.
+ let (types, _, full_span) = self.parse_paren_comma_list(|p| p.parse_type().map(Some))?;
+ // Expect the closing `>`.
+ self.expect(&Token::Gt)?;
+ Ok((
+ Type::Future(FutureType::new(types.into_iter().map(|t| t.0).collect(), None, true)),
+ span + full_span,
+ ))
+ } else {
+ Ok((Type::Future(Default::default()), span))
+ }
+ } else {
+ self.parse_primitive_type()
+ }
+ }
+}
diff --git a/compiler/parser/src/cst/tokenizer/lexer.rs b/compiler/parser/src/cst/tokenizer/lexer.rs
new file mode 100644
index 0000000000..6771b2cce3
--- /dev/null
+++ b/compiler/parser/src/cst/tokenizer/lexer.rs
@@ -0,0 +1,471 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see >) -> Option {
+ input.peek().filter(|c| c.is_ascii_alphabetic())?;
+ Some(from_fn(|| input.next_if(|c| c.is_ascii_alphanumeric() || c == &'_')).collect())
+}
+
+/// Checks if a char is a Unicode Bidirectional Override code point
+fn is_bidi_override(c: char) -> bool {
+ let i = c as u32;
+ (0x202A..=0x202E).contains(&i) || (0x2066..=0x2069).contains(&i)
+}
+
+/// Ensure that `string` contains no Unicode Bidirectional Override code points.
+fn ensure_no_bidi_override(string: &str) -> Result<()> {
+ if string.chars().any(is_bidi_override) {
+ return Err(ParserError::lexer_bidi_override().into());
+ }
+ Ok(())
+}
+
+impl Token {
+ // todo: remove this unused code or reference https://github.com/Geal/nom/blob/main/examples/string.rs
+ // // Eats the parts of the unicode character after \u.
+ // fn eat_unicode_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ // let mut unicode = String::new();
+ // // Account for the chars '\' and 'u'.
+ // let mut len = 2;
+ //
+ // if input.next_if_eq(&'{').is_some() {
+ // len += 1;
+ // } else if let Some(c) = input.next() {
+ // return Err(ParserError::lexer_unopened_escaped_unicode_char(c).into());
+ // } else {
+ // return Err(ParserError::lexer_empty_input_tendril().into());
+ // }
+ //
+ // while let Some(c) = input.next_if(|c| c != &'}') {
+ // len += 1;
+ // unicode.push(c);
+ // }
+ //
+ // if input.next_if_eq(&'}').is_some() {
+ // len += 1;
+ // } else {
+ // return Err(ParserError::lexer_unclosed_escaped_unicode_char(unicode).into());
+ // }
+ //
+ // // Max of 6 digits.
+ // // Minimum of 1 digit.
+ // if unicode.len() > 6 || unicode.is_empty() {
+ // return Err(ParserError::lexer_invalid_escaped_unicode_length(unicode).into());
+ // }
+ //
+ // if let Ok(hex) = u32::from_str_radix(&unicode, 16) {
+ // if let Some(character) = std::char::from_u32(hex) {
+ // Ok((len, Char::Scalar(character)))
+ // } else if hex <= 0x10FFFF {
+ // Ok((len, Char::NonScalar(hex)))
+ // } else {
+ // Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode).into())
+ // }
+ // } else {
+ // Err(ParserError::lexer_expected_valid_hex_char(unicode).into())
+ // }
+ // }
+
+ // // Eats the parts of the hex character after \x.
+ // fn eat_hex_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ // let mut hex = String::new();
+ // // Account for the chars '\' and 'x'.
+ // let mut len = 2;
+ //
+ // // First hex character.
+ // if let Some(c) = input.next_if(|c| c != &'\'') {
+ // len += 1;
+ // hex.push(c);
+ // } else if let Some(c) = input.next() {
+ // return Err(ParserError::lexer_expected_valid_hex_char(c).into());
+ // } else {
+ // return Err(ParserError::lexer_empty_input_tendril().into());
+ // }
+ //
+ // // Second hex character.
+ // if let Some(c) = input.next_if(|c| c != &'\'') {
+ // len += 1;
+ // hex.push(c);
+ // } else if let Some(c) = input.next() {
+ // return Err(ParserError::lexer_expected_valid_hex_char(c).into());
+ // } else {
+ // return Err(ParserError::lexer_empty_input_tendril().into());
+ // }
+ //
+ // if let Ok(ascii_number) = u8::from_str_radix(&hex, 16) {
+ // // According to RFC, we allow only values less than 128.
+ // if ascii_number > 127 {
+ // return Err(ParserError::lexer_expected_valid_hex_char(hex).into());
+ // }
+ //
+ // Ok((len, Char::Scalar(ascii_number as char)))
+ // } else {
+ // Err(ParserError::lexer_expected_valid_hex_char(hex).into())
+ // }
+ // }
+
+ // fn eat_escaped_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ // match input.next() {
+ // None => Err(ParserError::lexer_empty_input_tendril().into()),
+ // // Length of 2 to account the '\'.
+ // Some('0') => Ok((2, Char::Scalar(0 as char))),
+ // Some('t') => Ok((2, Char::Scalar(9 as char))),
+ // Some('n') => Ok((2, Char::Scalar(10 as char))),
+ // Some('r') => Ok((2, Char::Scalar(13 as char))),
+ // Some('\"') => Ok((2, Char::Scalar(34 as char))),
+ // Some('\'') => Ok((2, Char::Scalar(39 as char))),
+ // Some('\\') => Ok((2, Char::Scalar(92 as char))),
+ // Some('u') => Self::eat_unicode_char(input),
+ // Some('x') => Self::eat_hex_char(input),
+ // Some(c) => Err(ParserError::lexer_expected_valid_escaped_char(c).into()),
+ // }
+ // }
+
+ // /// Returns a `char` if a character can be eaten, otherwise returns [`None`].
+ // fn eat_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ // match input.next() {
+ // None => Err(ParserError::lexer_empty_input_tendril().into()),
+ // Some('\\') => Self::eat_escaped_char(input),
+ // Some(c) => Ok((c.len_utf8(), Char::Scalar(c))),
+ // }
+ // }
+
+ /// Returns a tuple: [(integer length, integer token)] if an integer can be eaten.
+ /// An integer can be eaten if its characters are at the front of the given `input` string.
+ /// If there is no input, this function returns an error.
+ /// If there is input but no integer, this function returns the tuple consisting of
+ /// length 0 and a dummy integer token that contains an empty string.
+ /// However, this function is always called when the next character is a digit.
+ /// This function eats a sequence of one or more digits and underscores
+ /// (starting from a digit, as explained above, given when it is called),
+ /// which corresponds to a numeral in the ABNF grammar.
+ fn eat_integer(input: &mut Peekable>) -> Result<(usize, Token)> {
+ if input.peek().is_none() {
+ return Err(ParserError::lexer_empty_input().into());
+ }
+
+ let mut int = String::new();
+
+ // Note that it is still impossible to have a number that starts with an `_` because eat_integer is only called when the first character is a digit.
+ while let Some(c) = input.next_if(|c| c.is_ascii_digit() || *c == '_') {
+ if c == '0' && matches!(input.peek(), Some('x')) {
+ int.push(c);
+ int.push(input.next().unwrap());
+ return Err(ParserError::lexer_hex_number_provided(int).into());
+ }
+
+ int.push(c);
+ }
+
+ Ok((int.len(), Token::Integer(int)))
+ }
+
+ /// Returns a tuple: [(token length, token)] if the next token can be eaten, otherwise returns an error.
+ /// The next token can be eaten if the characters at the front of the given `input` string can be scanned into a token.
+ pub(crate) fn eat(input: &str) -> Result<(usize, Token)> {
+ if input.is_empty() {
+ return Err(ParserError::lexer_empty_input().into());
+ }
+
+ let input_str = input;
+ let mut input = input.chars().peekable();
+
+ // Returns one token matching one character.
+ let match_one = |input: &mut Peekable<_>, token| {
+ input.next();
+ Ok((1, token))
+ };
+
+ // Returns one token matching one or two characters.
+ // If the `second` character matches, return the `second_token` that represents two characters.
+ // Otherwise, return the `first_token` that matches the one character.
+ let match_two = |input: &mut Peekable<_>, first_token, second_char, second_token| {
+ input.next();
+ Ok(if input.next_if_eq(&second_char).is_some() { (2, second_token) } else { (1, first_token) })
+ };
+
+ // Returns one token matching one or two characters.
+ // If the `second_char` character matches, return the `second_token` that represents two characters.
+ // If the `third_char` character matches, return the `third_token` that represents two characters.
+ // Otherwise, return the `first_token` that matches the one character.
+ let match_three = |input: &mut Peekable<_>, first_token, second_char, second_token, third_char, third_token| {
+ input.next();
+ Ok(if input.next_if_eq(&second_char).is_some() {
+ (2, second_token)
+ } else if input.next_if_eq(&third_char).is_some() {
+ (2, third_token)
+ } else {
+ (1, first_token)
+ })
+ };
+
+ // Returns one token matching one, two, or three characters.
+ // The `fourth_token` expects both the `third_char` and `fourth_char` to be present.
+ // See the example with the different combinations for Mul, MulAssign, Pow, PowAssign below.
+ let match_four = |
+ input: &mut Peekable<_>,
+ first_token, // e.e. Mul '*'
+ second_char, // e.g. '='
+ second_token, // e.g. MulAssign '*='
+ third_char, // e.g. '*'
+ third_token, // e.g. Pow '**'
+ fourth_char, // e.g. '='
+ fourth_token // e.g. PowAssign '**='
+ | {
+ input.next();
+ Ok(if input.next_if_eq(&second_char).is_some() {
+ // '*='
+ (2, second_token)
+ } else if input.next_if_eq(&third_char).is_some() {
+ if input.next_if_eq(&fourth_char).is_some() {
+ // '**='
+ return Ok((3, fourth_token))
+ }
+ // '**'
+ (2, third_token)
+ } else {
+ // '*'
+ (1, first_token)
+ })
+ };
+
+ match *input.peek().ok_or_else(ParserError::lexer_empty_input)? {
+ x if matches!(x, '\n') => return match_one(&mut input, Token::NewLine),
+ x if x.is_ascii_whitespace() => return match_one(&mut input, Token::WhiteSpace),
+ '"' => {
+ // Find end string quotation mark.
+ // Instead of checking each `char` and pushing, we can avoid reallocations.
+ // This works because the code 34 of double quote cannot appear as a byte
+ // in the middle of a multi-byte UTF-8 encoding of a character,
+ // because those bytes all have the high bit set to 1;
+ // in UTF-8, the byte 34 can only appear as the single-byte encoding of double quote.
+ let rest = &input_str[1..];
+ let string = match rest.as_bytes().iter().position(|c| *c == b'"') {
+ None => return Err(ParserError::lexer_string_not_closed(rest).into()),
+ Some(idx) => rest[..idx].to_owned(),
+ };
+
+ ensure_no_bidi_override(&string)?;
+
+ // + 2 to account for parsing quotation marks.
+ return Ok((string.len() + 2, Token::StaticString(string)));
+ }
+
+ x if x.is_ascii_digit() => return Self::eat_integer(&mut input),
+ '!' => return match_two(&mut input, Token::Not, '=', Token::NotEq),
+ '?' => return match_one(&mut input, Token::Question),
+ '&' => {
+ return match_four(
+ &mut input,
+ Token::BitAnd,
+ '=',
+ Token::BitAndAssign,
+ '&',
+ Token::And,
+ '=',
+ Token::AndAssign,
+ );
+ }
+ '(' => return match_one(&mut input, Token::LeftParen),
+ ')' => return match_one(&mut input, Token::RightParen),
+ '_' => return match_one(&mut input, Token::Underscore),
+ '*' => {
+ return match_four(
+ &mut input,
+ Token::Mul,
+ '=',
+ Token::MulAssign,
+ '*',
+ Token::Pow,
+ '=',
+ Token::PowAssign,
+ );
+ }
+ '+' => return match_two(&mut input, Token::Add, '=', Token::AddAssign),
+ ',' => return match_one(&mut input, Token::Comma),
+ '-' => return match_three(&mut input, Token::Sub, '=', Token::SubAssign, '>', Token::Arrow),
+ '.' => return match_two(&mut input, Token::Dot, '.', Token::DotDot),
+ '/' => {
+ input.next();
+ if input.next_if_eq(&'/').is_some() {
+ // Find the end of the comment line.
+ // This works because the code 10 of line feed cannot appear as a byte
+ // in the middle of a multi-byte UTF-8 encoding of a character,
+ // because those bytes all have the high bit set to 1;
+ // in UTF-8, the byte 10 can only appear as the single-byte encoding of line feed.
+ let comment = match input_str.as_bytes().iter().position(|c| *c == b'\n') {
+ None => input_str,
+ Some(idx) => &input_str[..idx + 1],
+ };
+ ensure_no_bidi_override(comment)?;
+ return Ok((comment.len(), Token::CommentLine(comment.to_owned())));
+ } else if input.next_if_eq(&'*').is_some() {
+ let mut comment = String::from("/*");
+
+ if input.peek().is_none() {
+ return Err(ParserError::lexer_empty_block_comment().into());
+ }
+
+ let mut ended = false;
+ while let Some(c) = input.next() {
+ comment.push(c);
+ if c == '*' && input.next_if_eq(&'/').is_some() {
+ comment.push('/');
+ ended = true;
+ break;
+ }
+ }
+
+ ensure_no_bidi_override(&comment)?;
+
+ if !ended {
+ return Err(ParserError::lexer_block_comment_does_not_close_before_eof(comment).into());
+ }
+ return Ok((comment.len(), Token::CommentBlock(comment)));
+ } else if input.next_if_eq(&'=').is_some() {
+ // '/='
+ return Ok((2, Token::DivAssign));
+ }
+ // '/'
+ return Ok((1, Token::Div));
+ }
+ '%' => return match_two(&mut input, Token::Rem, '=', Token::RemAssign),
+ ':' => return match_two(&mut input, Token::Colon, ':', Token::DoubleColon),
+ ';' => return match_one(&mut input, Token::Semicolon),
+ '<' => return match_four(&mut input, Token::Lt, '=', Token::LtEq, '<', Token::Shl, '=', Token::ShlAssign),
+ '>' => return match_four(&mut input, Token::Gt, '=', Token::GtEq, '>', Token::Shr, '=', Token::ShrAssign),
+ '=' => return match_three(&mut input, Token::Assign, '=', Token::Eq, '>', Token::BigArrow),
+ '[' => return match_one(&mut input, Token::LeftSquare),
+ ']' => return match_one(&mut input, Token::RightSquare),
+ '{' => return match_one(&mut input, Token::LeftCurly),
+ '}' => return match_one(&mut input, Token::RightCurly),
+ '|' => {
+ return match_four(
+ &mut input,
+ Token::BitOr,
+ '=',
+ Token::BitOrAssign,
+ '|',
+ Token::Or,
+ '=',
+ Token::OrAssign,
+ );
+ }
+ '^' => return match_two(&mut input, Token::BitXor, '=', Token::BitXorAssign),
+ '@' => return Ok((1, Token::At)),
+ _ => (),
+ }
+ if let Some(identifier) = eat_identifier(&mut input) {
+ return Ok((
+ identifier.len(),
+ // todo: match on symbols instead of hard-coded &str's
+ match &*identifier {
+ x if x.starts_with("aleo1") => Token::AddressLit(identifier),
+ "address" => Token::Address,
+ "aleo" => Token::Aleo,
+ "as" => Token::As,
+ "assert" => Token::Assert,
+ "assert_eq" => Token::AssertEq,
+ "assert_neq" => Token::AssertNeq,
+ "async" => Token::Async,
+ "block" => Token::Block,
+ "bool" => Token::Bool,
+ "console" => Token::Console,
+ "const" => Token::Const,
+ "constant" => Token::Constant,
+ "else" => Token::Else,
+ "false" => Token::False,
+ "field" => Token::Field,
+ "Fn" => Token::Fn,
+ "for" => Token::For,
+ "function" => Token::Function,
+ "Future" => Token::Future,
+ "group" => Token::Group,
+ "i8" => Token::I8,
+ "i16" => Token::I16,
+ "i32" => Token::I32,
+ "i64" => Token::I64,
+ "i128" => Token::I128,
+ "if" => Token::If,
+ "import" => Token::Import,
+ "in" => Token::In,
+ "inline" => Token::Inline,
+ "let" => Token::Let,
+ "leo" => Token::Leo,
+ "mapping" => Token::Mapping,
+ "private" => Token::Private,
+ "program" => Token::Program,
+ "public" => Token::Public,
+ "record" => Token::Record,
+ "return" => Token::Return,
+ "scalar" => Token::Scalar,
+ "self" => Token::SelfLower,
+ "signature" => Token::Signature,
+ "string" => Token::String,
+ "struct" => Token::Struct,
+ "transition" => Token::Transition,
+ "true" => Token::True,
+ "u8" => Token::U8,
+ "u16" => Token::U16,
+ "u32" => Token::U32,
+ "u64" => Token::U64,
+ "u128" => Token::U128,
+ _ => Token::Identifier(Symbol::intern(&identifier)),
+ },
+ ));
+ }
+
+ Err(ParserError::could_not_lex(input.take_while(|c| *c != ';' && !c.is_whitespace()).collect::())
+ .into())
+ }
+}
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct SpannedToken {
+ pub token: Token,
+ pub span: Span,
+}
+
+impl SpannedToken {
+ /// Returns a dummy token at a dummy span.
+ pub const fn dummy() -> Self {
+ Self { token: Token::Question, span: Span::dummy() }
+ }
+}
+
+impl fmt::Display for SpannedToken {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "'{}' @ ", self.token.to_string().trim())?;
+ self.span.fmt(f)
+ }
+}
+
+impl fmt::Debug for SpannedToken {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ ::fmt(self, f)
+ }
+}
diff --git a/compiler/parser/src/cst/tokenizer/mod.rs b/compiler/parser/src/cst/tokenizer/mod.rs
new file mode 100644
index 0000000000..8b8a8d6668
--- /dev/null
+++ b/compiler/parser/src/cst/tokenizer/mod.rs
@@ -0,0 +1,280 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see > {
+ tokenize_iter(input, start_pos).collect()
+}
+
+/// Yields spanned tokens from the given source code text.
+///
+/// The `lo` byte position determines where spans will start.
+pub(crate) fn tokenize_iter(mut input: &str, mut lo: BytePos) -> impl '_ + Iterator- > {
+ let mut prev_prev = Token::WhiteSpace;
+ let mut prev = Token::WhiteSpace;
+ iter::from_fn(move || {
+ while !input.is_empty() {
+ let (token_len, token) = match Token::eat(input) {
+ Err(e) => return Some(Err(e)),
+ Ok(t) => t,
+ };
+ input = &input[token_len..];
+
+ let span = Span::new(lo, lo + BytePos::from_usize(token_len));
+ lo = span.hi;
+ match token {
+ Token::WhiteSpace => continue,
+ Token::NewLine => {
+ if prev != Token::NewLine {
+ prev_prev = prev.clone();
+ prev = token.clone();
+ }
+ continue;
+ }
+ Token::CommentBlock(string) => {
+ if matches!(prev, Token::Semicolon) {
+ return Some(Ok(SpannedToken { token: Token::_CommentBlock(string), span }));
+ }else if matches!(prev, Token::LeftCurly){
+ prev_prev = prev.clone();
+ prev = Token::CommentBlock(string.clone());
+ return Some(Ok(SpannedToken { token: Token::_CommentBlock(string), span }));
+ }else if matches!(prev, Token::Comma){
+ prev_prev = prev.clone();
+ prev = Token::CommentBlock(string.clone());
+ return Some(Ok(SpannedToken { token: Token::_CommentBlock(string), span }));
+ }else if matches!(prev, Token::NewLine){
+ if
+ matches!(prev_prev, Token::CommentLine(_)) ||
+ matches!(prev_prev, Token::CommentBlock(_)) ||
+ matches!(prev_prev, Token::Semicolon) ||
+ matches!(prev_prev, Token::LeftCurly) ||
+ matches!(prev_prev, Token::RightCurly) ||
+ matches!(prev_prev, Token::Comma)
+ {
+ return Some(Ok(SpannedToken { token: Token::CommentBlock(string), span }));
+ }
+ }else if matches!(prev, Token::WhiteSpace) & matches!(prev_prev, Token::WhiteSpace){
+ return Some(Ok(SpannedToken { token: Token::CommentBlock(string), span }));
+ }else if matches!(prev, Token::CommentLine(_)){
+ return Some(Ok(SpannedToken { token: Token::CommentBlock(string), span }));
+ }
+ continue;
+ }
+ Token::CommentLine(string) => {
+ if matches!(prev, Token::Semicolon) {
+ prev_prev = prev.clone();
+ prev = Token::CommentLine(string.clone());
+ return Some(Ok(SpannedToken { token: Token::_CommentLine(string), span }));
+ }else if matches!(prev, Token::LeftCurly){
+ prev_prev = prev.clone();
+ prev = Token::CommentLine(string.clone());
+ return Some(Ok(SpannedToken { token: Token::_CommentLine(string), span }));
+ }else if matches!(prev, Token::Comma){
+ prev_prev = prev.clone();
+ prev = Token::CommentLine(string.clone());
+ return Some(Ok(SpannedToken { token: Token::_CommentLine(string), span }));
+ }else if matches!(prev, Token::NewLine){
+ if
+ matches!(prev_prev, Token::CommentLine(_)) ||
+ matches!(prev_prev, Token::CommentBlock(_)) ||
+ matches!(prev_prev, Token::Semicolon) ||
+ matches!(prev_prev, Token::LeftCurly) ||
+ matches!(prev_prev, Token::RightCurly) ||
+ matches!(prev_prev, Token::Comma)
+ {
+ return Some(Ok(SpannedToken { token: Token::CommentLine(string), span }));
+ }
+ }else if matches!(prev, Token::WhiteSpace) & matches!(prev_prev, Token::WhiteSpace){
+ return Some(Ok(SpannedToken { token: Token::CommentLine(string), span }));
+ }else if matches!(prev, Token::CommentLine(_)){
+ return Some(Ok(SpannedToken { token: Token::CommentLine(string), span }));
+ }
+ continue;
+ }
+ _ => {
+ prev_prev = prev.clone();
+ prev = token.clone();
+ return Some(Ok(SpannedToken { token, span }));
+ },
+ }
+ }
+
+ None
+ })
+}
+
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use leo_span::{source_map::FileName, symbol::create_session_if_not_set_then};
+ use std::fmt::Write;
+
+ #[test]
+ fn test_tokenizer() {
+ create_session_if_not_set_then(|s| {
+ let raw = r#"
+ "test"
+ "test{}test"
+ "test{}"
+ "{}test"
+ "test{"
+ "test}"
+ "test{test"
+ "test}test"
+ "te{{}}"
+ test_ident
+ 12345
+ address
+ as
+ assert
+ assert_eq
+ assert_neq
+ async
+ bool
+ const
+ else
+ false
+ field
+ for
+ function
+ Future
+ group
+ i128
+ i64
+ i32
+ i16
+ i8
+ if
+ in
+ inline
+ input
+ let
+ mut
+ private
+ program
+ public
+ return
+ scalar
+ self
+ signature
+ string
+ struct
+ test
+ transition
+ true
+ u128
+ u64
+ u32
+ u16
+ u8
+ console
+ !
+ !=
+ &&
+ (
+ )
+ *
+ **
+ +
+ ,
+ -
+ ->
+ =>
+ _
+ .
+ ..
+ /
+ :
+ ;
+ <
+ <=
+ =
+ ==
+ >
+ >=
+ [
+ ]
+ {{
+ }}
+ ||
+ ?
+ @
+ // test
+ /* test */
+ //"#;
+ let sf = s.source_map.new_source(raw, FileName::Custom("test".into()));
+ let tokens = tokenize(&sf.src, sf.start_pos).unwrap();
+ let mut output = String::new();
+ for SpannedToken { token, .. } in tokens.iter() {
+ write!(output, "{token} ").expect("failed to write string");
+ }
+
+ assert_eq!(
+ output,
+ r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" test_ident 12345 address as assert assert_eq assert_neq async bool const else false field for function Future group i128 i64 i32 i16 i8 if in inline input let mut private program public return scalar self signature string struct test transition true u128 u64 u32 u16 u8 console ! != && ( ) * ** + , - -> => _ . .. / : ; < <= = == > >= [ ] { { } } || ? @ // test
+ /* test */ // "#
+ );
+ });
+ }
+
+ #[test]
+ fn test_spans() {
+ create_session_if_not_set_then(|s| {
+ let raw = r#"
+ppp test
+ // test
+ test
+ /* test */
+ test
+ /* test
+ test */
+ test
+ "#;
+
+ let sm = &s.source_map;
+ let sf = sm.new_source(raw, FileName::Custom("test".into()));
+ let tokens = tokenize(&sf.src, sf.start_pos).unwrap();
+ let mut line_indices = vec![0];
+ for (i, c) in raw.chars().enumerate() {
+ if c == '\n' {
+ line_indices.push(i + 1);
+ }
+ }
+ for token in tokens.iter() {
+ assert_eq!(token.token.to_string(), sm.contents_of_span(token.span).unwrap());
+ }
+ })
+ }
+}
diff --git a/compiler/parser/src/cst/tokenizer/token.rs b/compiler/parser/src/cst/tokenizer/token.rs
new file mode 100644
index 0000000000..3fd2dc317c
--- /dev/null
+++ b/compiler/parser/src/cst/tokenizer/token.rs
@@ -0,0 +1,440 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see
{
+ Some(match self {
+ Token::Address => sym::address,
+ Token::Aleo => sym::aleo,
+ Token::As => sym::As,
+ Token::Assert => sym::assert,
+ Token::AssertEq => sym::assert_eq,
+ Token::AssertNeq => sym::assert_neq,
+ Token::Block => sym::block,
+ Token::Bool => sym::bool,
+ Token::Console => sym::console,
+ Token::Const => sym::Const,
+ Token::Constant => sym::constant,
+ Token::Else => sym::Else,
+ Token::False => sym::False,
+ Token::Field => sym::field,
+ Token::For => sym::For,
+ Token::Function => sym::function,
+ Token::Group => sym::group,
+ Token::I8 => sym::i8,
+ Token::I16 => sym::i16,
+ Token::I32 => sym::i32,
+ Token::I64 => sym::i64,
+ Token::I128 => sym::i128,
+ Token::If => sym::If,
+ Token::Import => sym::import,
+ Token::In => sym::In,
+ Token::Inline => sym::inline,
+ Token::Let => sym::Let,
+ Token::Leo => sym::leo,
+ Token::Mapping => sym::mapping,
+ Token::Network => sym::network,
+ Token::Private => sym::private,
+ Token::Program => sym::program,
+ Token::Public => sym::public,
+ Token::Record => sym::record,
+ Token::Return => sym::Return,
+ Token::Scalar => sym::scalar,
+ Token::Signature => sym::signature,
+ Token::SelfLower => sym::SelfLower,
+ Token::String => sym::string,
+ Token::Struct => sym::Struct,
+ Token::Transition => sym::transition,
+ Token::True => sym::True,
+ Token::U8 => sym::u8,
+ Token::U16 => sym::u16,
+ Token::U32 => sym::u32,
+ Token::U64 => sym::u64,
+ Token::U128 => sym::u128,
+ _ => return None,
+ })
+ }
+}
+
+impl fmt::Display for Token {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ use Token::*;
+ match self {
+ CommentLine(s) => write!(f, "{s}"),
+ CommentBlock(s) => write!(f, "{s}"),
+ _CommentLine(s) => write!(f, "{s}_inline"),
+ _CommentBlock(s) => write!(f, "{s}_inline"),
+ WhiteSpace => write!(f, "whitespace"),
+ NewLine => write!(f, "newline"),
+ True => write!(f, "true"),
+ False => write!(f, "false"),
+ Integer(s) => write!(f, "{s}"),
+ AddressLit(s) => write!(f, "{s}"),
+ StaticString(s) => write!(f, "\"{s}\""),
+
+ Identifier(s) => write!(f, "{s}"),
+
+ Not => write!(f, "!"),
+ And => write!(f, "&&"),
+ AndAssign => write!(f, "&&="),
+ Or => write!(f, "||"),
+ OrAssign => write!(f, "||="),
+ BitAnd => write!(f, "&"),
+ BitAndAssign => write!(f, "&="),
+ BitOr => write!(f, "|"),
+ BitOrAssign => write!(f, "|="),
+ BitXor => write!(f, "^"),
+ BitXorAssign => write!(f, "^="),
+ Eq => write!(f, "=="),
+ NotEq => write!(f, "!="),
+ Lt => write!(f, "<"),
+ LtEq => write!(f, "<="),
+ Gt => write!(f, ">"),
+ GtEq => write!(f, ">="),
+ Add => write!(f, "+"),
+ AddAssign => write!(f, "+="),
+ Sub => write!(f, "-"),
+ SubAssign => write!(f, "-="),
+ Mul => write!(f, "*"),
+ MulAssign => write!(f, "*="),
+ Div => write!(f, "/"),
+ DivAssign => write!(f, "/="),
+ Pow => write!(f, "**"),
+ PowAssign => write!(f, "**="),
+ Rem => write!(f, "%"),
+ RemAssign => write!(f, "%="),
+ Shl => write!(f, "<<"),
+ ShlAssign => write!(f, "<<="),
+ Shr => write!(f, ">>"),
+ ShrAssign => write!(f, ">>="),
+ Assign => write!(f, "="),
+ LeftParen => write!(f, "("),
+ RightParen => write!(f, ")"),
+ LeftSquare => write!(f, "["),
+ RightSquare => write!(f, "]"),
+ LeftCurly => write!(f, "{{"),
+ RightCurly => write!(f, "}}"),
+ Comma => write!(f, ","),
+ Dot => write!(f, "."),
+ DotDot => write!(f, ".."),
+ Semicolon => write!(f, ";"),
+ Colon => write!(f, ":"),
+ DoubleColon => write!(f, "::"),
+ Question => write!(f, "?"),
+ Arrow => write!(f, "->"),
+ BigArrow => write!(f, "=>"),
+ Underscore => write!(f, "_"),
+ At => write!(f, "@"),
+
+ Address => write!(f, "address"),
+ Bool => write!(f, "bool"),
+ Field => write!(f, "field"),
+ Group => write!(f, "group"),
+ I8 => write!(f, "i8"),
+ I16 => write!(f, "i16"),
+ I32 => write!(f, "i32"),
+ I64 => write!(f, "i64"),
+ I128 => write!(f, "i128"),
+ Record => write!(f, "record"),
+ Scalar => write!(f, "scalar"),
+ Signature => write!(f, "signature"),
+ String => write!(f, "string"),
+ Struct => write!(f, "struct"),
+ U8 => write!(f, "u8"),
+ U16 => write!(f, "u16"),
+ U32 => write!(f, "u32"),
+ U64 => write!(f, "u64"),
+ U128 => write!(f, "u128"),
+
+ Aleo => write!(f, "aleo"),
+ As => write!(f, "as"),
+ Assert => write!(f, "assert"),
+ AssertEq => write!(f, "assert_eq"),
+ AssertNeq => write!(f, "assert_neq"),
+ Async => write!(f, "async"),
+ Block => write!(f, "block"),
+ Console => write!(f, "console"),
+ Const => write!(f, "const"),
+ Constant => write!(f, "constant"),
+ Else => write!(f, "else"),
+ Fn => write!(f, "Fn"),
+ For => write!(f, "for"),
+ Function => write!(f, "function"),
+ Future => write!(f, "Future"),
+ If => write!(f, "if"),
+ Import => write!(f, "import"),
+ In => write!(f, "in"),
+ Inline => write!(f, "inline"),
+ Let => write!(f, "let"),
+ Mapping => write!(f, "mapping"),
+ Network => write!(f, "network"),
+ Private => write!(f, "private"),
+ Program => write!(f, "program"),
+ Public => write!(f, "public"),
+ Return => write!(f, "return"),
+ SelfLower => write!(f, "self"),
+ Transition => write!(f, "transition"),
+
+ Eof => write!(f, ""),
+ Leo => write!(f, "leo"),
+ }
+ }
+}
+
+/// Describes delimiters of a token sequence.
+#[derive(Copy, Clone)]
+pub enum Delimiter {
+ /// `( ... )`
+ Parenthesis,
+ /// `{ ... }`
+ Brace,
+ /// `[ ... ]`
+ Bracket,
+}
+
+impl Delimiter {
+ /// Returns the open/close tokens that the delimiter corresponds to.
+ pub fn open_close_pair(self) -> (Token, Token) {
+ match self {
+ Self::Parenthesis => (Token::LeftParen, Token::RightParen),
+ Self::Brace => (Token::LeftCurly, Token::RightCurly),
+ Self::Bracket => (Token::LeftSquare, Token::RightSquare),
+ }
+ }
+}
\ No newline at end of file
diff --git a/compiler/parser/src/lib.rs b/compiler/parser/src/lib.rs
index 82fff54479..4a31a09ad2 100644
--- a/compiler/parser/src/lib.rs
+++ b/compiler/parser/src/lib.rs
@@ -31,6 +31,7 @@ pub(crate) use tokenizer::*;
pub mod parser;
pub use parser::*;
+pub mod cst;
use leo_ast::{Ast, NodeBuilder};
use leo_errors::{Result, emitter::Handler};
diff --git a/leo/cli/cli.rs b/leo/cli/cli.rs
index 9f5a3b4465..3f916441bf 100644
--- a/leo/cli/cli.rs
+++ b/leo/cli/cli.rs
@@ -39,7 +39,7 @@ pub struct CLI {
pub home: Option,
}
-///Leo compiler and package manager
+///The Leo CLI commands
#[derive(Parser, Debug)]
enum Commands {
#[clap(about = "Create a new Aleo account, sign and verify messages")]
@@ -47,7 +47,7 @@ enum Commands {
#[clap(subcommand)]
command: Account,
},
- #[clap(about = "Create a new Leo package in a new directory")]
+ #[clap(about = "Create a new Leo project in a new directory")]
New {
#[clap(flatten)]
command: New,
@@ -57,6 +57,11 @@ enum Commands {
#[clap(flatten)]
command: Example,
},
+ #[clap(about = "Format the current Leo project", name = "fmt")]
+ Format {
+ #[clap(flatten)]
+ command: Format,
+ },
#[clap(about = "Run a program with input variables")]
Run {
#[clap(flatten)]
@@ -77,17 +82,17 @@ enum Commands {
#[clap(flatten)]
command: Query,
},
- #[clap(about = "Compile the current package as a program")]
+ #[clap(about = "Compile the current project")]
Build {
#[clap(flatten)]
command: Build,
},
- #[clap(about = "Add a new on-chain or local dependency to the current package.")]
+ #[clap(about = "Add a new on-chain or local dependency to the current project.")]
Add {
#[clap(flatten)]
command: Add,
},
- #[clap(about = "Remove a dependency from the current package.")]
+ #[clap(about = "Remove a dependency from the current project.")]
Remove {
#[clap(flatten)]
command: Remove,
@@ -137,6 +142,7 @@ pub fn run_with_args(cli: CLI) -> Result<()> {
Commands::Clean { command } => command.try_execute(context),
Commands::Deploy { command } => command.try_execute(context),
Commands::Example { command } => command.try_execute(context),
+ Commands::Format { command } => command.try_execute(context),
Commands::Run { command } => command.try_execute(context),
Commands::Execute { command } => command.try_execute(context),
Commands::Remove { command } => command.try_execute(context),
diff --git a/leo/cli/commands/format.rs b/leo/cli/commands/format.rs
new file mode 100644
index 0000000000..233055824c
--- /dev/null
+++ b/leo/cli/commands/format.rs
@@ -0,0 +1,75 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see {
+ Ok(())
+ }
+
+ fn apply(self, _: Context, _: Self::Input) -> Result<()> {
+ let input_str = self.input.as_str();
+ let output_str = self.output.as_str();
+ let input_path = Path::new(&input_str);
+ let output_path = Path::new(&output_str);
+ // Parses the Leo file constructing an ast which is then serialized.
+ let serialized_leo_tree = create_session_if_not_set_then(|s| {
+ let code = s.source_map.load_file(input_path).expect("failed to open file");
+ Handler::with(|h| {
+ let node_builder = NodeBuilder::default();
+ let cst = leo_parser::cst::parse::(h, &node_builder, &code.src, code.start_pos)?;
+ let mut output = String::new();
+ write!(output, "{}", cst.cst).unwrap();
+ Ok(output)
+ })
+ .map_err(|b| b.to_string())
+ });
+ match serialized_leo_tree {
+ Ok(tree) => {
+ fs::write(Path::new(output_path), tree).expect("failed to write output");
+ },
+ Err(e) => eprintln!("Error: {}", e),
+ };
+
+ Ok(())
+ }
+}
diff --git a/leo/cli/commands/mod.rs b/leo/cli/commands/mod.rs
index 593ed11084..c483d17a1b 100644
--- a/leo/cli/commands/mod.rs
+++ b/leo/cli/commands/mod.rs
@@ -35,6 +35,9 @@ pub use example::Example;
pub mod execute;
pub use execute::Execute;
+pub mod format;
+pub use format::Format;
+
pub mod query;
pub use query::Query;
diff --git a/utils/linter/Cargo.toml b/utils/linter/Cargo.toml
new file mode 100644
index 0000000000..ce303feec4
--- /dev/null
+++ b/utils/linter/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "leo-linter"
+version = "2.1.0"
+authors = [ "The Aleo Team " ]
+description = "A linter for the Leo programming language"
+homepage = "https://aleo.org"
+repository = "https://github.com/AleoHQ/leo"
+keywords = [
+ "aleo",
+ "cryptography",
+ "leo",
+ "programming-language",
+ "zero-knowledge"
+]
+categories = [ "compilers", "cryptography", "web-programming" ]
+include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ]
+license = "GPL-3.0"
+edition = "2021"
+rust-version = "1.69"
+
diff --git a/utils/linter/src/lib.rs b/utils/linter/src/lib.rs
new file mode 100644
index 0000000000..01efeb5a7c
--- /dev/null
+++ b/utils/linter/src/lib.rs
@@ -0,0 +1,15 @@
+// Copyright (C) 2019-2024 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see