From d69b413c8efe38443ee297295f07f06e5b2e568c Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi <35780660+anakrish@users.noreply.github.com> Date: Tue, 7 Nov 2023 23:15:28 -0800 Subject: [PATCH] Engine (#38) - Avoid lifetime parameter for Source, Span. Use Rc instead. - Engine for simplified API Signed-off-by: Anand Krishnamoorthi --- Cargo.toml | 2 +- examples/regorus.rs | 118 +++---------- src/ast.rs | 233 ++++++++++++------------- src/engine.rs | 96 ++++++++++ src/interpreter.rs | 300 +++++++++++++++----------------- src/lexer.rs | 144 ++++++++++----- src/lib.rs | 2 + src/parser.rs | 264 +++++++++++++--------------- src/scheduler.rs | 139 +++++++-------- src/utils.rs | 14 +- src/value.rs | 18 ++ tests/interpreter/mod.rs | 28 +-- tests/lexer/mod.rs | 33 +--- tests/parser/mod.rs | 10 +- tests/scheduler/analyzer/mod.rs | 12 +- 15 files changed, 722 insertions(+), 691 deletions(-) create mode 100644 src/engine.rs diff --git a/Cargo.toml b/Cargo.toml index 8f09cf51..30a4fcb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ anyhow = "1.0.66" ordered-float = "3.4.0" serde = {version = "1.0.150", features = ["derive", "rc"] } serde_json = "1.0.89" +serde_yaml = "0.9.16" log = "0.4.17" env_logger="0.10.0" lazy_static = "1.4.0" @@ -18,7 +19,6 @@ data-encoding = "2.4.0" [dev-dependencies] clap = { version = "4.4.7", features = ["derive"] } -serde_yaml = "0.9.16" test-generator = "0.3.1" walkdir = "2.3.2" diff --git a/examples/regorus.rs b/examples/regorus.rs index 2f5ec5af..06d995aa 100644 --- a/examples/regorus.rs +++ b/examples/regorus.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use clap::{Parser, Subcommand}; fn rego_eval( @@ -10,111 +10,50 @@ fn rego_eval( query: String, enable_tracing: bool, ) -> Result<()> { - // User specified data. - let mut data = regorus::Value::new_object(); + // Create engine. + let mut engine = regorus::Engine::new(); - // Read all policy files. - let mut policies = vec![]; + // Load given files. for file in files.iter() { - let contents = - std::fs::read_to_string(file).with_context(|| format!("Failed to read {file}"))?; - if file.ends_with(".rego") { - policies.push(contents); + // Read policy file. + engine.add_policy_from_file(file.to_string())?; } else { - let value: regorus::Value = if file.ends_with(".json") { - serde_json::from_str(&contents)? + // Read data file. + let data = if file.ends_with(".json") { + regorus::Value::from_json_file(file)? } else if file.ends_with(".yaml") { - serde_yaml::from_str(&contents)? + regorus::Value::from_yaml_file(file)? } else { bail!("Unsupported data file `{file}`. Must be rego, json or yaml.") }; - if let Err(err) = data.merge(value) { - bail!("Error processing {file}. {err}"); - } + // Merge given data. + engine.add_data(data)?; } } - // Create source objects. - let mut sources = vec![]; - for (idx, rego) in policies.iter().enumerate() { - sources.push(regorus::Source { - file: &files[idx], - contents: rego.as_str(), - lines: rego.split('\n').collect(), - }); - } - - // Parse the policy files. - let mut modules = vec![]; - for source in &sources { - let mut parser = regorus::Parser::new(source)?; - modules.push(parser.parse()?); - } - - // Parse input file. - let input = if let Some(file) = input { - let input_contents = std::fs::read_to_string(file.clone()) - .with_context(|| format!("Failed to read {file}"))?; - - Some(if file.ends_with(".json") { - serde_json::from_str(&input_contents)? + if let Some(file) = input { + let input = if file.ends_with(".json") { + regorus::Value::from_json_file(&file)? } else if file.ends_with(".yaml") { - serde_yaml::from_str(&input_contents)? + regorus::Value::from_yaml_file(&file)? } else { - bail!("invalid input file {file}"); - }) - } else { - None - }; - - let modules_ref: Vec<®orus::Module> = modules.iter().collect(); - - // Analyze the modules and determine how statements must be schedules. - let analyzer = regorus::Analyzer::new(); - let schedule = analyzer.analyze(&modules_ref)?; - - // Create interpreter object. - let mut interpreter = regorus::Interpreter::new(&modules_ref)?; - - // Evaluate all the modules. - interpreter.eval(&Some(data), &input, false, Some(schedule))?; - - // Parse the query. - let query_source = regorus::Source { - file: "", - contents: &query, - lines: query.split('\n').collect(), - }; - let query_span = regorus::Span { - source: &query_source, - line: 1, - col: 1, - start: 0, - end: query.len() as u16, - }; - let mut parser = regorus::Parser::new(&query_source)?; - let query_node = parser.parse_query(query_span, "")?; - let query_schedule = - regorus::Analyzer::new().analyze_query_snippet(&modules_ref, &query_node)?; - - let results = interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?; + bail!("Unsupported input file `{file}`. Must be json or yaml.") + }; + engine.set_input(input); + } + + // Evaluate query. + let results = engine.eval_query(query, enable_tracing)?; println!("{}", serde_json::to_string_pretty(&results)?); Ok(()) } fn rego_lex(file: String, verbose: bool) -> Result<()> { - let contents = - std::fs::read_to_string(file.clone()).with_context(|| format!("Failed to read {file}"))?; - // Create source. - let source = regorus::Source { - file: file.as_str(), - contents: contents.as_str(), - lines: contents.split('\n').collect(), - }; + let source = regorus::Source::from_file(file)?; // Create lexer. let mut lexer = regorus::Lexer::new(&source); @@ -138,15 +77,8 @@ fn rego_lex(file: String, verbose: bool) -> Result<()> { } fn rego_parse(file: String) -> Result<()> { - let contents = - std::fs::read_to_string(file.clone()).with_context(|| format!("Failed to read {file}"))?; - // Create source. - let source = regorus::Source { - file: file.as_str(), - contents: contents.as_str(), - lines: contents.split('\n').collect(), - }; + let source = regorus::Source::from_file(file)?; // Create a parser and parse the source. let mut parser = regorus::Parser::new(&source)?; diff --git a/src/ast.rs b/src/ast.rs index e7f2f494..be96ff0f 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -35,115 +35,115 @@ pub enum AssignOp { } #[derive(Debug, Clone)] -pub enum Expr<'source> { +pub enum Expr { // Simple items that only have a span as content. - String(Span<'source>), - RawString(Span<'source>), - Number(Span<'source>), - True(Span<'source>), - False(Span<'source>), - Null(Span<'source>), - Var(Span<'source>), + String(Span), + RawString(Span), + Number(Span), + True(Span), + False(Span), + Null(Span), + Var(Span), // array Array { - span: Span<'source>, - items: Vec>, + span: Span, + items: Vec, }, // set Set { - span: Span<'source>, - items: Vec>, + span: Span, + items: Vec, }, Object { - span: Span<'source>, - fields: Vec<(Span<'source>, Expr<'source>, Expr<'source>)>, + span: Span, + fields: Vec<(Span, Expr, Expr)>, }, // Comprehensions ArrayCompr { - span: Span<'source>, - term: Box>, - query: Query<'source>, + span: Span, + term: Box, + query: Query, }, SetCompr { - span: Span<'source>, - term: Box>, - query: Query<'source>, + span: Span, + term: Box, + query: Query, }, ObjectCompr { - span: Span<'source>, - key: Box>, - value: Box>, - query: Query<'source>, + span: Span, + key: Box, + value: Box, + query: Query, }, Call { - span: Span<'source>, - fcn: Box>, - params: Vec>, + span: Span, + fcn: Box, + params: Vec, }, UnaryExpr { - span: Span<'source>, - expr: Box>, + span: Span, + expr: Box, }, // ref RefDot { - span: Span<'source>, - refr: Box>, - field: Span<'source>, + span: Span, + refr: Box, + field: Span, }, RefBrack { - span: Span<'source>, - refr: Box>, - index: Box>, + span: Span, + refr: Box, + index: Box, }, // Infix expressions BinExpr { - span: Span<'source>, + span: Span, op: BinOp, - lhs: Box>, - rhs: Box>, + lhs: Box, + rhs: Box, }, BoolExpr { - span: Span<'source>, + span: Span, op: BoolOp, - lhs: Box>, - rhs: Box>, + lhs: Box, + rhs: Box, }, ArithExpr { - span: Span<'source>, + span: Span, op: ArithOp, - lhs: Box>, - rhs: Box>, + lhs: Box, + rhs: Box, }, AssignExpr { - span: Span<'source>, + span: Span, op: AssignOp, - lhs: Box>, - rhs: Box>, + lhs: Box, + rhs: Box, }, Membership { - span: Span<'source>, - key: Box>>, - value: Box>, - collection: Box>, + span: Span, + key: Box>, + value: Box, + collection: Box, }, } -impl<'source> Expr<'source> { - pub fn span(&self) -> &Span<'source> { +impl Expr { + pub fn span(&self) -> &Span { use Expr::*; match self { String(s) | RawString(s) | Number(s) | True(s) | False(s) | Null(s) | Var(s) => s, @@ -167,121 +167,121 @@ impl<'source> Expr<'source> { } #[derive(Debug, Clone)] -pub enum Literal<'source> { +pub enum Literal { SomeVars { - span: Span<'source>, - vars: Vec>, + span: Span, + vars: Vec, }, SomeIn { - span: Span<'source>, - key: Option>, - value: Expr<'source>, - collection: Expr<'source>, + span: Span, + key: Option, + value: Expr, + collection: Expr, }, Expr { - span: Span<'source>, - expr: Expr<'source>, + span: Span, + expr: Expr, }, NotExpr { - span: Span<'source>, - expr: Expr<'source>, + span: Span, + expr: Expr, }, Every { - span: Span<'source>, - key: Option>, - value: Span<'source>, - domain: Expr<'source>, - query: Query<'source>, + span: Span, + key: Option, + value: Span, + domain: Expr, + query: Query, }, } #[derive(Debug, Clone)] -pub struct WithModifier<'source> { - pub span: Span<'source>, - pub refr: Expr<'source>, - pub r#as: Expr<'source>, +pub struct WithModifier { + pub span: Span, + pub refr: Expr, + pub r#as: Expr, } #[derive(Debug, Clone)] -pub struct LiteralStmt<'source> { - pub span: Span<'source>, - pub literal: Literal<'source>, - pub with_mods: Vec>, +pub struct LiteralStmt { + pub span: Span, + pub literal: Literal, + pub with_mods: Vec, } #[derive(Debug, Clone)] -pub struct Query<'source> { - pub span: Span<'source>, - pub stmts: Vec>, +pub struct Query { + pub span: Span, + pub stmts: Vec, } #[derive(Debug, Clone)] -pub struct RuleAssign<'source> { - pub span: Span<'source>, +pub struct RuleAssign { + pub span: Span, pub op: AssignOp, - pub value: Expr<'source>, + pub value: Expr, } #[derive(Debug, Clone)] -pub struct RuleBody<'source> { - pub span: Span<'source>, - pub assign: Option>, - pub query: Query<'source>, +pub struct RuleBody { + pub span: Span, + pub assign: Option, + pub query: Query, } #[derive(Debug, Clone)] -pub enum RuleHead<'source> { +pub enum RuleHead { Compr { - span: Span<'source>, - refr: Expr<'source>, - assign: Option>, + span: Span, + refr: Expr, + assign: Option, }, Set { - span: Span<'source>, - refr: Expr<'source>, - key: Option>, + span: Span, + refr: Expr, + key: Option, }, Func { - span: Span<'source>, - refr: Expr<'source>, - args: Vec>, - assign: Option>, + span: Span, + refr: Expr, + args: Vec, + assign: Option, }, } #[derive(Debug, Clone)] -pub enum Rule<'source> { +pub enum Rule { Spec { - span: Span<'source>, - head: RuleHead<'source>, - bodies: Vec>, + span: Span, + head: RuleHead, + bodies: Vec, }, Default { - span: Span<'source>, - refr: Expr<'source>, + span: Span, + refr: Expr, op: AssignOp, - value: Expr<'source>, + value: Expr, }, } #[derive(Debug, Clone)] -pub struct Package<'source> { - pub span: Span<'source>, - pub refr: Expr<'source>, +pub struct Package { + pub span: Span, + pub refr: Expr, } #[derive(Debug, Clone)] -pub struct Import<'source> { - pub span: Span<'source>, - pub refr: Expr<'source>, - pub r#as: Option>, +pub struct Import { + pub span: Span, + pub refr: Expr, + pub r#as: Option, } #[derive(Debug, Clone)] -pub struct Module<'source> { - pub package: Package<'source>, - pub imports: Vec>, - pub policy: Vec>, +pub struct Module { + pub package: Package, + pub imports: Vec, + pub policy: Vec, } #[derive(Debug, Clone)] @@ -315,7 +315,6 @@ impl<'a, T> PartialOrd for Ref<'a, T> { impl<'a, T> Ord for Ref<'a, T> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - //std::ptr::from_ref(self.r).partial_cmp(std::ptr::from_ref(other.r)) (self.r as *const T).cmp(&(other.r as *const T)) } } diff --git a/src/engine.rs b/src/engine.rs new file mode 100644 index 00000000..96b87001 --- /dev/null +++ b/src/engine.rs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::ast::*; +use crate::interpreter::*; +use crate::lexer::*; +use crate::parser::*; +use crate::scheduler::*; +use crate::value::*; + +use anyhow::Result; + +#[derive(Clone)] +pub struct Engine { + modules: Vec>, + input: Value, + data: Value, +} + +impl Default for Engine { + fn default() -> Self { + Self::new() + } +} + +impl Engine { + pub fn new() -> Self { + Self { + modules: vec![], + input: Value::new_object(), + data: Value::new_object(), + } + } + + pub fn add_policy(&mut self, path: String, rego: String) -> Result<()> { + let source = Source::new(path, rego); + let mut parser = Parser::new(&source)?; + self.modules.push(std::rc::Rc::new(parser.parse()?)); + Ok(()) + } + + pub fn add_policy_from_file(&mut self, path: String) -> Result<()> { + let source = Source::from_file(path)?; + let mut parser = Parser::new(&source)?; + self.modules.push(std::rc::Rc::new(parser.parse()?)); + Ok(()) + } + + pub fn set_input(&mut self, input: Value) { + self.input = input; + } + + pub fn clear_data(&mut self) { + self.data = Value::new_object(); + } + + pub fn add_data(&mut self, data: Value) -> Result<()> { + self.data.merge(data) + } + + pub fn eval_query(&self, query: String, enable_tracing: bool) -> Result { + let modules_ref: Vec<&Module> = self.modules.iter().map(|m| &**m).collect(); + + // Analyze the modules and determine how statements must be scheduled. + let analyzer = Analyzer::new(); + let schedule = analyzer.analyze(&modules_ref)?; + + // Create interpreter object. + let mut interpreter = Interpreter::new(&modules_ref)?; + + // Evaluate all the modules. + interpreter.eval( + &Some(self.data.clone()), + &Some(self.input.clone()), + false, + Some(schedule), + )?; + + // Parse the query. + let query_len = query.len(); + let query_source = Source::new("".to_string(), query); + let query_span = Span { + source: query_source.clone(), + line: 1, + col: 1, + start: 0, + end: query_len as u16, + }; + let mut parser = Parser::new(&query_source)?; + let query_node = parser.parse_query(query_span, "")?; + let query_schedule = Analyzer::new().analyze_query_snippet(&modules_ref, &query_node)?; + + let results = interpreter.eval_user_query(&query_node, &query_schedule, enable_tracing)?; + Ok(results) + } +} diff --git a/src/interpreter.rs b/src/interpreter.rs index 77958c27..de75e8b4 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -18,8 +18,8 @@ use std::rc::Rc; type Scope = BTreeMap; pub struct Interpreter<'source> { - modules: Vec<&'source Module<'source>>, - module: Option<&'source Module<'source>>, + modules: Vec<&'source Module>, + module: Option<&'source Module>, schedule: Option>, current_module_path: String, prepared: bool, @@ -29,13 +29,13 @@ pub struct Interpreter<'source> { with_document: Value, scopes: Vec, // TODO: handle recursive calls where same expr could have different values. - loop_var_values: BTreeMap>, Value>, + loop_var_values: BTreeMap, Value>, contexts: Vec>, functions: FunctionTable<'source>, - rules: HashMap>>, - default_rules: HashMap, Option)>>, - processed: BTreeSet>>, - active_rules: Vec<&'source Rule<'source>>, + rules: HashMap>, + default_rules: HashMap)>>, + processed: BTreeSet>, + active_rules: Vec<&'source Rule>, builtins_cache: BTreeMap<(&'static str, Vec), Value>, no_rules_lookup: bool, traces: Option>, @@ -66,8 +66,8 @@ pub struct QueryResults { #[derive(Debug, Clone)] struct Context<'source> { - key_expr: Option<&'source Expr<'source>>, - output_expr: Option<&'source Expr<'source>>, + key_expr: Option<&'source Expr>, + output_expr: Option<&'source Expr>, value: Value, result: Option, results: QueryResults, @@ -75,14 +75,14 @@ struct Context<'source> { #[derive(Debug)] struct LoopExpr<'source> { - span: &'source Span<'source>, - expr: &'source Expr<'source>, - value: &'source Expr<'source>, + span: &'source Span, + expr: &'source Expr, + value: &'source Expr, index: &'source str, } impl<'source> Interpreter<'source> { - pub fn new(modules: &[&'source Module<'source>]) -> Result> { + pub fn new(modules: &[&'source Module]) -> Result> { let mut with_document = Value::new_object(); *Self::make_or_get_value_mut(&mut with_document, &["data"])? = Value::new_object(); *Self::make_or_get_value_mut(&mut with_document, &["input"])? = Value::new_object(); @@ -111,7 +111,7 @@ impl<'source> Interpreter<'source> { }) } - pub fn get_modules(&mut self) -> &mut Vec<&'source Module<'source>> { + pub fn get_modules(&mut self) -> &mut Vec<&'source Module> { &mut self.modules } @@ -149,7 +149,7 @@ impl<'source> Interpreter<'source> { Ok(()) } - fn current_module(&self) -> Result<&'source Module<'source>> { + fn current_module(&self) -> Result<&'source Module> { self.module .ok_or_else(|| anyhow!("internal error: current module not set")) } @@ -201,7 +201,7 @@ impl<'source> Interpreter<'source> { } } - fn eval_chained_ref_dot_or_brack(&mut self, mut expr: &'source Expr<'source>) -> Result { + fn eval_chained_ref_dot_or_brack(&mut self, mut expr: &'source Expr) -> Result { // Collect a chaing of '.field' or '["field"]' let mut path = vec![]; loop { @@ -218,13 +218,13 @@ impl<'source> Interpreter<'source> { // Accumulate chained . field accesses. Expr::RefDot { refr, field, .. } => { expr = refr; - path.push(field.text()); + path.push(*field.text()); } Expr::RefBrack { refr, index, .. } => match index.as_ref() { // refr["field"] is the same as refr.field Expr::String(s) => { expr = refr; - path.push(s.text()); + path.push(*s.text()); } // Handle other forms of refr. // Note, we have the choice to evaluate a non-string index @@ -263,7 +263,7 @@ impl<'source> Interpreter<'source> { } } - fn hoist_loops_impl(&self, expr: &'source Expr<'source>, loops: &mut Vec>) { + fn hoist_loops_impl(&self, expr: &'source Expr, loops: &mut Vec>) { use Expr::*; match expr { RefBrack { refr, index, span } => { @@ -272,11 +272,11 @@ impl<'source> Interpreter<'source> { // Then hoist the current bracket operation. match index.as_ref() { - Var(ident) if self.is_loop_index_var(ident.text()) => loops.push(LoopExpr { + Var(ident) if self.is_loop_index_var(*ident.text()) => loops.push(LoopExpr { span, expr, value: refr, - index: ident.text(), + index: *ident.text(), }), _ => { // hoist any loops in index expression. @@ -334,7 +334,7 @@ impl<'source> Interpreter<'source> { } } - fn hoist_loops(&self, literal: &'source Literal<'source>) -> Vec> { + fn hoist_loops(&self, literal: &'source Literal) -> Vec> { let mut loops = vec![]; use Literal::*; match literal { @@ -362,8 +362,8 @@ impl<'source> Interpreter<'source> { fn eval_bool_expr( &mut self, op: &BoolOp, - lhs_expr: &'source Expr<'source>, - rhs_expr: &'source Expr<'source>, + lhs_expr: &'source Expr, + rhs_expr: &'source Expr, ) -> Result { let lhs = self.eval_expr(lhs_expr)?; let rhs = self.eval_expr(rhs_expr)?; @@ -378,8 +378,8 @@ impl<'source> Interpreter<'source> { fn eval_bin_expr( &mut self, op: &BinOp, - lhs: &'source Expr<'source>, - rhs: &'source Expr<'source>, + lhs: &'source Expr, + rhs: &'source Expr, ) -> Result { let lhs_value = self.eval_expr(lhs)?; let rhs_value = self.eval_expr(rhs)?; @@ -397,8 +397,8 @@ impl<'source> Interpreter<'source> { fn eval_arith_expr( &mut self, op: &ArithOp, - lhs: &'source Expr<'source>, - rhs: &'source Expr<'source>, + lhs: &'source Expr, + rhs: &'source Expr, ) -> Result { let lhs_value = self.eval_expr(lhs)?; let rhs_value = self.eval_expr(rhs)?; @@ -418,8 +418,8 @@ impl<'source> Interpreter<'source> { fn eval_assign_expr( &mut self, op: &AssignOp, - lhs: &'source Expr<'source>, - rhs: &'source Expr<'source>, + lhs: &'source Expr, + rhs: &'source Expr, ) -> Result { let (name, value) = match op { AssignOp::Eq => { @@ -477,7 +477,7 @@ impl<'source> Interpreter<'source> { // TODO: Check this // Allow variable overwritten inside a loop - if self.lookup_local_var(name).is_some() + if self.lookup_local_var(*name).is_some() && self.loop_var_values.get(&Ref::make(rhs)).is_none() { bail!(rhs @@ -494,10 +494,10 @@ impl<'source> Interpreter<'source> { return Ok(Value::Bool(false)); } - self.add_variable_or(name)?; + self.add_variable_or(*name)?; // TODO: optimize this - self.variables_assignment(name, &value)?; + self.variables_assignment(*name, &value)?; info!( "eval_assign_expr before, op: {:?}, lhs: {:?}, rhs: {:?}", @@ -509,11 +509,11 @@ impl<'source> Interpreter<'source> { fn eval_every( &mut self, - _span: &'source Span<'source>, - key: &'source Option>, - value: &'source Span<'source>, - domain: &'source Expr<'source>, - query: &'source Query<'source>, + _span: &'source Span, + key: &'source Option, + value: &'source Span, + domain: &'source Expr, + query: &'source Query, ) -> Result { let domain = self.eval_expr(domain)?; @@ -529,9 +529,9 @@ impl<'source> Interpreter<'source> { match domain { Value::Array(a) => { for (idx, v) in a.iter().enumerate() { - self.add_variable(value.text(), v.clone())?; + self.add_variable(*value.text(), v.clone())?; if let Some(key) = key { - self.add_variable(key.text(), Value::from_float(idx as Float))?; + self.add_variable(*key.text(), Value::from_float(idx as Float))?; } if !self.eval_query(query)? { r = false; @@ -541,9 +541,9 @@ impl<'source> Interpreter<'source> { } Value::Set(s) => { for v in s.iter() { - self.add_variable(value.text(), v.clone())?; + self.add_variable(*value.text(), v.clone())?; if let Some(key) = key { - self.add_variable(key.text(), v.clone())?; + self.add_variable(*key.text(), v.clone())?; } if !self.eval_query(query)? { r = false; @@ -553,9 +553,9 @@ impl<'source> Interpreter<'source> { } Value::Object(o) => { for (k, v) in o.iter() { - self.add_variable(value.text(), v.clone())?; + self.add_variable(*value.text(), v.clone())?; if let Some(key) = key { - self.add_variable(key.text(), k.clone())?; + self.add_variable(*key.text(), k.clone())?; } if !self.eval_query(query)? { r = false; @@ -574,8 +574,8 @@ impl<'source> Interpreter<'source> { fn lookup_or_eval_expr( &mut self, - cache: &mut BTreeMap>, Value>, - expr: &'source Expr<'source>, + cache: &mut BTreeMap, Value>, + expr: &'source Expr, ) -> Result { match cache.get(&Ref::make(expr)) { Some(v) => Ok(v.clone()), @@ -590,9 +590,9 @@ impl<'source> Interpreter<'source> { fn make_bindings_impl( &mut self, is_last: bool, - type_match: &mut BTreeSet>>, - cache: &mut BTreeMap>, Value>, - expr: &'source Expr<'source>, + type_match: &mut BTreeSet>, + cache: &mut BTreeMap, Value>, + expr: &'source Expr, value: &Value, ) -> Result { // Propagate undefined. @@ -604,7 +604,7 @@ impl<'source> Interpreter<'source> { match (expr, value) { (Expr::Var(ident), _) => { - self.add_variable(ident.text(), value.clone())?; + self.add_variable(*ident.text(), value.clone())?; Ok(true) } @@ -688,9 +688,9 @@ impl<'source> Interpreter<'source> { fn make_bindings( &mut self, is_last: bool, - type_match: &mut BTreeSet>>, - cache: &mut BTreeMap>, Value>, - expr: &'source Expr<'source>, + type_match: &mut BTreeSet>, + cache: &mut BTreeMap, Value>, + expr: &'source Expr, value: &Value, ) -> Result { let prev = self.no_rules_lookup; @@ -703,9 +703,9 @@ impl<'source> Interpreter<'source> { fn make_key_value_bindings( &mut self, is_last: bool, - type_match: &mut BTreeSet>>, - cache: &mut BTreeMap>, Value>, - exprs: (&'source Option>, &'source Expr<'source>), + type_match: &mut BTreeSet>, + cache: &mut BTreeMap, Value>, + exprs: (&'source Option, &'source Expr), values: (&Value, &Value), ) -> Result { let (key_expr, value_expr) = exprs; @@ -720,11 +720,11 @@ impl<'source> Interpreter<'source> { fn eval_some_in( &mut self, - _span: &'source Span<'source>, - key_expr: &'source Option>, - value_expr: &'source Expr<'source>, - collection: &'source Expr<'source>, - stmts: &[&'source LiteralStmt<'source>], + _span: &'source Span, + key_expr: &'source Option, + value_expr: &'source Expr, + collection: &'source Expr, + stmts: &[&'source LiteralStmt], ) -> Result { let scope_saved = self.current_scope()?.clone(); let mut type_match = BTreeSet::new(); @@ -821,8 +821,8 @@ impl<'source> Interpreter<'source> { fn eval_stmt_impl( &mut self, - stmt: &'source LiteralStmt<'source>, - stmts: &[&'source LiteralStmt<'source>], + stmt: &'source LiteralStmt, + stmts: &[&'source LiteralStmt], ) -> Result { Ok(match &stmt.literal { Literal::Expr { span, expr, .. } => { @@ -879,7 +879,7 @@ impl<'source> Interpreter<'source> { Literal::SomeVars { span, vars, .. } => { for var in vars { let name = var.text(); - if let Ok(variable) = self.add_variable_or(name) { + if let Ok(variable) = self.add_variable_or(*name) { if variable != Value::Undefined { return Err(anyhow!( "duplicated definition of local variable {}", @@ -933,8 +933,8 @@ impl<'source> Interpreter<'source> { fn eval_stmt( &mut self, - stmt: &'source LiteralStmt<'source>, - stmts: &[&'source LiteralStmt<'source>], + stmt: &'source LiteralStmt, + stmts: &[&'source LiteralStmt], ) -> Result { let saved_state = if !stmt.with_mods.is_empty() { // Save state; @@ -948,7 +948,7 @@ impl<'source> Interpreter<'source> { // Evaluate value and ref let value = self.eval_expr(&wm.r#as)?; let path = Parser::get_path_ref_components(&wm.refr)?; - let path: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let path: Vec<&str> = path.iter().map(|s| *s.text()).collect(); if path[0] == "input" || path[0] == "data" { *Self::make_or_get_value_mut(&mut self.with_document, &path[..])? = value; @@ -986,7 +986,7 @@ impl<'source> Interpreter<'source> { fn eval_stmts_in_loop( &mut self, - stmts: &[&'source LiteralStmt<'source>], + stmts: &[&'source LiteralStmt], loops: &[LoopExpr<'source>], ) -> Result { if loops.is_empty() { @@ -1208,12 +1208,7 @@ impl<'source> Interpreter<'source> { } } - fn get_exprs_from_context( - &self, - ) -> Result<( - Option<&'source Expr<'source>>, - Option<&'source Expr<'source>>, - )> { + fn get_exprs_from_context(&self) -> Result<(Option<&'source Expr>, Option<&'source Expr>)> { let ctx = self.get_current_context()?; Ok((ctx.key_expr, ctx.output_expr)) } @@ -1242,7 +1237,7 @@ impl<'source> Interpreter<'source> { } } - fn eval_stmts(&mut self, stmts: &[&'source LiteralStmt<'source>]) -> Result { + fn eval_stmts(&mut self, stmts: &[&'source LiteralStmt]) -> Result { let mut result = true; for (idx, stmt) in stmts.iter().enumerate() { @@ -1270,28 +1265,27 @@ impl<'source> Interpreter<'source> { Ok(result) } - fn eval_query(&mut self, query: &'source Query<'source>) -> Result { + fn eval_query(&mut self, query: &'source Query) -> Result { // Execute the query in a new scope self.scopes.push(Scope::new()); - let ordered_stmts: Vec<&'source LiteralStmt<'source>> = - if let Some(schedule) = &self.schedule { - match schedule.order.get(&Ref::make(query)) { - Some(ord) => ord.iter().map(|i| &query.stmts[*i as usize]).collect(), - // TODO - _ => bail!(query - .span - .error("statements not scheduled in query {query:?}")), - } - } else { - query.stmts.iter().collect() - }; + let ordered_stmts: Vec<&'source LiteralStmt> = if let Some(schedule) = &self.schedule { + match schedule.order.get(&Ref::make(query)) { + Some(ord) => ord.iter().map(|i| &query.stmts[*i as usize]).collect(), + // TODO + _ => bail!(query + .span + .error("statements not scheduled in query {query:?}")), + } + } else { + query.stmts.iter().collect() + }; let r = self.eval_stmts(&ordered_stmts); self.scopes.pop(); r } - fn eval_array(&mut self, items: &'source Vec>) -> Result { + fn eval_array(&mut self, items: &'source Vec) -> Result { let mut array = Vec::new(); for item in items { @@ -1322,7 +1316,7 @@ impl<'source> Interpreter<'source> { Ok(Value::from_map(object)) } - fn eval_set(&mut self, items: &'source Vec>) -> Result { + fn eval_set(&mut self, items: &'source Vec) -> Result { let mut set = BTreeSet::new(); for item in items { @@ -1338,9 +1332,9 @@ impl<'source> Interpreter<'source> { fn eval_membership( &mut self, - key: &'source Option>, - value: &'source Expr<'source>, - collection: &'source Expr<'source>, + key: &'source Option, + value: &'source Expr, + collection: &'source Expr, ) -> Result { let value = self.eval_expr(value)?; let collection = self.eval_expr(collection)?; @@ -1378,11 +1372,7 @@ impl<'source> Interpreter<'source> { Ok(Value::Bool(result)) } - fn eval_array_compr( - &mut self, - term: &'source Expr<'source>, - query: &'source Query<'source>, - ) -> Result { + fn eval_array_compr(&mut self, term: &'source Expr, query: &'source Query) -> Result { // Push new context self.contexts.push(Context { key_expr: None, @@ -1401,11 +1391,7 @@ impl<'source> Interpreter<'source> { } } - fn eval_set_compr( - &mut self, - term: &'source Expr<'source>, - query: &'source Query<'source>, - ) -> Result { + fn eval_set_compr(&mut self, term: &'source Expr, query: &'source Query) -> Result { // Push new context self.contexts.push(Context { key_expr: None, @@ -1425,9 +1411,9 @@ impl<'source> Interpreter<'source> { fn eval_object_compr( &mut self, - key: &'source Expr<'source>, - value: &'source Expr<'source>, - query: &'source Query<'source>, + key: &'source Expr, + value: &'source Expr, + query: &'source Query, ) -> Result { // Push new context self.contexts.push(Context { @@ -1446,7 +1432,7 @@ impl<'source> Interpreter<'source> { } } - fn lookup_function(&self, fcn: &'source Expr<'source>) -> Result<&Vec<&'source Rule<'source>>> { + fn lookup_function(&self, fcn: &'source Expr) -> Result<&Vec<&'source Rule>> { let mut path = Self::get_path_string(fcn, None)?; if !path.starts_with("data.") { path = self.current_module_path.clone() + "." + &path; @@ -1462,10 +1448,10 @@ impl<'source> Interpreter<'source> { fn eval_builtin_call( &mut self, - span: &'source Span<'source>, + span: &'source Span, name: String, builtin: builtins::BuiltinFcn, - params: &'source [Expr<'source>], + params: &'source [Expr], ) -> Result { let mut args = vec![]; let allow_undefined = name == "print"; // TODO: with modifier @@ -1501,9 +1487,9 @@ impl<'source> Interpreter<'source> { fn eval_call_impl( &mut self, - span: &'source Span<'source>, - fcn: &'source Expr<'source>, - params: &'source [Expr<'source>], + span: &'source Span, + fcn: &'source Expr, + params: &'source [Expr], ) -> Result { let fcns_rules = match self.lookup_function(fcn) { Ok(r) => r, @@ -1637,20 +1623,20 @@ impl<'source> Interpreter<'source> { fn eval_call( &mut self, - span: &'source Span<'source>, - fcn: &'source Expr<'source>, - params: &'source Vec>, - extra_arg: Option<&'source Expr<'source>>, + span: &'source Span, + fcn: &'source Expr, + params: &'source Vec, + extra_arg: Option<&'source Expr>, allow_return_arg: bool, ) -> Result { // TODO: global var check; interop with `some var` match extra_arg { Some(Expr::Var(var)) - if allow_return_arg && self.lookup_local_var(var.text()).is_none() => + if allow_return_arg && self.lookup_local_var(*var.text()).is_none() => { let value = self.eval_call_impl(span, fcn, ¶ms[..params.len() - 1])?; - if var.text() != "_" { - self.add_variable(var.text(), value)?; + if *var.text() != "_" { + self.add_variable(*var.text(), value)?; } Ok(Value::Bool(true)) } @@ -1696,16 +1682,16 @@ impl<'source> Interpreter<'source> { Ok(()) } - fn lookup_var(&mut self, span: &'source Span<'source>, fields: &[&str]) -> Result { + fn lookup_var(&mut self, span: &'source Span, fields: &[&str]) -> Result { let name = span.text(); // Return local variable/argument. - if let Some(v) = self.lookup_local_var(name) { + if let Some(v) = self.lookup_local_var(*name) { return Ok(Self::get_value_chained(v, fields)); } // Handle input. - if name == "input" { + if *name == "input" { return Ok(Self::get_value_chained(self.input.clone(), fields)); } @@ -1715,7 +1701,7 @@ impl<'source> Interpreter<'source> { } // Ensure that rules are evaluated - if name == "data" { + if *name == "data" { let v = Self::get_value_chained(self.data.clone(), fields); // If the rule has already been evaluated or specified via a with modifier, @@ -1735,12 +1721,9 @@ impl<'source> Interpreter<'source> { Ok(Self::get_value_chained(self.data.clone(), fields)) } else if !self.modules.is_empty() { - let mut path: Vec<&str> = - Parser::get_path_ref_components(&self.module.unwrap().package.refr)? - .iter() - .map(|s| s.text()) - .collect(); - path.push(name); + let path = Parser::get_path_ref_components(&self.module.unwrap().package.refr)?; + let mut path: Vec<&str> = path.iter().map(|s| *s.text()).collect(); + path.push(*name); let v = Self::get_value_chained(self.data.clone(), &path); @@ -1753,7 +1736,7 @@ impl<'source> Interpreter<'source> { // Add module prefix and ensure that any matching rule is evaluated. let module_path = Self::get_path_string(&self.current_module()?.package.refr, Some("data"))?; - let rule_path = module_path + "." + name; + let rule_path = module_path + "." + *name; self.ensure_rule_evaluated(rule_path)?; @@ -1764,12 +1747,12 @@ impl<'source> Interpreter<'source> { } } - fn eval_expr(&mut self, expr: &'source Expr<'source>) -> Result { + fn eval_expr(&mut self, expr: &'source Expr) -> Result { match expr { Expr::Null(_) => Ok(Value::Null), Expr::True(_) => Ok(Value::Bool(true)), Expr::False(_) => Ok(Value::Bool(false)), - Expr::Number(span) => match serde_json::from_str::(span.text()) { + Expr::Number(span) => match serde_json::from_str::(*span.text()) { Ok(v) => Ok(v), Err(e) => Err(span.source.error( span.line, @@ -1814,10 +1797,7 @@ impl<'source> Interpreter<'source> { } } - fn make_rule_context( - &self, - head: &'source RuleHead<'source>, - ) -> Result<(Context<'source>, Vec>)> { + fn make_rule_context(&self, head: &'source RuleHead) -> Result<(Context<'source>, Vec)> { //TODO: include "data" ? let mut path = Parser::get_path_ref_components(&self.module.unwrap().package.refr)?; @@ -1861,7 +1841,7 @@ impl<'source> Interpreter<'source> { } } - fn get_rule_module(&self, rule: &'source Rule<'source>) -> Result<&'source Module<'source>> { + fn get_rule_module(&self, rule: &'source Rule) -> Result<&'source Module> { for m in &self.modules { if m.policy.iter().any(|r| Ref::make(r) == Ref::make(rule)) { return Ok(m); @@ -1873,8 +1853,8 @@ impl<'source> Interpreter<'source> { fn eval_rule_bodies( &mut self, ctx: Context<'source>, - span: &'source Span<'source>, - bodies: &'source Vec>, + span: &'source Span, + bodies: &'source Vec, ) -> Result { let n_scopes = self.scopes.len(); let result = if bodies.is_empty() { @@ -1974,7 +1954,7 @@ impl<'source> Interpreter<'source> { } } - pub fn merge_value(span: &Span<'source>, value: &mut Value, new: Value) -> Result<()> { + pub fn merge_value(span: &Span, value: &mut Value, new: Value) -> Result<()> { match value.merge(new) { Ok(()) => Ok(()), Err(err) => return Err(span.error(format!("{err}").as_str())), @@ -1987,19 +1967,19 @@ impl<'source> Interpreter<'source> { while expr.is_some() { match expr { Some(Expr::RefDot { refr, field, .. }) => { - comps.push(field.text()); + comps.push(*field.text()); expr = Some(refr); } Some(Expr::RefBrack { refr, index, .. }) if matches!(index.as_ref(), Expr::String(_)) => { if let Expr::String(s) = index.as_ref() { - comps.push(s.text()); + comps.push(*s.text()); expr = Some(refr); } } Some(Expr::Var(v)) => { - comps.push(v.text()); + comps.push(*v.text()); expr = None; } _ => bail!("internal error: not a simple ref"), @@ -2014,8 +1994,8 @@ impl<'source> Interpreter<'source> { fn set_current_module( &mut self, - module: Option<&'source Module<'source>>, - ) -> Result>> { + module: Option<&'source Module>, + ) -> Result> { let m = self.module; if let Some(m) = module { self.current_module_path = Self::get_path_string(&m.package.refr, Some("data"))?; @@ -2024,7 +2004,7 @@ impl<'source> Interpreter<'source> { Ok(m) } - fn get_rule_refr(rule: &'source Rule<'source>) -> &'source Expr<'source> { + fn get_rule_refr(rule: &'source Rule) -> &'source Expr { match rule { Rule::Spec { head, .. } => match &head { RuleHead::Compr { refr, .. } @@ -2035,7 +2015,7 @@ impl<'source> Interpreter<'source> { } } - fn check_default_value(expr: &'source Expr<'source>) -> Result<()> { + fn check_default_value(expr: &'source Expr) -> Result<()> { use Expr::*; let (kind, span) = match expr { // Scalars are supported @@ -2091,7 +2071,7 @@ impl<'source> Interpreter<'source> { Ok(()) } - fn eval_default_rule(&mut self, rule: &'source Rule<'source>) -> Result<()> { + fn eval_default_rule(&mut self, rule: &'source Rule) -> Result<()> { // Skip reprocessing rule. if self.processed.contains(&Ref::make(rule)) { return Ok(()); @@ -2113,7 +2093,7 @@ impl<'source> Interpreter<'source> { }; Parser::get_path_ref_components_into(refr, &mut path)?; - let paths: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let paths: Vec<&str> = path.iter().map(|s| *s.text()).collect(); Self::check_default_value(value)?; let value = self.eval_expr(value)?; @@ -2149,11 +2129,7 @@ impl<'source> Interpreter<'source> { Ok(()) } - fn eval_rule( - &mut self, - module: &'source Module<'source>, - rule: &'source Rule<'source>, - ) -> Result<()> { + fn eval_rule(&mut self, module: &'source Module, rule: &'source Rule) -> Result<()> { // Skip reprocessing rule if self.processed.contains(&Ref::make(rule)) { return Ok(()); @@ -2207,14 +2183,14 @@ impl<'source> Interpreter<'source> { Value::Set(_) if special_set => { let entry = path[path.len() - 1].text(); let mut s = BTreeSet::new(); - s.insert(Value::String(entry.to_owned())); + s.insert(Value::String(entry.to_owned().to_string())); path = path[0..path.len() - 1].to_vec(); Value::from_set(s) } v => v, }; if value != Value::Undefined { - let paths: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let paths: Vec<&str> = path.iter().map(|s| *s.text()).collect(); let vref = Self::make_or_get_value_mut(&mut self.data, &paths[..])?; Self::merge_value(span, vref, value)?; } @@ -2225,7 +2201,7 @@ impl<'source> Interpreter<'source> { Parser::get_path_ref_components(&self.current_module()?.package.refr)?; Parser::get_path_ref_components_into(refr, &mut path)?; - let path: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let path: Vec<&str> = path.iter().map(|s| *s.text()).collect(); // Ensure that for functions with a nesting level (e.g: a.foo), // `a` is created as an empty object. @@ -2249,8 +2225,8 @@ impl<'source> Interpreter<'source> { pub fn eval_rule_with_input( &mut self, - module: &'source Module<'source>, - rule: &'source Rule<'source>, + module: &'source Module, + rule: &'source Rule, input: &Option, enable_tracing: bool, ) -> Result { @@ -2277,7 +2253,7 @@ impl<'source> Interpreter<'source> { // Ensure that each module has an empty object for m in &self.modules { let path = Parser::get_path_ref_components(&m.package.refr)?; - let path: Vec<&str> = path.iter().map(|s| s.text()).collect(); + let path: Vec<&str> = path.iter().map(|s| *s.text()).collect(); let vref = Self::make_or_get_value_mut(&mut self.data, &path[..])?; if *vref == Value::Undefined { *vref = Value::new_object(); @@ -2297,7 +2273,7 @@ impl<'source> Interpreter<'source> { pub fn eval_module( &mut self, - module: &'source Module<'source>, + module: &'source Module, input: &Option, enable_tracing: bool, ) -> Result { @@ -2352,7 +2328,7 @@ impl<'source> Interpreter<'source> { pub fn eval_user_query( &mut self, - query: &'source Query<'source>, + query: &'source Query, schedule: &Schedule<'source>, enable_tracing: bool, ) -> Result { diff --git a/src/lexer.rs b/src/lexer.rs index 93a81370..b639df23 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -9,16 +9,80 @@ use crate::value::Value; use anyhow::{anyhow, bail, Result}; #[derive(Clone)] -pub struct Source<'source> { - pub file: &'source str, - pub contents: &'source str, - pub lines: Vec<&'source str>, +struct SourceInternal { + pub file: String, + pub contents: String, + pub lines: Vec<(u16, u16)>, } -impl<'source> Source<'source> { +#[derive(Clone)] +pub struct Source { + src: std::rc::Rc, +} + +impl Source { + pub fn new(file: String, contents: String) -> Source { + let mut lines = vec![]; + let mut prev_ch = ' '; + let mut prev_pos = 0u16; + let mut start = 0u16; + for (i, ch) in contents.char_indices() { + if ch == '\n' { + let end = match prev_ch { + '\r' => prev_pos, + _ => i as u16, + }; + lines.push((start, end)); + start = i as u16 + 1; + } + prev_ch = ch; + prev_pos = i as u16; + } + + if (start as usize) < contents.len() { + lines.push((start, contents.len() as u16)); + } else if contents.is_empty() { + lines.push((0, 0)); + } else { + let s = (contents.len() - 1) as u16; + lines.push((s, s)); + } + Self { + src: std::rc::Rc::new(SourceInternal { + file, + contents, + lines, + }), + } + } + + pub fn from_file(path: String) -> Result { + let contents = match std::fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => bail!("Failed to read {path}. {e}"), + }; + Ok(Self::new(path, contents)) + } + + pub fn file(&self) -> &String { + &self.src.file + } + pub fn contents(&self) -> &String { + &self.src.contents + } + pub fn line(&self, idx: u16) -> &str { + let idx = idx as usize; + if idx < self.src.lines.len() { + let (start, end) = self.src.lines[idx]; + &self.src.contents[start as usize..end as usize] + } else { + "" + } + } + pub fn message(&self, line: u16, col: u16, kind: &str, msg: &str) -> String { - if line as usize > self.lines.len() { - return format!("{}: invalid line {} specified", self.file, line); + if line as usize > self.src.lines.len() { + return format!("{}: invalid line {} specified", self.src.file, line); } let line_str = format!("{line}"); @@ -30,12 +94,12 @@ impl<'source> Source<'source> { {: Source<'source> { } #[derive(Clone)] -pub struct Span<'source> { - pub source: &'source Source<'source>, +pub struct Span { + pub source: Source, pub line: u16, pub col: u16, pub start: u16, pub end: u16, } -impl<'source> Span<'source> { - pub fn text(&self) -> &'source str { - &self.source.contents[self.start as usize..self.end as usize] +impl Span { + pub fn text(&self) -> std::rc::Rc<&str> { + std::rc::Rc::new(&self.source.contents()[self.start as usize..self.end as usize]) } pub fn message(&self, kind: &str, msg: &str) -> String { @@ -71,7 +135,7 @@ impl<'source> Span<'source> { } } -impl<'source> Debug for Span<'source> { +impl Debug for Span { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { let t = self.text().escape_debug().to_string(); let max = 32; @@ -99,21 +163,21 @@ pub enum TokenKind { } #[derive(Debug, Clone)] -pub struct Token<'source>(pub TokenKind, pub Span<'source>); +pub struct Token(pub TokenKind, pub Span); #[derive(Clone)] pub struct Lexer<'source> { - source: &'source Source<'source>, + source: Source, iter: Peekable>, line: u16, col: u16, } impl<'source> Lexer<'source> { - pub fn new(source: &'source Source<'source>) -> Self { + pub fn new(source: &'source Source) -> Self { Self { - source, - iter: source.contents.char_indices().peekable(), + source: source.clone(), + iter: source.contents().char_indices().peekable(), line: 1, col: 1, } @@ -122,18 +186,18 @@ impl<'source> Lexer<'source> { fn peek(&mut self) -> (usize, char) { match self.iter.peek() { Some((index, chr)) => (*index, *chr), - _ => (self.source.contents.len(), '\x00'), + _ => (self.source.contents().len(), '\x00'), } } fn peekahead(&mut self, n: usize) -> (usize, char) { match self.iter.clone().nth(n) { Some((index, chr)) => (index, chr), - _ => (self.source.contents.len(), '\x00'), + _ => (self.source.contents().len(), '\x00'), } } - fn read_ident(&mut self) -> Result> { + fn read_ident(&mut self) -> Result { let start = self.peek().0; let col = self.col; loop { @@ -149,7 +213,7 @@ impl<'source> Lexer<'source> { Ok(Token( TokenKind::Ident, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -165,7 +229,7 @@ impl<'source> Lexer<'source> { } // See https://www.json.org/json-en.html for number's grammar - fn read_number(&mut self) -> Result> { + fn read_number(&mut self) -> Result { let (start, chr) = self.peek(); let col = self.col; self.iter.next(); @@ -206,7 +270,7 @@ impl<'source> Lexer<'source> { } // Ensure that the number is parsable in Rust. - match serde_json::from_str::<'source, Value>(&self.source.contents[start..end]) { + match serde_json::from_str::(&self.source.contents()[start..end]) { Ok(_) => (), Err(e) => { let serde_msg = &e.to_string(); @@ -233,7 +297,7 @@ impl<'source> Lexer<'source> { Ok(Token( TokenKind::Number, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -242,7 +306,7 @@ impl<'source> Lexer<'source> { )) } - fn read_raw_string(&mut self) -> Result> { + fn read_raw_string(&mut self) -> Result { self.iter.next(); self.col += 1; let (start, _) = self.peek(); @@ -270,7 +334,7 @@ impl<'source> Lexer<'source> { Ok(Token( TokenKind::RawString, Span { - source: self.source, + source: self.source.clone(), line, col, start: start as u16, @@ -279,7 +343,7 @@ impl<'source> Lexer<'source> { )) } - fn read_string(&mut self) -> Result> { + fn read_string(&mut self) -> Result { let (line, col) = (self.line, self.col); self.iter.next(); self.col += 1; @@ -335,7 +399,7 @@ impl<'source> Lexer<'source> { self.col += (end - start) as u16; // Ensure that the string is parsable in Rust. - match serde_json::from_str::<'source, String>(&self.source.contents[start - 1..end]) { + match serde_json::from_str::(&self.source.contents()[start - 1..end]) { Ok(_) => (), Err(e) => { let serde_msg = &e.to_string(); @@ -352,7 +416,7 @@ impl<'source> Lexer<'source> { Ok(Token( TokenKind::String, Span { - source: self.source, + source: self.source.clone(), line, col: col + 1, start: start as u16, @@ -399,7 +463,7 @@ impl<'source> Lexer<'source> { Ok(()) } - pub fn next_token(&mut self) -> Result> { + pub fn next_token(&mut self) -> Result { self.skip_ws()?; let (start, chr) = self.peek(); @@ -423,7 +487,7 @@ impl<'source> Lexer<'source> { self.col += 1; self.iter.next(); Ok(Token(TokenKind::Symbol, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -440,7 +504,7 @@ impl<'source> Lexer<'source> { end += 1; } Ok(Token(TokenKind::Symbol, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -456,7 +520,7 @@ impl<'source> Lexer<'source> { self.iter.next(); }; Ok(Token(TokenKind::Symbol, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -468,7 +532,7 @@ impl<'source> Lexer<'source> { self.iter.next(); self.iter.next(); Ok(Token(TokenKind::Symbol, Span { - source: self.source, + source: self.source.clone(), line: self.line, col, start: start as u16, @@ -478,7 +542,7 @@ impl<'source> Lexer<'source> { '"' => self.read_string(), '`' => self.read_raw_string(), '\x00' => Ok(Token(TokenKind::Eof, Span { - source: self.source, + source: self.source.clone(), line:self.line, col, start: start as u16, @@ -487,7 +551,7 @@ impl<'source> Lexer<'source> { _ if chr.is_ascii_digit() => self.read_number(), _ if chr.is_ascii_alphabetic() || chr == '_' => { let mut ident = self.read_ident()?; - if ident.1.text() == "set" && self.peek().1 == '(' { + if *ident.1.text() == "set" && self.peek().1 == '(' { // set immediately followed by ( is treated as set( if // the next token is ). let state = (self.iter.clone(), self.line, self.col); @@ -495,7 +559,7 @@ impl<'source> Lexer<'source> { // Check it next token is ). let next_tok = self.next_token()?; - let is_setp = next_tok.1.text() == ")"; + let is_setp = *next_tok.1.text() == ")"; // Restore state (self.iter, self.line, self.col) = state; diff --git a/src/lib.rs b/src/lib.rs index f3787594..34d3bf38 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ pub mod ast; pub mod builtins; +pub mod engine; pub mod interpreter; pub mod lexer; pub mod parser; @@ -11,6 +12,7 @@ mod utils; pub mod value; pub use ast::*; +pub use engine::*; pub use interpreter::*; pub use lexer::*; pub use parser::*; diff --git a/src/parser.rs b/src/parser.rs index 0b57c56a..3cbd2ac0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -9,22 +9,22 @@ use anyhow::{anyhow, bail, Result}; #[derive(Clone)] pub struct Parser<'source> { - source: &'source Source<'source>, + source: Source, lexer: Lexer<'source>, - tok: Token<'source>, + tok: Token, line: u16, end: u16, - future_keywords: BTreeMap<&'source str, Span<'source>>, + future_keywords: BTreeMap, } const FUTURE_KEYWORDS: [&str; 4] = ["contains", "every", "if", "in"]; impl<'source> Parser<'source> { - pub fn new(source: &'source Source<'source>) -> Result { + pub fn new(source: &'source Source) -> Result { let mut lexer = Lexer::new(source); let tok = lexer.next_token()?; Ok(Self { - source, + source: source.clone(), lexer, tok, line: 0, @@ -41,7 +41,7 @@ impl<'source> Parser<'source> { } fn expect(&mut self, text: &str, context: &str) -> Result<()> { - if self.tok.1.text() == text { + if *self.tok.1.text() == text { self.next_token() } else { let msg = format!("expecting `{text}` {context}"); @@ -65,7 +65,7 @@ impl<'source> Parser<'source> { ); } - pub fn set_future_keyword(&mut self, kw: &'source str, span: &Span<'source>) -> Result<()> { + pub fn set_future_keyword(&mut self, kw: &str, span: &Span) -> Result<()> { match &self.future_keywords.get(kw) { Some(s) => Err(self.source.error( span.line, @@ -78,16 +78,13 @@ impl<'source> Parser<'source> { .as_str(), )), None => { - self.future_keywords.insert(kw, span.clone()); + self.future_keywords.insert(kw.to_string(), span.clone()); Ok(()) } } } - pub fn get_path_ref_components_into( - refr: &Expr<'source>, - comps: &mut Vec>, - ) -> Result<()> { + pub fn get_path_ref_components_into(refr: &Expr, comps: &mut Vec) -> Result<()> { match refr { Expr::RefDot { refr, field, .. } => { Self::get_path_ref_components_into(refr, comps)?; @@ -104,16 +101,16 @@ impl<'source> Parser<'source> { Ok(()) } - pub fn get_path_ref_components(refr: &Expr<'source>) -> Result>> { + pub fn get_path_ref_components(refr: &Expr) -> Result> { let mut comps = vec![]; Self::get_path_ref_components_into(refr, &mut comps)?; Ok(comps) } - fn handle_import_future_keywords(&mut self, comps: &Vec>) -> Result { - if comps.len() >= 2 && comps[0].text() == "future" && comps[1].text() == "keywords" { + fn handle_import_future_keywords(&mut self, comps: &Vec) -> Result { + if comps.len() >= 2 && *comps[0].text() == "future" && *comps[1].text() == "keywords" { match comps.len() - 2 { - 1 => self.set_future_keyword(comps[2].text(), &comps[2])?, + 1 => self.set_future_keyword(&comps[2].text(), &comps[2])?, 0 => { let span = &comps[1]; for kw in FUTURE_KEYWORDS.iter() { @@ -128,7 +125,7 @@ impl<'source> Parser<'source> { } } Ok(true) - } else if !comps.is_empty() && comps[0].text() == "future" { + } else if !comps.is_empty() && *comps[0].text() == "future" { let s = &comps[0]; Err(self .source @@ -144,7 +141,7 @@ impl<'source> Parser<'source> { is_optional: bool, context: &str, ) -> Result<()> { - if self.tok.1.text() == kw { + if *self.tok.1.text() == kw { match &self.future_keywords.get(kw) { Some(_) => self.next_token(), None => { @@ -161,7 +158,7 @@ impl<'source> Parser<'source> { } } - fn is_keyword(&self, ident: &'source str) -> bool { + fn is_keyword(&self, ident: &str) -> bool { matches!( ident, "as" | "default" @@ -177,10 +174,10 @@ impl<'source> Parser<'source> { ) } - fn parse_ident(&mut self) -> Result> { + fn parse_ident(&mut self) -> Result { let span = self.tok.1.clone(); match self.tok.0 { - TokenKind::Ident if self.is_keyword(span.text()) => Err(self.source.error( + TokenKind::Ident if self.is_keyword(*span.text()) => Err(self.source.error( self.tok.1.line, self.tok.1.col, &format!("unexpected keyword `{}`", span.text()), @@ -195,11 +192,12 @@ impl<'source> Parser<'source> { } } - fn parse_var(&mut self) -> Result> { + fn parse_var(&mut self) -> Result { let span = self.tok.1.clone(); match self.tok.0 { TokenKind::Ident - if self.is_keyword(span.text()) || self.is_imported_future_keyword(span.text()) => + if self.is_keyword(*span.text()) + || self.is_imported_future_keyword(*span.text()) => { Err(self.source.error( self.tok.1.line, @@ -217,13 +215,13 @@ impl<'source> Parser<'source> { } } - fn parse_scalar_or_var(&mut self) -> Result> { + fn parse_scalar_or_var(&mut self) -> Result { let span = self.tok.1.clone(); let node = match &self.tok.0 { TokenKind::Number => Expr::Number(span), TokenKind::String => Expr::String(span), TokenKind::RawString => Expr::RawString(span), - TokenKind::Ident => match self.tok.1.text() { + TokenKind::Ident => match *self.tok.1.text() { "null" => Expr::Null(span), "true" => Expr::True(span), "false" => Expr::False(span), @@ -241,14 +239,14 @@ impl<'source> Parser<'source> { Ok(node) } - fn parse_compr(&mut self, delim: &str) -> Result<(Expr<'source>, Query<'source>)> { + fn parse_compr(&mut self, delim: &str) -> Result<(Expr, Query)> { // Save the state. let state = self.clone(); let mut span = self.tok.1.clone(); // Parse the first expression as a ref. let term = match self.parse_ref() { - Ok(e) if self.tok.1.text() == "|" => e, + Ok(e) if *self.tok.1.text() == "|" => e, _ => { // Not a comprehension. Restore state. *self = state; @@ -274,7 +272,7 @@ impl<'source> Parser<'source> { } } - fn parse_compr_or_array(&mut self) -> Result> { + fn parse_compr_or_array(&mut self) -> Result { // Save the state. let mut span = self.tok.1.clone(); self.expect("[", "while parsing array comprehension or array")?; @@ -293,11 +291,11 @@ impl<'source> Parser<'source> { // No progress was made in parsing comprehension. // Parse as array. let mut items = vec![]; - if self.tok.1.text() != "]" { + if *self.tok.1.text() != "]" { items.push(self.parse_in_expr()?); - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; - match self.tok.1.text() { + match *self.tok.1.text() { "]" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => items.push(self.parse_in_expr()?), @@ -312,7 +310,7 @@ impl<'source> Parser<'source> { } } - fn parse_compr_set_or_object(&mut self) -> Result> { + fn parse_compr_set_or_object(&mut self) -> Result { let mut span = self.tok.1.clone(); self.expect("{", "while parsing set, object or comprehension")?; @@ -336,7 +334,7 @@ impl<'source> Parser<'source> { // It could be a set, object or object comprehension. // In all the cases, the first expressoin must parse successfully. - if self.tok.1.text() == "}" { + if *self.tok.1.text() == "}" { self.next_token()?; span.end = self.end; return Ok(Expr::Object { @@ -348,12 +346,12 @@ impl<'source> Parser<'source> { let mut item_span = self.tok.1.clone(); let first = self.parse_in_expr()?; - if self.tok.1.text() != ":" { + if *self.tok.1.text() != ":" { // Parse as set. let mut items = vec![first]; - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; - match self.tok.1.text() { + match *self.tok.1.text() { "}" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => items.push(self.parse_in_expr()?), @@ -393,10 +391,10 @@ impl<'source> Parser<'source> { item_span.end = self.end; items.push((item_span, first, value)); - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; let item_start = self.tok.1.start; - let key = match self.tok.1.text() { + let key = match *self.tok.1.text() { "}" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => self.parse_in_expr()?, @@ -420,7 +418,7 @@ impl<'source> Parser<'source> { }) } - fn parse_empty_set(&mut self) -> Result> { + fn parse_empty_set(&mut self) -> Result { let mut span = self.tok.1.clone(); self.expect("set(", "while parsing empty set")?; self.expect(")", "while parsing empty set")?; @@ -431,7 +429,7 @@ impl<'source> Parser<'source> { }) } - fn parse_parens_expr(&mut self) -> Result> { + fn parse_parens_expr(&mut self) -> Result { self.next_token()?; let expr = self.parse_membership_expr()?; self.expect(")", "while parsing parenthesized expression")?; @@ -439,7 +437,7 @@ impl<'source> Parser<'source> { Ok(expr) } - fn parse_unary_expr(&mut self) -> Result> { + fn parse_unary_expr(&mut self) -> Result { let mut span = self.tok.1.clone(); self.next_token()?; let expr = self.parse_in_expr()?; @@ -450,9 +448,9 @@ impl<'source> Parser<'source> { }) } - fn parse_ref(&mut self) -> Result> { + fn parse_ref(&mut self) -> Result { let start = self.tok.1.start; - let mut term = match self.tok.1.text() { + let mut term = match *self.tok.1.text() { "[" => self.parse_compr_or_array()?, "{" => self.parse_compr_set_or_object()?, "set(" => self.parse_empty_set()?, @@ -467,7 +465,7 @@ impl<'source> Parser<'source> { let mut span = self.tok.1.clone(); let sep_pos = span.start; span.start = start; - match self.tok.1.text() { + match *self.tok.1.text() { "." | "[" if self.tok.1.start != self.end => { if self.line != self.tok.1.line { // Newline encountered. This could be a separate @@ -524,16 +522,16 @@ impl<'source> Parser<'source> { } "(" if possible_fcn => { self.next_token()?; - if self.tok.1.text() == ")" { + if *self.tok.1.text() == ")" { return Err(self .tok .1 .error("at least one argument required for function calls")); } let mut args = vec![self.parse_in_expr()?]; - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; - match self.tok.1.text() { + match *self.tok.1.text() { ")" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => args.push(self.parse_in_expr()?), @@ -557,18 +555,18 @@ impl<'source> Parser<'source> { Ok(term) } - fn parse_term(&mut self) -> Result> { + fn parse_term(&mut self) -> Result { self.parse_ref() } - fn parse_mul_div_mod_expr(&mut self) -> Result> { + fn parse_mul_div_mod_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_term()?; loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "*" => ArithOp::Mul, "/" => ArithOp::Div, "%" => ArithOp::Mod, @@ -586,14 +584,14 @@ impl<'source> Parser<'source> { } } - fn parse_arith_expr(&mut self) -> Result> { + fn parse_arith_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_mul_div_mod_expr()?; loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "+" => ArithOp::Add, "-" => ArithOp::Sub, _ => return Ok(expr), @@ -610,11 +608,11 @@ impl<'source> Parser<'source> { } } - fn parse_and_expr(&mut self) -> Result> { + fn parse_and_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_arith_expr()?; - while self.tok.1.text() == "&" { + while *self.tok.1.text() == "&" { let mut span = self.tok.1.clone(); span.start = start; self.next_token()?; @@ -630,11 +628,11 @@ impl<'source> Parser<'source> { Ok(expr) } - fn parse_or_expr(&mut self) -> Result> { + fn parse_or_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_and_expr()?; - while self.tok.1.text() == "|" { + while *self.tok.1.text() == "|" { let mut span = self.tok.1.clone(); span.start = start; self.next_token()?; @@ -650,13 +648,13 @@ impl<'source> Parser<'source> { Ok(expr) } - fn parse_bool_expr(&mut self) -> Result> { + fn parse_bool_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_or_expr()?; loop { let mut span = self.tok.1.clone(); span.start = start; - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "<" => BoolOp::Lt, "<=" => BoolOp::Le, "==" => BoolOp::Eq, @@ -681,9 +679,9 @@ impl<'source> Parser<'source> { fn parse_membership_tail( &mut self, start: u16, - mut expr1: Expr<'source>, - mut expr2: Option>, - ) -> Result> { + mut expr1: Expr, + mut expr2: Option, + ) -> Result { loop { let mut span = self.tok.1.clone(); span.start = start; @@ -702,7 +700,7 @@ impl<'source> Parser<'source> { }; expr2 = None; - if self.tok.1.text() != "in" { + if *self.tok.1.text() != "in" { break; } } @@ -710,42 +708,42 @@ impl<'source> Parser<'source> { Ok(expr1) } - fn parse_in_expr(&mut self) -> Result> { + fn parse_in_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_bool_expr()?; - while self.tok.1.text() == "in" && self.future_keywords.get("in").is_some() { + while *self.tok.1.text() == "in" && self.future_keywords.get("in").is_some() { expr = self.parse_membership_tail(start, expr, None)?; } Ok(expr) } - pub fn parse_membership_expr(&mut self) -> Result> { + pub fn parse_membership_expr(&mut self) -> Result { let start = self.tok.1.start; let mut expr = self.parse_bool_expr()?; - if self.tok.1.text() == "," { + if *self.tok.1.text() == "," { self.next_token()?; let value = self.parse_bool_expr()?; expr = self.parse_membership_tail(start, expr, Some(value))?; } - while self.tok.1.text() == "in" && self.is_imported_future_keyword("in") { + while *self.tok.1.text() == "in" && self.is_imported_future_keyword("in") { expr = self.parse_membership_tail(start, expr, None)?; } Ok(expr) } - pub fn parse_assign_expr(&mut self) -> Result> { + pub fn parse_assign_expr(&mut self) -> Result { let state = self.clone(); let start = self.tok.1.start; let expr = self.parse_ref()?; let mut span = self.tok.1.clone(); span.start = start; - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "=" => AssignOp::Eq, ":=" => AssignOp::ColEq, _ => { @@ -765,9 +763,9 @@ impl<'source> Parser<'source> { }) } - fn parse_with_modifiers(&mut self) -> Result>> { + fn parse_with_modifiers(&mut self) -> Result> { let mut modifiers = vec![]; - while self.tok.1.text() == "with" { + while *self.tok.1.text() == "with" { let mut span = self.tok.1.clone(); self.next_token()?; let refr = self.parse_path_ref()?; @@ -779,13 +777,13 @@ impl<'source> Parser<'source> { Ok(modifiers) } - fn parse_every_stmt(&mut self) -> Result> { + fn parse_every_stmt(&mut self) -> Result { let mut span = self.tok.1.clone(); let context = "Failed to parse `every` statement."; self.parse_future_keyword("every", false, context)?; let ident = self.parse_var()?; - let (key, value) = match self.tok.1.text() { + let (key, value) = match *self.tok.1.text() { "," => { self.next_token()?; match self.parse_var() { @@ -818,7 +816,7 @@ impl<'source> Parser<'source> { }) } - fn parse_some_stmt(&mut self) -> Result> { + fn parse_some_stmt(&mut self) -> Result { let mut span = self.tok.1.clone(); self.expect("some", "while parsing some-decl")?; @@ -826,7 +824,7 @@ impl<'source> Parser<'source> { let mut vars = vec![self.tok.1.clone()]; let mut refs = vec![self.parse_ref()?]; - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; let mut span = self.tok.1.clone(); refs.push(self.parse_ref()?); @@ -834,8 +832,8 @@ impl<'source> Parser<'source> { vars.push(span); } - if self.tok.1.text() != "in" || !self.is_imported_future_keyword("in") { - if self.tok.1.text() == "in" { + if *self.tok.1.text() != "in" || !self.is_imported_future_keyword("in") { + if *self.tok.1.text() == "in" { self.warn_future_keyword(); } // All the refs must be identifiers @@ -846,7 +844,7 @@ impl<'source> Parser<'source> { _ => { return Err(anyhow!( "{}:{}:{} error: encountered `{}` while expecting identifier", - span.source.file, + span.source.file(), span.line, span.col, span.text() @@ -866,7 +864,7 @@ impl<'source> Parser<'source> { let span = &vars[2]; return Err(anyhow!( "{}:{}:{} error: encountered `{}` while expecting `in`", - span.source.file, + span.source.file(), span.line, span.col, span.text() @@ -884,8 +882,8 @@ impl<'source> Parser<'source> { }) } - fn parse_literal(&mut self) -> Result> { - match self.tok.1.text() { + fn parse_literal(&mut self) -> Result { + match *self.tok.1.text() { "some" => return self.parse_some_stmt(), "every" => { if self.future_keywords.get("every").is_some() { @@ -896,7 +894,7 @@ impl<'source> Parser<'source> { _ => (), } let mut span = self.tok.1.clone(); - let not_expr = if self.tok.1.text() == "not" { + let not_expr = if *self.tok.1.text() == "not" { self.next_token()?; true } else { @@ -912,7 +910,7 @@ impl<'source> Parser<'source> { } } - pub fn parse_literal_stmt(&mut self) -> Result> { + pub fn parse_literal_stmt(&mut self) -> Result { let mut span = self.tok.1.clone(); let literal = self.parse_literal()?; let with_mods = self.parse_with_modifiers()?; @@ -925,13 +923,9 @@ impl<'source> Parser<'source> { }) } - pub fn parse_query( - &mut self, - mut span: Span<'source>, - end_delim: &str, - ) -> Result> { + pub fn parse_query(&mut self, mut span: Span, end_delim: &str) -> Result { let state = self.clone(); - let _is_definite_query = matches!(self.tok.1.text(), "some" | "every"); + let _is_definite_query = matches!(*self.tok.1.text(), "some" | "every"); // TODO: empty query? let mut literals = vec![]; @@ -947,7 +941,7 @@ impl<'source> Parser<'source> { } }; - if self.tok.1.text() == "," { + if *self.tok.1.text() == "," { // This is likely an array or set. // Restore the state. *self = state; @@ -957,7 +951,7 @@ impl<'source> Parser<'source> { literals.push(stmt); loop { - match self.tok.1.text() { + match *self.tok.1.text() { t if t == end_delim => break, "" if self.tok.0 == TokenKind::Eof => break, ";" => self.next_token()?, @@ -982,10 +976,10 @@ impl<'source> Parser<'source> { }) } - pub fn parse_rule_assign(&mut self) -> Result>> { + pub fn parse_rule_assign(&mut self) -> Result> { let mut span = self.tok.1.clone(); - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "=" => { self.next_token()?; AssignOp::Eq @@ -1006,7 +1000,7 @@ impl<'source> Parser<'source> { })) } - fn parse_path_ref(&mut self) -> Result> { + fn parse_path_ref(&mut self) -> Result { let start = self.tok.1.start; let var = self.parse_var()?; @@ -1015,14 +1009,14 @@ impl<'source> Parser<'source> { let mut span = self.tok.1.clone(); let sep_pos = span.start; span.start = start; - match self.tok.1.text() { + match *self.tok.1.text() { "." | "[" if self.tok.1.start != self.end => { bail!( "{}", self.source.error( self.tok.1.line, self.tok.1.col - 1, - format!("invalid whitespace before {}", self.tok.1.text()).as_str() + format!("invalid whitespace before {}", *self.tok.1.text()).as_str() ) ); } @@ -1098,7 +1092,7 @@ impl<'source> Parser<'source> { } } - fn parse_rule_ref(&mut self) -> Result> { + fn parse_rule_ref(&mut self) -> Result { let start = self.tok.1.start; let span = self.tok.1.clone(); @@ -1115,7 +1109,7 @@ impl<'source> Parser<'source> { loop { let mut span = self.tok.1.clone(); span.start = start; - match self.tok.1.text() { + match *self.tok.1.text() { // . and [ must not have any space between the previous token. "." | "[" if self.tok.1.start != self.end => { bail!( @@ -1123,7 +1117,7 @@ impl<'source> Parser<'source> { self.source.error( self.tok.1.line, self.tok.1.col - 1, - format!("invalid whitespace before {}", self.tok.1.text()).as_str() + format!("invalid whitespace before {}", *self.tok.1.text()).as_str() ) ); } @@ -1168,18 +1162,18 @@ impl<'source> Parser<'source> { Ok(term) } - pub fn parse_rule_head(&mut self) -> Result> { + pub fn parse_rule_head(&mut self) -> Result { let mut span = self.tok.1.clone(); let rule_ref = self.parse_rule_ref()?; - match self.tok.1.text() { + match *self.tok.1.text() { "(" => { self.check_rule_ref(&rule_ref)?; self.next_token()?; let mut args = vec![self.parse_term()?]; - while self.tok.1.text() == "," { + while *self.tok.1.text() == "," { self.next_token()?; - match self.tok.1.text() { + match *self.tok.1.text() { ")" => break, "" if self.tok.0 == TokenKind::Eof => break, _ => args.push(self.parse_term()?), @@ -1219,8 +1213,8 @@ impl<'source> Parser<'source> { } // Determine whether to create a set or a compr - let is_set_follower = !self.is_keyword(self.tok.1.text()) - && !self.is_imported_future_keyword(self.tok.1.text()); + let is_set_follower = !self.is_keyword(*self.tok.1.text()) + && !self.is_imported_future_keyword(*self.tok.1.text()); if assign.is_none() && is_set_follower { match &rule_ref { Expr::RefBrack { refr, index, .. } @@ -1257,11 +1251,11 @@ impl<'source> Parser<'source> { self.future_keywords.get("if").is_some() } - pub fn parse_query_or_literal_stmt(&mut self) -> Result> { + pub fn parse_query_or_literal_stmt(&mut self) -> Result { let state = self.clone(); let mut span = self.tok.1.clone(); - if self.tok.1.text() == "{" { + if *self.tok.1.text() == "{" { self.next_token()?; let pos = self.end; match self.parse_query(span.clone(), "}") { @@ -1281,12 +1275,12 @@ impl<'source> Parser<'source> { Ok(Query { span, stmts }) } - pub fn parse_rule_bodies(&mut self) -> Result>> { + pub fn parse_rule_bodies(&mut self) -> Result> { let mut span = self.tok.1.clone(); let mut bodies = vec![]; let assign = None; - let has_query = match self.tok.1.text() { + let has_query = match *self.tok.1.text() { "if" if self.if_is_keyword() => { self.next_token()?; let query = self.parse_query_or_literal_stmt()?; @@ -1316,7 +1310,7 @@ impl<'source> Parser<'source> { _ => false, }; - match self.tok.1.text() { + match *self.tok.1.text() { "{" if has_query => self.parse_query_blocks(&mut bodies)?, "else" if has_query => self.parse_else_blocks(&mut bodies)?, _ => (), @@ -1325,8 +1319,8 @@ impl<'source> Parser<'source> { Ok(bodies) } - pub fn parse_query_blocks(&mut self, bodies: &mut Vec>) -> Result<()> { - while self.tok.1.text() == "{" { + pub fn parse_query_blocks(&mut self, bodies: &mut Vec) -> Result<()> { + while *self.tok.1.text() == "{" { let mut span = self.tok.1.clone(); self.next_token()?; let query = self.parse_query(span.clone(), "}")?; @@ -1340,11 +1334,11 @@ impl<'source> Parser<'source> { Ok(()) } - pub fn parse_else_blocks(&mut self, bodies: &mut Vec>) -> Result<()> { + pub fn parse_else_blocks(&mut self, bodies: &mut Vec) -> Result<()> { loop { let mut span = self.tok.1.clone(); - match self.tok.1.text() { + match *self.tok.1.text() { "{" => { return Err(self.source.error( self.tok.1.line, @@ -1358,7 +1352,7 @@ impl<'source> Parser<'source> { let assign = self.parse_rule_assign()?; - match self.tok.1.text() { + match *self.tok.1.text() { "if" if self.if_is_keyword() => { self.next_token()?; let query = self.parse_query_or_literal_stmt()?; @@ -1380,7 +1374,7 @@ impl<'source> Parser<'source> { }); } _ if assign.is_none() => { - if self.tok.1.text() == "if" { + if *self.tok.1.text() == "if" { self.warn_future_keyword(); } return Err(self.source.error( @@ -1395,12 +1389,12 @@ impl<'source> Parser<'source> { Ok(()) } - pub fn parse_default_rule(&mut self) -> Result> { + pub fn parse_default_rule(&mut self) -> Result { let mut span = self.tok.1.clone(); self.expect("default", "while parsing default rule")?; let rule_ref = self.parse_rule_ref()?; - let op = match self.tok.1.text() { + let op = match *self.tok.1.text() { "=" => AssignOp::Eq, ":=" => AssignOp::ColEq, _ => { @@ -1423,7 +1417,7 @@ impl<'source> Parser<'source> { }) } - pub fn parse_rule(&mut self) -> Result> { + pub fn parse_rule(&mut self) -> Result { let pos = self.end; match self.parse_default_rule() { Ok(r) => return Ok(r), @@ -1438,7 +1432,7 @@ impl<'source> Parser<'source> { Ok(Rule::Spec { span, head, bodies }) } - fn parse_package(&mut self) -> Result> { + fn parse_package(&mut self) -> Result { let mut span = self.tok.1.clone(); self.expect("package", "Missing package declaration.")?; let name = self.parse_path_ref()?; @@ -1446,21 +1440,13 @@ impl<'source> Parser<'source> { Ok(Package { span, refr: name }) } - fn check_and_add_import( - &self, - import: Import<'source>, - imports: &mut Vec>, - ) -> Result<()> { - let comps: Vec<&str> = Self::get_path_ref_components(&import.refr)? - .iter() - .map(|s| s.text()) - .collect(); + fn check_and_add_import(&self, import: Import, imports: &mut Vec) -> Result<()> { + let ref_comps = Self::get_path_ref_components(&import.refr)?; + let comps: Vec> = ref_comps.iter().map(|s| s.text()).collect(); for imp in imports.iter() { - let imp_comps: Vec<&str> = Self::get_path_ref_components(&imp.refr)? - .iter() - .map(|s| s.text()) - .collect(); + let imp_comps = Self::get_path_ref_components(&imp.refr)?; + let imp_comps: Vec> = imp_comps.iter().map(|s| s.text()).collect(); let shadow = match (&imp.r#as, &import.r#as) { (Some(i1), Some(i2)) if i1.text() == i2.text() => true, @@ -1490,15 +1476,15 @@ impl<'source> Parser<'source> { Ok(()) } - fn parse_imports(&mut self) -> Result>> { + fn parse_imports(&mut self) -> Result> { let mut imports = vec![]; - while self.tok.1.text() == "import" { + while *self.tok.1.text() == "import" { let mut span = self.tok.1.clone(); self.next_token()?; let refr = self.parse_path_ref()?; let comps = Self::get_path_ref_components(&refr)?; - if !matches!(comps[0].text(), "data" | "future" | "input") { + if !matches!(*comps[0].text(), "data" | "future" | "input") { return Err(self.source.error( comps[0].line, comps[0].col, @@ -1508,7 +1494,7 @@ impl<'source> Parser<'source> { let is_future_kw = self.handle_import_future_keywords(&comps)?; - let var = if self.tok.1.text() == "as" { + let var = if *self.tok.1.text() == "as" { if is_future_kw { return Err(self.source.error( self.tok.1.line, @@ -1519,7 +1505,7 @@ impl<'source> Parser<'source> { self.next_token()?; let var = self.parse_var()?; - if var.text() == "_" { + if *var.text() == "_" { return Err(self.source.error( var.line, var.col, @@ -1547,7 +1533,7 @@ impl<'source> Parser<'source> { Ok(imports) } - pub fn parse(&mut self) -> Result> { + pub fn parse(&mut self) -> Result { let package = self.parse_package()?; let imports = self.parse_imports()?; diff --git a/src/scheduler.rs b/src/scheduler.rs index a7e6a91c..c1d548b2 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -200,7 +200,7 @@ pub struct Scope<'a> { pub inputs: BTreeSet<&'a str>, } -fn traverse<'a>(expr: &'a Expr<'a>, f: &mut dyn FnMut(&'a Expr<'a>) -> Result) -> Result<()> { +fn traverse<'a>(expr: &'a Expr, f: &mut dyn FnMut(&'a Expr) -> Result) -> Result<()> { if !f(expr)? { return Ok(()); } @@ -267,30 +267,30 @@ fn var_exists<'a>(name: &'a str, parent_scopes: &[Scope<'a>]) -> bool { } fn gather_assigned_vars<'a>( - expr: &'a Expr<'a>, + expr: &'a Expr, can_shadow: bool, parent_scopes: &[Scope<'a>], scope: &mut Scope<'a>, ) -> Result<()> { traverse(expr, &mut |e| match e { // Ignore _, input, data. - Var(v) if matches!(v.text(), "_" | "input" | "data") => Ok(false), + Var(v) if matches!(*v.text(), "_" | "input" | "data") => Ok(false), // Record local var that can shadow input var. Var(v) if can_shadow => { - scope.locals.insert(v.text()); + scope.locals.insert(*v.text()); Ok(false) } // Record input vars. - Var(v) if var_exists(v.text(), parent_scopes) => { - scope.inputs.insert(v.text()); + Var(v) if var_exists(*v.text(), parent_scopes) => { + scope.inputs.insert(*v.text()); Ok(false) } // Record local var. Var(v) => { - scope.locals.insert(v.text()); + scope.locals.insert(*v.text()); Ok(false) } @@ -301,15 +301,15 @@ fn gather_assigned_vars<'a>( } fn gather_input_vars<'a>( - expr: &'a Expr<'a>, + expr: &'a Expr, parent_scopes: &[Scope<'a>], scope: &mut Scope<'a>, ) -> Result<()> { traverse(expr, &mut |e| match e { - Var(v) if var_exists(v.text(), parent_scopes) => { + Var(v) if var_exists(*v.text(), parent_scopes) => { let var = v.text(); - if !scope.locals.contains(var) { - scope.inputs.insert(var); + if !scope.locals.contains(&*var) { + scope.inputs.insert(*var); } Ok(false) } @@ -318,19 +318,19 @@ fn gather_input_vars<'a>( } fn gather_loop_vars<'a>( - expr: &'a Expr<'a>, + expr: &'a Expr, parent_scopes: &[Scope<'a>], scope: &mut Scope<'a>, ) -> Result<()> { traverse(expr, &mut |e| match e { - Var(v) if var_exists(v.text(), parent_scopes) => Ok(false), + Var(v) if var_exists(*v.text(), parent_scopes) => Ok(false), RefBrack { index, .. } => { if let Var(v) = index.as_ref() { - if !matches!(v.text(), "_" | "input" | "data") - && !var_exists(v.text(), parent_scopes) + if !matches!(*v.text(), "_" | "input" | "data") + && !var_exists(*v.text(), parent_scopes) { // Treat this as an index var. - scope.locals.insert(v.text()); + scope.locals.insert(*v.text()); } } Ok(true) @@ -346,7 +346,7 @@ fn gather_loop_vars<'a>( // {k:y} = t // Try inlining value of t fn gather_vars<'a>( - expr: &'a Expr<'a>, + expr: &'a Expr, can_shadow: bool, parent_scopes: &[Scope<'a>], scope: &mut Scope<'a>, @@ -366,9 +366,9 @@ fn gather_vars<'a>( gather_loop_vars(expr, parent_scopes, scope) } -fn get_rule_prefix<'a>(expr: &Expr<'a>) -> Result<&'a str> { +fn get_rule_prefix(expr: &Expr) -> Result<&str> { match expr { - Expr::Var(v) => Ok(v.text()), + Expr::Var(v) => Ok(*v.text()), Expr::RefDot { refr, .. } => get_rule_prefix(refr), Expr::RefBrack { refr, .. } => get_rule_prefix(refr), _ => bail!("internal error: analyzer: could not get rule prefix"), @@ -377,16 +377,16 @@ fn get_rule_prefix<'a>(expr: &Expr<'a>) -> Result<&'a str> { pub struct Analyzer<'a> { packages: BTreeMap>, - locals: BTreeMap>, Scope<'a>>, + locals: BTreeMap, Scope<'a>>, scopes: Vec>, - order: BTreeMap>, Vec>, + order: BTreeMap, Vec>, functions: FunctionTable<'a>, } #[derive(Clone)] pub struct Schedule<'a> { - pub scopes: BTreeMap>, Scope<'a>>, - pub order: BTreeMap>, Vec>, + pub scopes: BTreeMap, Scope<'a>>, + pub order: BTreeMap, Vec>, } impl<'a> Default for Analyzer<'a> { @@ -406,7 +406,7 @@ impl<'a> Analyzer<'a> { } } - pub fn analyze(mut self, modules: &'a [&'a Module<'a>]) -> Result { + pub fn analyze(mut self, modules: &'a [&'a Module]) -> Result { self.add_rules(modules)?; self.functions = gather_functions(modules)?; @@ -422,8 +422,8 @@ impl<'a> Analyzer<'a> { pub fn analyze_query_snippet( mut self, - modules: &'a [&'a Module<'a>], - query: &'a Query<'a>, + modules: &'a [&'a Module], + query: &'a Query, ) -> Result> { self.add_rules(modules)?; self.analyze_query(None, None, query, Scope::default())?; @@ -434,7 +434,7 @@ impl<'a> Analyzer<'a> { }) } - fn add_rules(&mut self, modules: &'a [&'a Module<'a>]) -> Result<()> { + fn add_rules(&mut self, modules: &'a [&'a Module]) -> Result<()> { for m in modules { let path = get_path_string(&m.package.refr, Some("data"))?; let scope: &mut Scope = self.packages.entry(path).or_default(); @@ -456,7 +456,7 @@ impl<'a> Analyzer<'a> { Ok(()) } - fn analyze_module(&mut self, m: &'a Module<'a>) -> Result<()> { + fn analyze_module(&mut self, m: &'a Module) -> Result<()> { let path = get_path_string(&m.package.refr, Some("data"))?; let scope = match self.packages.get(&path) { Some(s) => s, @@ -472,7 +472,7 @@ impl<'a> Analyzer<'a> { Ok(()) } - fn analyze_rule(&mut self, r: &'a Rule<'a>) -> Result<()> { + fn analyze_rule(&mut self, r: &'a Rule) -> Result<()> { match r { Rule::Spec { head, bodies, .. } => { let (key, value, scope) = self.analyze_rule_head(head)?; @@ -497,7 +497,7 @@ impl<'a> Analyzer<'a> { } } - fn analyze_value_expr(&mut self, expr: &'a Expr<'a>) -> Result<()> { + fn analyze_value_expr(&mut self, expr: &'a Expr) -> Result<()> { let mut comprs = vec![]; traverse(expr, &mut |e| match e { ArrayCompr { .. } | SetCompr { .. } | ObjectCompr { .. } => { @@ -527,8 +527,8 @@ impl<'a> Analyzer<'a> { fn analyze_rule_head( &mut self, - head: &'a RuleHead<'a>, - ) -> Result<(Option<&'a Expr<'a>>, Option<&'a Expr<'a>>, Scope<'a>)> { + head: &'a RuleHead, + ) -> Result<(Option<&'a Expr>, Option<&'a Expr>, Scope<'a>)> { let mut scope = Scope::default(); Ok(match head { RuleHead::Compr { assign, .. } => (None, assign.as_ref().map(|a| &a.value), scope), @@ -536,7 +536,7 @@ impl<'a> Analyzer<'a> { RuleHead::Func { args, assign, .. } => { for a in args.iter() { if let Var(v) = a { - scope.locals.insert(v.text()); + scope.locals.insert(*v.text()); } } (None, assign.as_ref().map(|a| &a.value), scope) @@ -546,16 +546,16 @@ impl<'a> Analyzer<'a> { fn gather_local_vars( &mut self, - key: Option<&'a Expr<'a>>, - value: Option<&'a Expr<'a>>, - query: &'a Query<'a>, + key: Option<&'a Expr>, + value: Option<&'a Expr>, + query: &'a Query, scope: &mut Scope<'a>, ) -> Result<()> { // First process assign, some expressions and gather local vars. for stmt in &query.stmts { match &stmt.literal { Literal::SomeVars { vars, .. } => vars.iter().for_each(|v| { - scope.locals.insert(v.text()); + scope.locals.insert(*v.text()); }), Literal::SomeIn { key, @@ -602,16 +602,16 @@ impl<'a> Analyzer<'a> { } fn gather_used_vars_comprs_index_vars( - expr: &'a Expr<'a>, + expr: &'a Expr, scope: &mut Scope<'a>, - first_use: &mut BTreeMap<&'a str, Span<'a>>, + first_use: &mut BTreeMap<&'a str, Span>, definitions: &mut Vec>, - ) -> Result<(Vec<&'a str>, Vec<&'a Expr<'a>>)> { + ) -> Result<(Vec<&'a str>, Vec<&'a Expr>)> { let mut used_vars = vec![]; let mut comprs = vec![]; traverse(expr, &mut |e| match e { - Var(v) if !matches!(v.text(), "_" | "input" | "data") => { - let name = v.text(); + Var(v) if !matches!(*v.text(), "_" | "input" | "data") => { + let name = *v.text(); if scope.locals.contains(name) /*|| scope.inputs.contains(name) */ { @@ -625,7 +625,7 @@ impl<'a> Analyzer<'a> { RefBrack { refr, index, .. } => { if let Var(v) = index.as_ref() { - let var = v.text(); + let var = *v.text(); if scope.locals.contains(var) { let (rb_used_vars, rb_comprs) = Self::gather_used_vars_comprs_index_vars( refr, @@ -658,9 +658,9 @@ impl<'a> Analyzer<'a> { fn process_comprs( &mut self, - comprs: &[&'a Expr<'a>], + comprs: &[&'a Expr], scope: &mut Scope<'a>, - first_use: &mut BTreeMap<&'a str, Span<'a>>, + first_use: &mut BTreeMap<&'a str, Span>, used_vars: &mut Vec<&'a str>, ) -> Result<()> { self.scopes.push(scope.clone()); @@ -706,15 +706,15 @@ impl<'a> Analyzer<'a> { fn gather_assigned_vars( &self, - expr: &'a Expr<'a>, + expr: &'a Expr, scope: &Scope<'a>, check_first_use: bool, - first_use: &BTreeMap<&'a str, Span<'a>>, + first_use: &BTreeMap<&'a str, Span>, ) -> Result> { let mut vars = vec![]; traverse(expr, &mut |e| match e { Var(v) => { - let var = v.text(); + let var = *v.text(); if scope.locals.contains(var) { if check_first_use { Self::check_first_use(v, first_use)?; @@ -733,10 +733,10 @@ impl<'a> Analyzer<'a> { fn process_assign_expr( &mut self, op: &AssignOp, - lhs: &'a Expr<'a>, - rhs: &'a Expr<'a>, + lhs: &'a Expr, + rhs: &'a Expr, scope: &mut Scope<'a>, - first_use: &mut BTreeMap<&'a str, Span<'a>>, + first_use: &mut BTreeMap<&'a str, Span>, definitions: &mut Vec>, ) -> Result<()> { match (lhs, rhs) { @@ -807,9 +807,9 @@ impl<'a> Analyzer<'a> { fn process_expr( &mut self, - expr: &'a Expr<'a>, + expr: &'a Expr, scope: &mut Scope<'a>, - first_use: &mut BTreeMap<&'a str, Span<'a>>, + first_use: &mut BTreeMap<&'a str, Span>, definitions: &mut Vec>, ) -> Result<()> { match expr { @@ -826,8 +826,8 @@ impl<'a> Analyzer<'a> { } } - fn check_first_use(var: &Span<'a>, first_use: &BTreeMap<&'a str, Span<'a>>) -> Result<()> { - let name = var.text(); + fn check_first_use(var: &Span, first_use: &BTreeMap<&'a str, Span>) -> Result<()> { + let name = *var.text(); if let Some(r#use) = first_use.get(name) { if r#use.line < var.line || (r#use.line == var.line && r#use.col < var.col) { bail!(r#use.error( @@ -843,15 +843,15 @@ impl<'a> Analyzer<'a> { } fn gather_some_vars( - expr: &'a Expr<'a>, + expr: &'a Expr, scope: &Scope<'a>, - _first_use: &BTreeMap<&'a str, Span<'a>>, + _first_use: &BTreeMap<&'a str, Span>, vars: &mut Vec<&'a str>, - non_vars: &mut Vec<&'a Expr<'a>>, + non_vars: &mut Vec<&'a Expr>, ) -> Result<()> { traverse(expr, &mut |e| match e { - Var(v) if scope.locals.contains(v.text()) => { - vars.push(v.text()); + Var(v) if scope.locals.contains(*v.text()) => { + vars.push(*v.text()); Ok(false) } // TODO: Object key/value @@ -865,9 +865,9 @@ impl<'a> Analyzer<'a> { fn analyze_query( &mut self, - key: Option<&'a Expr<'a>>, - value: Option<&'a Expr<'a>>, - query: &'a Query<'a>, + key: Option<&'a Expr>, + value: Option<&'a Expr>, + query: &'a Query, mut scope: Scope<'a>, ) -> Result<()> { self.gather_local_vars(key, value, query, &mut scope)?; @@ -963,14 +963,14 @@ impl<'a> Analyzer<'a> { &mut first_use, &mut definitions, )?; - let var = if return_arg.text() != "_" { + let var = if *return_arg.text() != "_" { // The var in the return argument slot would have been processed as // an used var. Remove it from used vars and add it as the variable being // defined. used_vars.pop(); return_arg.text() } else { - "" + std::rc::Rc::new("") }; self.process_comprs( &comprs[..], @@ -978,7 +978,10 @@ impl<'a> Analyzer<'a> { &mut first_use, &mut used_vars, )?; - definitions.push(Definition { var, used_vars }); + definitions.push(Definition { + var: *var, + used_vars, + }); } else { self.process_expr(expr, &mut scope, &mut first_use, &mut definitions)?; } @@ -1009,9 +1012,9 @@ impl<'a> Analyzer<'a> { self.scopes.push(scope.clone()); let mut e_scope = Scope::default(); if let Some(key) = key { - e_scope.locals.insert(key.text()); + e_scope.locals.insert(*key.text()); } - e_scope.locals.insert(value.text()); + e_scope.locals.insert(*value.text()); self.scopes.push(e_scope); // TODO: mark first use of key, value so that they cannot be := assigned diff --git a/src/utils.rs b/src/utils.rs index 5f824da8..4e3cb0eb 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -9,24 +9,24 @@ use std::collections::BTreeMap; use anyhow::{bail, Result}; pub fn get_path_string(refr: &Expr, document: Option<&str>) -> Result { - let mut comps = vec![]; + let mut comps: Vec<&str> = vec![]; let mut expr = Some(refr); while expr.is_some() { match expr { Some(Expr::RefDot { refr, field, .. }) => { - comps.push(field.text()); + comps.push(&field.text()); expr = Some(refr); } Some(Expr::RefBrack { refr, index, .. }) if matches!(index.as_ref(), Expr::String(_)) => { if let Expr::String(s) = index.as_ref() { - comps.push(s.text()); + comps.push(&s.text()); expr = Some(refr); } } Some(Expr::Var(v)) => { - comps.push(v.text()); + comps.push(&v.text()); expr = None; } _ => bail!("internal error: not a simple ref"), @@ -39,9 +39,9 @@ pub fn get_path_string(refr: &Expr, document: Option<&str>) -> Result { Ok(comps.join(".")) } -pub type FunctionTable<'a> = BTreeMap>, u8)>; +pub type FunctionTable<'a> = BTreeMap, u8)>; -pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&'a Expr<'a>> { +pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&'a Expr> { if let Expr::Call { fcn, params, .. } = expr { if let Ok(path) = get_path_string(fcn, None) { let n_args = if let Some((_, n_args)) = functions.get(&path) { @@ -62,7 +62,7 @@ pub fn get_extra_arg<'a>(expr: &'a Expr, functions: &FunctionTable) -> Option<&' None } -pub fn gather_functions<'a>(modules: &[&'a Module<'a>]) -> Result> { +pub fn gather_functions<'a>(modules: &[&'a Module]) -> Result> { let mut table = FunctionTable::new(); for module in modules { diff --git a/src/value.rs b/src/value.rs index e0898ccb..e91e0a3b 100644 --- a/src/value.rs +++ b/src/value.rs @@ -165,6 +165,24 @@ impl Value { pub fn to_json_str(&self) -> Result { Ok(serde_json::to_string_pretty(self)?) } + + pub fn from_json_file(path: &String) -> Result { + match std::fs::read_to_string(path) { + Ok(c) => Self::from_json_str(c.as_str()), + Err(e) => bail!("Failed to read {path}. {e}"), + } + } + + pub fn from_yaml_str(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml)?) + } + + pub fn from_yaml_file(path: &String) -> Result { + match std::fs::read_to_string(path) { + Ok(c) => Self::from_yaml_str(c.as_str()), + Err(e) => bail!("Failed to read {path}. {e}"), + } + } } impl Value { diff --git a/tests/interpreter/mod.rs b/tests/interpreter/mod.rs index 0e1b875b..dafe041c 100644 --- a/tests/interpreter/mod.rs +++ b/tests/interpreter/mod.rs @@ -210,11 +210,7 @@ pub fn eval_file_first_rule( for (idx, file) in files.iter().enumerate() { let contents = regos[idx].as_str(); - sources.push(Source { - file, - contents, - lines: contents.split('\n').collect(), - }); + sources.push(Source::new(file.to_string(), contents.to_string())); } for source in &sources { @@ -226,13 +222,9 @@ pub fn eval_file_first_rule( modules_ref.push(m); } - let query_source = regorus::Source { - file: "", - contents: query, - lines: query.split('\n').collect(), - }; + let query_source = regorus::Source::new("".to_string(), query.to_string()); let query_span = regorus::Span { - source: &query_source, + source: query_source.clone(), line: 1, col: 1, start: 0, @@ -303,11 +295,7 @@ pub fn eval_file( for (idx, file) in files.iter().enumerate() { let contents = regos[idx].as_str(); - sources.push(Source { - file, - contents, - lines: contents.split('\n').collect(), - }); + sources.push(Source::new(file.to_string(), contents.to_string())); } for source in &sources { @@ -319,13 +307,9 @@ pub fn eval_file( modules_ref.push(m); } - let query_source = regorus::Source { - file: "", - contents: query, - lines: query.split('\n').collect(), - }; + let query_source = regorus::Source::new("(source: &'source Source<'source>) -> Result>> { +fn get_tokens(source: &Source) -> Result> { let mut tokens = vec![]; let mut lex = Lexer::new(source); loop { @@ -35,7 +35,9 @@ fn check_loc(tok: &Token) -> Result<()> { false if tok.0 == TokenKind::Eof && source_idx >= source_line.len() => return Ok(()), // Handle case where a raw string's first char is a newline. false if tok.0 == TokenKind::RawString && &tok.1.text()[0..1] == "\n" => return Ok(()), - _ => bail!("could not find caret for {tok:#?} {msg}"), + _ => { + bail!("could not find caret for {tok:#?} {msg}"); + } } match &caret_line[idx..idx + 1] { "^" => { @@ -76,12 +78,7 @@ fn yaml_test_impl(file: &str) -> Result<()> { let test: Test = serde_yaml::from_str(&yaml)?; for case in &test.cases { - let source = Source { - file: "case.rego", - contents: case.rego.as_str(), - lines: case.rego.as_str().split('\n').collect(), - }; - + let source = Source::new("case.rego".to_string(), case.rego.clone()); print!("case {} ", &case.note); match get_tokens(&source) { @@ -91,7 +88,7 @@ fn yaml_test_impl(file: &str) -> Result<()> { break; } assert_eq!( - tok.1.text(), + *tok.1.text(), case.tokens[idx], "{} Expected token `{}` not found", source.message(tok.1.line, tok.1.col, "mismatch-error", &case.tokens[idx]), @@ -184,11 +181,7 @@ fn run(path: &str) { #[test] fn debug() -> Result<()> { let rego = "\"This string is 35 characters long.\"\"short string\""; - let source = Source { - file: "case.rego", - contents: rego, - lines: rego.split('\n').collect(), - }; + let source = Source::new("case.rego".to_string(), rego.to_string()); let mut lexer = Lexer::new(&source); let tok = lexer.next_token()?; @@ -210,11 +203,7 @@ fn debug() -> Result<()> { #[test] fn tab() -> Result<()> { let rego = r#" "This string is 35 characters long."`raw string`p"#; - let source = Source { - file: "case.rego", - contents: rego, - lines: rego.split('\n').collect(), - }; + let source = Source::new("case.rego".to_string(), rego.to_string()); let mut lexer = Lexer::new(&source); @@ -244,11 +233,7 @@ fn tab() -> Result<()> { #[test] fn invalid_line() -> Result<()> { let rego = ""; - let source = Source { - file: "case.rego", - contents: rego, - lines: rego.split('\n').collect(), - }; + let source = Source::new("case.rego".to_string(), rego.to_string()); assert_eq!( source.message(2, 0, "", ""), diff --git a/tests/parser/mod.rs b/tests/parser/mod.rs index 20611b40..48954d35 100644 --- a/tests/parser/mod.rs +++ b/tests/parser/mod.rs @@ -28,7 +28,7 @@ fn match_span(s: &Span, v: &Value) -> Result<()> { match &v { Value::String(vs) => { my_assert_eq!( - s.text(), + *s.text(), vs, "{}", s.source @@ -37,7 +37,7 @@ fn match_span(s: &Span, v: &Value) -> Result<()> { } _ => { my_assert_eq!( - s.text(), + *s.text(), serde_json::to_string_pretty(v)?, "{}", s.source @@ -629,11 +629,7 @@ fn yaml_test_impl(file: &str) -> Result<()> { for case in &test.cases { print!("\ncase {} ", case.note); - let source = Source { - file: "case.rego", - contents: case.rego.as_str(), - lines: case.rego.split('\n').collect(), - }; + let source = Source::new("case.rego".to_string(), case.rego.clone()); let mut parser = Parser::new(&source)?; match parser.parse() { Ok(module) => { diff --git a/tests/scheduler/analyzer/mod.rs b/tests/scheduler/analyzer/mod.rs index c4b6afef..26f5a936 100644 --- a/tests/scheduler/analyzer/mod.rs +++ b/tests/scheduler/analyzer/mod.rs @@ -32,20 +32,10 @@ fn to_string_set(s: &BTreeSet<&str>) -> BTreeSet { } fn analyze_file(regos: &[String], expected_scopes: &[Scope]) -> Result<()> { - let mut files = vec![]; let mut sources = vec![]; let mut modules = vec![]; for (idx, _) in regos.iter().enumerate() { - files.push(format!("rego_{idx}")); - } - - for (idx, file) in files.iter().enumerate() { - let contents = regos[idx].as_str(); - sources.push(Source { - file, - contents, - lines: contents.split('\n').collect(), - }); + sources.push(Source::new(format!("rego_{idx}"), regos[idx].clone())); } for source in &sources {