From a75f7c51df0e14f079ca19f6f317bc3914b7d3c7 Mon Sep 17 00:00:00 2001 From: Anand Krishnamoorthi <35780660+anakrish@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:09:53 -0800 Subject: [PATCH] OPA conformance tests (#45) Run regorus against OPA test suite. To run full test suite: cargo test --test opa To run specific folder (e.g semver): cargo test --test opa -- semver Signed-off-by: Anand Krishnamoorthi --- Cargo.toml | 4 + src/interpreter.rs | 4 +- tests/opa.rs | 250 +++++++++++++++++++++++++++++++++++++++++++++ tests/opa/mod.rs | 169 ------------------------------ tests/tests.rs | 1 - 5 files changed, 255 insertions(+), 173 deletions(-) create mode 100644 tests/opa.rs delete mode 100644 tests/opa/mod.rs diff --git a/Cargo.toml b/Cargo.toml index 30a4fcb0..e8723451 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,3 +28,7 @@ anyhow = "1.0.66" [profile.release] debug = true +[[test]] +name="opa" +harness=false +test=false \ No newline at end of file diff --git a/src/interpreter.rs b/src/interpreter.rs index 97b549ac..6723a1ea 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1567,9 +1567,7 @@ impl Interpreter { } return r; } - return Err(span - .source - .error(span.line, span.col, "could not find function")); + bail!(span.error(format!("could not find function {fcn_path}").as_str())); } }; diff --git a/tests/opa.rs b/tests/opa.rs new file mode 100644 index 00000000..7d35f0c9 --- /dev/null +++ b/tests/opa.rs @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +use regorus::*; + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{self, Write}; +use std::path::Path; +use std::process::Command; + +use anyhow::{bail, Result}; +use clap::Parser; +use serde::{Deserialize, Serialize}; +use walkdir::WalkDir; + +const OPA_REPO: &str = "https://github.com/open-policy-agent/opa"; +const OPA_BRANCH: &str = "v0.58.0"; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +struct TestCase { + data: Option, + input: Option, + modules: Option>, + note: String, + query: String, + sort_bindings: Option, + want_result: Option, + skip: Option, + error: Option, + traces: Option, + want_error: Option, + want_error_code: Option, +} + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +struct YamlTest { + cases: Vec, +} + +fn eval_test_case(case: &TestCase) -> Result { + let mut engine = Engine::new(); + + if let Some(data) = &case.data { + engine.add_data(data.clone())?; + } + if let Some(input) = &case.input { + engine.set_input(input.clone()); + } + if let Some(modules) = &case.modules { + for (idx, rego) in modules.iter().enumerate() { + engine.add_policy(format!("rego{idx}.rego"), rego.clone())?; + } + } + let query_results = engine.eval_query(case.query.clone(), true)?; + + let mut values = vec![]; + for qr in query_results.result { + values.push(if !qr.bindings.is_empty_object() { + qr.bindings.clone() + } else if let Some(v) = qr.expressions.last() { + v["value"].clone() + } else { + Value::Undefined + }); + } + let result = Value::from_array(values); + // Make result json compatible. (E.g: avoid sets). + Value::from_json_str(&result.to_string()) +} + +fn run_opa_tests(opa_tests_dir: String, folders: &[String]) -> Result<()> { + println!("OPA TESTSUITE: {opa_tests_dir}"); + let tests_path = Path::new(&opa_tests_dir); + let mut status = BTreeMap::::new(); + let mut n = 0; + let mut missing_functions = BTreeSet::new(); + for entry in WalkDir::new(&opa_tests_dir) + .sort_by_file_name() + .into_iter() + .filter_map(|e| e.ok()) + { + let path_str = entry.path().to_string_lossy().to_string(); + if path_str == opa_tests_dir { + continue; + } + let path = Path::new(&path_str); + let path_dir = path.strip_prefix(tests_path)?.parent().unwrap(); + let path_dir_str = path_dir.to_string_lossy().to_string(); + + if path.is_dir() { + n = 0; + continue; + } else if !path.is_file() || !path_str.ends_with(".yaml") { + continue; + } + + let run_test = folders.is_empty() || folders.iter().any(|f| path_dir_str.contains(f)); + if !run_test { + continue; + } + + let entry = status.entry(path_dir_str).or_insert((0, 0)); + + let yaml_str = std::fs::read_to_string(&path_str)?; + let test: YamlTest = serde_yaml::from_str(&yaml_str)?; + + for case in &test.cases { + match (eval_test_case(case), &case.want_result) { + (Ok(actual), Some(expected)) if &actual == expected => { + entry.0 += 1; + } + (Err(_), None) if case.want_error.is_some() => { + // Expected failure. + entry.0 += 1; + } + (r, _) => { + print!("\n{} failed.", case.note); + if let Err(e) = r { + let msg = e.to_string(); + let pat = "could not find function "; + if let Some(pos) = msg.find(pat) { + let fcn = &msg[pos + pat.len()..]; + missing_functions.insert(fcn.to_string()); + } + } + let path = Path::new("target/opa/failures").join(path_dir); + std::fs::create_dir_all(path.clone())?; + + let mut cmd = "target/debug/examples/regorus eval".to_string(); + if let Some(data) = &case.data { + let json_path = path.join(format!("data{n}.json")); + cmd += format!(" -d {}", json_path.display()).as_str(); + std::fs::write(json_path, data.to_json_str()?.as_bytes())?; + }; + if let Some(input) = &case.input { + let input_path = path.join(format!("data{n}.json")); + cmd += format!(" -i {}", input_path.display()).as_str(); + std::fs::write(input_path, input.to_json_str()?.as_bytes())?; + }; + + if let Some(modules) = &case.modules { + if modules.len() == 1 { + let rego_path = path.join(format!("rego{n}.rego")); + cmd += format!(" -d {}", rego_path.display()).as_str(); + std::fs::write(rego_path, modules[0].as_bytes())?; + } else { + for (i, m) in modules.iter().enumerate() { + let rego_path = path.join(format!("rego{n}_{i}.rego")); + cmd += format!(" -d {}", rego_path.display()).as_str(); + std::fs::write(rego_path, m.as_bytes())?; + } + } + } + + std::fs::write(path.join(format!("query{n}.text")), case.query.as_bytes())?; + cmd += format!(" \"{}\"", &case.query).as_str(); + + println!(" To debug, run:\n\x1b[31m{cmd}\x1b[0m"); + entry.1 += 1; + n += 1; + continue; + } + }; + } + } + + println!("\nTESTSUITE STATUS"); + println!(" {:40} {:4} {:4}", "FOLDER", "PASS", "FAIL"); + let (mut npass, mut nfail) = (0, 0); + for (dir, (pass, fail)) in status { + if fail == 0 { + println!("\x1b[32m {dir:40}: {pass:4} {fail:4}\x1b[0m"); + } else { + println!("\x1b[31m {dir:40}: {pass:4} {fail:4}\x1b[0m"); + } + npass += pass; + nfail += fail; + } + println!(); + + if npass == 0 && nfail == 0 { + bail!("no matching tests found."); + } else if nfail == 0 { + println!("\x1b[32m {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL"); + } else { + println!("\x1b[31m {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL"); + } + + if !missing_functions.is_empty() { + println!("\nMISSING FUNCTIONS"); + for (idx, fcn) in missing_functions.iter().enumerate() { + println!("\x1b[31m {:4}: {fcn}\x1b[0m", idx + 1); + } + } + + if nfail != 0 { + bail!("OPA tests failed"); + } + + Ok(()) +} + +#[derive(clap::Parser)] +#[command(author, version, about, long_about = None)] +struct Cli { + /// Path to OPA test suite. + #[arg(long, short)] + test_suite_path: Option, + + /// Specific test folder to run. + folders: Vec, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + let opa_tests_dir = match cli.test_suite_path { + Some(p) => p, + None => { + let branch_dir = format!("target/opa/branch/{OPA_BRANCH}"); + std::fs::create_dir_all(&branch_dir)?; + if !std::path::Path::exists(Path::new(format!("{branch_dir}/.git").as_str())) { + let output = match Command::new("git") + .arg("clone") + .arg(OPA_REPO) + .arg("--depth") + .arg("1") + .arg("--single-branch") + .arg("--branch") + .arg(OPA_BRANCH) + .arg(&branch_dir) + .output() + { + Ok(o) => o, + Err(e) => { + bail!("failed to execute git clone. {e}") + } + }; + println!("status: {}", output.status); + io::stdout().write_all(&output.stdout).unwrap(); + io::stderr().write_all(&output.stderr).unwrap(); + if !output.status.success() { + bail!("failed to clone OPA repository"); + } + } + format!("{branch_dir}/test/cases/testdata") + } + }; + + run_opa_tests(opa_tests_dir, &cli.folders) +} diff --git a/tests/opa/mod.rs b/tests/opa/mod.rs deleted file mode 100644 index 0852bf00..00000000 --- a/tests/opa/mod.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use regorus::*; - -use std::collections::BTreeMap; -use std::path::Path; - -use anyhow::Result; -use walkdir::WalkDir; - -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize, PartialEq, Debug)] -struct TestCase { - data: Option, - input: Option, - modules: Option>, - note: String, - query: String, - sort_bindings: Option, - want_result: Option, - skip: Option, - error: Option, - traces: Option, - want_error: Option, - want_error_code: Option, -} - -#[derive(Serialize, Deserialize, PartialEq, Debug)] -struct YamlTest { - cases: Vec, -} - -fn eval_test_case(case: &TestCase) -> Result { - let mut engine = Engine::new(); - - if let Some(data) = &case.data { - engine.add_data(data.clone())?; - } - if let Some(input) = &case.input { - engine.set_input(input.clone()); - } - if let Some(modules) = &case.modules { - for (idx, rego) in modules.iter().enumerate() { - engine.add_policy(format!("rego{idx}.rego"), rego.clone())?; - } - } - let query_results = engine.eval_query(case.query.clone(), true)?; - - let mut values = vec![]; - for qr in query_results.result { - values.push(if !qr.bindings.is_empty_object() { - qr.bindings.clone() - } else if let Some(v) = qr.expressions.last() { - v["value"].clone() - } else { - Value::Undefined - }); - } - let result = Value::from_array(values); - // Make result json compatible. (E.g: avoid sets). - Value::from_json_str(&result.to_string()) -} - -#[test] -fn run_opa_tests() -> Result<()> { - let opa_tests_dir = match std::env::var("OPA_TESTS_DIR") { - Ok(v) => v, - _ => { - println!("OPA_TESTS_DIR environment vairable not defined."); - return Ok(()); - } - }; - - let tests_path = Path::new(&opa_tests_dir); - let mut status = BTreeMap::::new(); - let mut n = 0; - for entry in WalkDir::new(&opa_tests_dir) - .sort_by_file_name() - .into_iter() - .filter_map(|e| e.ok()) - { - let path_str = entry.path().to_string_lossy().to_string(); - let path = Path::new(&path_str); - if !path.is_file() || !path_str.ends_with(".yaml") { - continue; - } - - let path_dir = path.strip_prefix(tests_path)?.parent().unwrap(); - - let path_dir_str = path_dir.to_string_lossy().to_string(); - let entry = status.entry(path_dir_str).or_insert((0, 0)); - - let yaml_str = std::fs::read_to_string(&path_str)?; - let test: YamlTest = serde_yaml::from_str(&yaml_str)?; - - for case in &test.cases { - print!("{} ...", case.note); - match (eval_test_case(case), &case.want_result) { - (Ok(actual), Some(expected)) if &actual == expected => { - println!("passed"); - entry.0 += 1; - } - (Err(_), None) if case.want_error.is_some() => { - // Expected failure. - println!("passed"); - entry.0 += 1; - } - _ => { - let path = Path::new("target/opa").join(path_dir); - std::fs::create_dir_all(path.clone())?; - - if let Some(data) = &case.data { - std::fs::write( - path.join(format!("data{n}.json")), - data.to_json_str()?.as_bytes(), - )?; - }; - if let Some(input) = &case.input { - std::fs::write( - path.join(format!("input{n}.json")), - input.to_json_str()?.as_bytes(), - )?; - }; - - if let Some(modules) = &case.modules { - if modules.len() == 1 { - std::fs::write( - path.join(format!("rego{n}.rego")), - modules[0].as_bytes(), - )?; - } else { - for (i, m) in modules.iter().enumerate() { - std::fs::write( - path.join(format!("rego{n}_{i}.rego")), - m.as_bytes(), - )?; - } - } - } - - std::fs::write(path.join(format!("query{n}.text")), case.query.as_bytes())?; - - println!("failed"); - entry.1 += 1; - n += 1; - continue; - } - }; - } - } - - println!("TESTSUITE STATUS"); - println!(" {:40} {:4} {:4}", "FOLDER", "PASS", "FAIL"); - let (mut npass, mut nfail) = (0, 0); - for (dir, (pass, fail)) in status { - if fail == 0 { - println!("\x1b[32m {dir:40}: {pass:4} {fail:4}\x1b[0m"); - } else { - println!("\x1b[31m {dir:40}: {pass:4} {fail:4}\x1b[0m"); - } - npass += pass; - nfail += fail; - } - println!(); - println!("\x1b[31m {:40}: {npass:4} {nfail:4}\x1b[0m", "TOTAL"); - Ok(()) -} diff --git a/tests/tests.rs b/tests/tests.rs index 599a1216..c735c9a8 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -3,7 +3,6 @@ mod interpreter; mod lexer; -mod opa; mod parser; mod scheduler; mod value;