From a3ba5a7f362b3ec3d0ba81e11ea2e66c35edf3a8 Mon Sep 17 00:00:00 2001
From: Erik Schierboom <erik_schierboom@hotmail.com>
Date: Tue, 12 Nov 2024 10:09:09 +0100
Subject: [PATCH] Initial commit (#1)

---
 .dockerignore                            | 18 ++++++-------
 Dockerfile                               | 13 +++++++---
 bin/run.sh                               | 32 +++++++-----------------
 tests/all-fail/all-fail.ua               |  1 +
 tests/all-fail/expected_results.json     |  2 +-
 tests/all-fail/tests.ua                  |  5 ++++
 tests/empty-file/empty-file.ua           |  0
 tests/empty-file/expected_results.json   |  4 +--
 tests/empty-file/tests.ua                |  5 ++++
 tests/partial-fail/expected_results.json |  2 +-
 tests/partial-fail/partial-fail.ua       |  1 +
 tests/partial-fail/tests.ua              |  5 ++++
 tests/success/success.ua                 |  1 +
 tests/success/tests.ua                   |  5 ++++
 tests/syntax-error/expected_results.json |  2 +-
 tests/syntax-error/syntax-error.ua       |  1 +
 tests/syntax-error/tests.ua              |  5 ++++
 17 files changed, 61 insertions(+), 41 deletions(-)
 create mode 100644 tests/all-fail/all-fail.ua
 create mode 100644 tests/all-fail/tests.ua
 create mode 100644 tests/empty-file/empty-file.ua
 create mode 100644 tests/empty-file/tests.ua
 create mode 100644 tests/partial-fail/partial-fail.ua
 create mode 100644 tests/partial-fail/tests.ua
 create mode 100644 tests/success/success.ua
 create mode 100644 tests/success/tests.ua
 create mode 100644 tests/syntax-error/syntax-error.ua
 create mode 100644 tests/syntax-error/tests.ua

diff --git a/.dockerignore b/.dockerignore
index 3eb8fac..194dac7 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,10 +1,8 @@
-.git/
-.appends/
-.github/
-.gitattributes
-.gitignore
-Dockerfile
-bin/run-in-docker.sh
-bin/run-tests-in-docker.sh
-bin/run-tests.sh
-tests/
+/.git/
+/.appends/
+/.github/
+/.gitattributes
+/.gitignore
+/bin/
+!bin/run.sh
+/tests/
diff --git a/Dockerfile b/Dockerfile
index bd3578c..4429b45 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,14 @@
-FROM alpine:3.18
+FROM rust:1.82.0-alpine3.20 AS builder
 
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apk add --no-cache linux-headers make musl-dev
+
+RUN cargo install uiua@0.13.0
+
+FROM alpine:3.20
+
+RUN apk add --no-cache jq
+
+COPY --from=builder /usr/local/cargo/bin/uiua /usr/local/bin
 
 WORKDIR /opt/test-runner
 COPY . .
diff --git a/bin/run.sh b/bin/run.sh
index f1e8b78..6a6076b 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -15,8 +15,7 @@
 # Example:
 # ./bin/run.sh two-fer path/to/solution/folder/ path/to/output/directory/
 
-# If any required arguments is missing, print the usage and exit
-if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+if [ $# -lt 3 ]; then
     echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
     exit 1
 fi
@@ -26,35 +25,22 @@ solution_dir=$(realpath "${2%/}")
 output_dir=$(realpath "${3%/}")
 results_file="${output_dir}/results.json"
 
-# Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
 
-# Run the tests for the provided implementation file and redirect stdout and
-# stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+test_output=$(uiua test "${solution_dir}/tests.ua")
 
-# Write the results.json file based on the exit code of the command that was 
-# just executed that tested the implementation file
 if [ $? -eq 0 ]; then
     jq -n '{version: 1, status: "pass"}' > ${results_file}
 else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
-
-    # OPTIONAL: Manually add colors to the output to help scanning the output for errors
-    # If the test output does not contain colors to help identify failing (or passing)
-    # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
-    #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
-    #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
-
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    if printf "${test_output}" | grep -q -E "tests? (failed|passed)"; then
+        status="fail"
+    else
+        status="error"
+    fi
+
+    jq -n --arg output "${test_output}" --arg status "${status}" '{version: 1, status: $status, message: $output}' > "${results_file}"
 fi
 
 echo "${slug}: done"
diff --git a/tests/all-fail/all-fail.ua b/tests/all-fail/all-fail.ua
new file mode 100644
index 0000000..05ca44a
--- /dev/null
+++ b/tests/all-fail/all-fail.ua
@@ -0,0 +1 @@
+Square ← ∘
diff --git a/tests/all-fail/expected_results.json b/tests/all-fail/expected_results.json
index 9ef8b6f..e313bdd 100644
--- a/tests/all-fail/expected_results.json
+++ b/tests/all-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "Error: 2\n  at /opt/test-runner/tests/all-fail/tests.ua:3:1\n3 | ⍤⤙≍ 4 Square 2\n    ─\nError: 4\n  at /opt/test-runner/tests/all-fail/tests.ua:4:1\n4 | ⍤⤙≍ 16 Square 4\n    ─\nError: 5\n  at /opt/test-runner/tests/all-fail/tests.ua:5:1\n5 | ⍤⤙≍ 25 Square 5\n    ─\nAll 3 tests failed"
 }
diff --git a/tests/all-fail/tests.ua b/tests/all-fail/tests.ua
new file mode 100644
index 0000000..9a6369a
--- /dev/null
+++ b/tests/all-fail/tests.ua
@@ -0,0 +1,5 @@
+~ "all-fail.ua" ~ Square
+
+⍤⤙≍ 4 Square 2
+⍤⤙≍ 16 Square 4
+⍤⤙≍ 25 Square 5
diff --git a/tests/empty-file/empty-file.ua b/tests/empty-file/empty-file.ua
new file mode 100644
index 0000000..e69de29
diff --git a/tests/empty-file/expected_results.json b/tests/empty-file/expected_results.json
index 9ef8b6f..7466634 100644
--- a/tests/empty-file/expected_results.json
+++ b/tests/empty-file/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
-  "message": "TODO: replace with correct output"
+  "status": "error",
+  "message": "Error: `Square` not found in module tests/empty-file/empty-file.ua\n  at /opt/test-runner/tests/empty-file/tests.ua:1:21\n1 | ~ \"empty-file.ua\" ~ Square\n                        ──────\nError: Unknown identifier `Square`\n  at /opt/test-runner/tests/empty-file/tests.ua:3:7\n3 | ⍤⤙≍ 4 Square 2\n          ──────"
 }
diff --git a/tests/empty-file/tests.ua b/tests/empty-file/tests.ua
new file mode 100644
index 0000000..bfce736
--- /dev/null
+++ b/tests/empty-file/tests.ua
@@ -0,0 +1,5 @@
+~ "empty-file.ua" ~ Square
+
+⍤⤙≍ 4 Square 2
+⍤⤙≍ 16 Square 4
+⍤⤙≍ 25 Square 5
diff --git a/tests/partial-fail/expected_results.json b/tests/partial-fail/expected_results.json
index 9ef8b6f..3cbed90 100644
--- a/tests/partial-fail/expected_results.json
+++ b/tests/partial-fail/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "Error: 8\n  at /opt/test-runner/tests/partial-fail/tests.ua:4:1\n4 | ⍤⤙≍ 16 Square 4\n    ─\nError: 10\n  at /opt/test-runner/tests/partial-fail/tests.ua:5:1\n5 | ⍤⤙≍ 25 Square 5\n    ─\n1 test passed, 2 failed"
 }
diff --git a/tests/partial-fail/partial-fail.ua b/tests/partial-fail/partial-fail.ua
new file mode 100644
index 0000000..43dda67
--- /dev/null
+++ b/tests/partial-fail/partial-fail.ua
@@ -0,0 +1 @@
+Square ← +.
diff --git a/tests/partial-fail/tests.ua b/tests/partial-fail/tests.ua
new file mode 100644
index 0000000..031db1f
--- /dev/null
+++ b/tests/partial-fail/tests.ua
@@ -0,0 +1,5 @@
+~ "partial-fail.ua" ~ Square
+
+⍤⤙≍ 4 Square 2
+⍤⤙≍ 16 Square 4
+⍤⤙≍ 25 Square 5
diff --git a/tests/success/success.ua b/tests/success/success.ua
new file mode 100644
index 0000000..d57c070
--- /dev/null
+++ b/tests/success/success.ua
@@ -0,0 +1 @@
+Square ← ×.
diff --git a/tests/success/tests.ua b/tests/success/tests.ua
new file mode 100644
index 0000000..e3226c7
--- /dev/null
+++ b/tests/success/tests.ua
@@ -0,0 +1,5 @@
+~ "success.ua" ~ Square
+
+⍤⤙≍ 4 Square 2
+⍤⤙≍ 16 Square 4
+⍤⤙≍ 25 Square 5
diff --git a/tests/syntax-error/expected_results.json b/tests/syntax-error/expected_results.json
index afc4d4e..75c4e4a 100644
--- a/tests/syntax-error/expected_results.json
+++ b/tests/syntax-error/expected_results.json
@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "TODO: replace with correct output"
+  "message": "Error: Unexpected token ]\n  at tests/syntax-error/syntax-error.ua:1:11\n1 | Square ←@[]]1)\n              ─"
 }
diff --git a/tests/syntax-error/syntax-error.ua b/tests/syntax-error/syntax-error.ua
new file mode 100644
index 0000000..4fbb686
--- /dev/null
+++ b/tests/syntax-error/syntax-error.ua
@@ -0,0 +1 @@
+Square ←@[]]1)
diff --git a/tests/syntax-error/tests.ua b/tests/syntax-error/tests.ua
new file mode 100644
index 0000000..ae514a6
--- /dev/null
+++ b/tests/syntax-error/tests.ua
@@ -0,0 +1,5 @@
+~ "syntax-error.ua" ~ Square
+
+⍤⤙≍ 4 Square 2
+⍤⤙≍ 16 Square 4
+⍤⤙≍ 25 Square 5