Replace regex-based lexer with character-at-a-time lexer (#406)

Cargo.toml

@@ -21,7 +21,6 @@ itertools      = "0.8.0"
 lazy_static    = "1.0.0"
 libc           = "0.2.21"
 log            = "0.4.4"
-regex          = "1.0.0"
 target         = "1.0.0"
 tempdir        = "0.3.5"
 unicode-width  = "0.1.3"

README.adoc

@@ -710,40 +710,6 @@ When a script with a shebang is executed, the system supplies the path to the sc
 
 With the above shebang, `just` will change its working directory to the location of the script. If you'd rather leave the working directory unchanged, use `#!/usr/bin/env just --working-directory . --justfile`.
 
-== Frequently Asked Questions
-
-=== What are the idiosyncrasies of make that just avoids?
-
-Make has some behaviors which are either confusing, complicated, or make it unsuitable for use as a general command runner.
-
-One example is that sometimes make won't run the commands in a recipe. For example, if you have a file called `test` and the following makefile that runs it:
-
-```make
-test:
-  ./test
-```
-
-Make will actually refuse to run it:
-
-```sh
-$ make test
-make: `test' is up to date.
-```
-
-Make sees the recipe `test` and assumes that it produces a file called `test`. It then sees that this file exists and thus assumes that the recipe doesn't need to be run.
-
-To be fair, this behavior is desirable when using make as a build system, but not when using it as a command runner.
-
-Some other examples include having to understand the difference between `=` and `:=` assignment, the confusing error messages that can be produced if you mess up your makefile, having to use `$$` to write recipes that use environment variables, and incompatibilites between different flavors of make.
-
-=== What's the relationship between just and cargo build scripts?
-
-http://doc.crates.io/build-script.html[Cargo build scripts] have a pretty specific use, which is to control how cargo builds your rust project. This might include adding flags to `rustc` invocations, building an external dependency, or running some kind of codegen step.
-
-`just`, on the other hand, is for all the other miscellaneous commands you might run as part of development. Things like running tests in different configurations, linting your code, pushing build artifacts to a server, removing temporary files, and the like.
-
-Also, although `just` is written in rust, it can be used regardless of the language or build system your project uses.
-
 == Miscellanea
 
 === Companion Tools
@@ -813,7 +779,7 @@ Before `just` was a fancy rust program it was a tiny shell script that called `m
 
 === Non-Project Specific Justfile
 
-If you want some commands to be available everwhere, put them in `~/.justfile` and add the following to your shell's initialization file:
+If you want some commands to be available everywhere, put them in `~/.justfile` and add the following to your shell's initialization file:
 
 ```sh
 alias .j='just --justfile ~/.justfile --working-directory ~'
@@ -829,6 +795,50 @@ I'm pretty sure that nobody actually uses this feature, but it's there.
 
 ¯\\_(ツ)_/¯
 
+== Contributing
+
+`just` welcomes your contributions! `just` is released under the maximally permissive [CC0](https://creativecommons.org/publicdomain/zero/1.0/legalcode.txt) public domain dedication and fallback license, so your changes must also released under this license.
+
+=== Janus
+
+[Janus](https://github.com/casey/janus) is a tool that collects and analyzes justfiles, and can determine if a new version of `just` breaks or changes the interpretation of existing justfiles.
+
+Before merging a particularly large or gruesome change, Janus should be run to make sure that nothing breaks. Don't worry about running Janus yourself, Casey will happily run it for you on changes that need it.
+
+== Frequently Asked Questions
+
+=== What are the idiosyncrasies of make that just avoids?
+
+Make has some behaviors which are either confusing, complicated, or make it unsuitable for use as a general command runner.
+
+One example is that sometimes make won't run the commands in a recipe. For example, if you have a file called `test` and the following makefile that runs it:
+
+```make
+test:
+  ./test
+```
+
+Make will actually refuse to run it:
+
+```sh
+$ make test
+make: `test' is up to date.
+```
+
+Make sees the recipe `test` and assumes that it produces a file called `test`. It then sees that this file exists and thus assumes that the recipe doesn't need to be run.
+
+To be fair, this behavior is desirable when using make as a build system, but not when using it as a command runner.
+
+Some other examples include having to understand the difference between `=` and `:=` assignment, the confusing error messages that can be produced if you mess up your makefile, having to use `$$` to write recipes that use environment variables, and incompatibilites between different flavors of make.
+
+=== What's the relationship between just and cargo build scripts?
+
+http://doc.crates.io/build-script.html[Cargo build scripts] have a pretty specific use, which is to control how cargo builds your rust project. This might include adding flags to `rustc` invocations, building an external dependency, or running some kind of codegen step.
+
+`just`, on the other hand, is for all the other miscellaneous commands you might run as part of development. Things like running tests in different configurations, linting your code, pushing build artifacts to a server, removing temporary files, and the like.
+
+Also, although `just` is written in rust, it can be used regardless of the language or build system your project uses.
+
 == Further Ramblings
 
 I personally find it very useful to write a `justfile` for almost every project, big or small.

functions.rs

@@ -0,0 +1,30 @@
+use crate::common::*;
+
+pub struct Functions<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Iterator for Functions<'a> {
+  type Item = (&'a Token<'a>, usize);
+
+  fn next(&mut self) -> Option<Self::Item> {
+    match self.stack.pop() {
+      None
+      | Some(Expression::String { .. })
+      | Some(Expression::Backtick { .. })
+      | Some(Expression::Variable { .. }) => None,
+      Some(Expression::Call {
+        token, arguments, ..
+      }) => Some((token, arguments.len())),
+      Some(Expression::Concatination { lhs, rhs }) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+      Some(Expression::Group { expression }) => {
+        self.stack.push(expression);
+        self.next()
+      }
+    }
+  }
+}

justfile

@@ -83,7 +83,7 @@ sloc:
 	! grep --color -En '.{101}' src/*.rs
 
 replace FROM TO:
-	find src -name '*.rs' | xargs sed -i '' -E 's/{{FROM}}/{{TO}}/g'
+	sd -i '{{FROM}}' '{{TO}}' src/*.rs
 
 test-quine:
 	cargo run -- quine

src/assignment_evaluator.rs

@@ -1,7 +1,5 @@
 use crate::common::*;
 
-use brev;
-
 pub struct AssignmentEvaluator<'a: 'b, 'b> {
   pub assignments: &'b BTreeMap<&'a str, Expression<'a>>,
   pub invocation_directory: &'b Result<PathBuf, String>,
@@ -53,7 +51,7 @@ impl<'a, 'b> AssignmentEvaluator<'a, 'b> {
     let mut evaluated = String::new();
     for fragment in line {
       match *fragment {
-        Fragment::Text { ref text } => evaluated += text.lexeme,
+        Fragment::Text { ref text } => evaluated += text.lexeme(),
         Fragment::Expression { ref expression } => {
           evaluated += &self.evaluate_expression(expression, arguments)?;
         }
@@ -183,7 +181,7 @@ mod test {
         output_error: OutputError::Code(code),
       } => {
         assert_eq!(code, 100);
-        assert_eq!(token.lexeme, "`f() { return 100; }; f`");
+        assert_eq!(token.lexeme(), "`f() { return 100; }; f`");
       }
       other => panic!("expected a code run error, but got: {}", other),
     }
@@ -211,7 +209,7 @@ recipe:
         token,
         output_error: OutputError::Code(_),
       } => {
-        assert_eq!(token.lexeme, "`echo $exported_variable`");
+        assert_eq!(token.lexeme(), "`echo $exported_variable`");
       }
       other => panic!("expected a backtick code errror, but got: {}", other),
     }

src/assignment_resolver.rs

@@ -45,10 +45,10 @@ impl<'a: 'b, 'b> AssignmentResolver<'a, 'b> {
       let message = format!("attempted to resolve unknown assignment `{}`", name);
       return Err(CompilationError {
         text: "",
-        index: 0,
+        offset: 0,
         line: 0,
         column: 0,
-        width: None,
+        width: 0,
         kind: Internal { message },
       });
     }
@@ -96,40 +96,40 @@ mod test {
   compilation_error_test! {
     name:   circular_variable_dependency,
     input:   "a = b\nb = a",
-    index:  0,
+    offset:  0,
     line:   0,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   CircularVariableDependency{variable: "a", circle: vec!["a", "b", "a"]},
   }
 
   compilation_error_test! {
     name:   self_variable_dependency,
     input:  "a = a",
-    index:  0,
+    offset:  0,
     line:   0,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   CircularVariableDependency{variable: "a", circle: vec!["a", "a"]},
   }
 
   compilation_error_test! {
     name:   unknown_expression_variable,
     input:  "x = yy",
-    index:  4,
+    offset:  4,
     line:   0,
     column: 4,
-    width:  Some(2),
+    width:  2,
     kind:   UndefinedVariable{variable: "yy"},
   }
 
   compilation_error_test! {
     name:   unknown_function,
     input:  "a = foo()",
-    index:  4,
+    offset:  4,
     line:   0,
     column: 4,
-    width:  Some(3),
+    width:  3,
     kind:   UnknownFunction{function: "foo"},
   }
 

src/color.rs

@@ -2,41 +2,23 @@ use crate::common::*;
 
 use ansi_term::Color::*;
 use ansi_term::{ANSIGenericString, Prefix, Style, Suffix};
-use atty::is as is_atty;
 use atty::Stream;
 
-#[derive(Copy, Clone)]
-pub enum UseColor {
-  Auto,
-  Always,
-  Never,
-}
-
 #[derive(Copy, Clone)]
 pub struct Color {
   use_color: UseColor,
   atty: bool,
   style: Style,
 }
 
-impl Default for Color {
-  fn default() -> Color {
-    Color {
-      use_color: UseColor::Never,
-      atty: false,
-      style: Style::new(),
-    }
-  }
-}
-
 impl Color {
   fn restyle(self, style: Style) -> Color {
     Color { style, ..self }
   }
 
   fn redirect(self, stream: Stream) -> Color {
     Color {
-      atty: is_atty(stream),
+      atty: atty::is(stream),
       ..self
     }
   }
@@ -138,3 +120,13 @@ impl Color {
     self.effective_style().suffix()
   }
 }
+
+impl Default for Color {
+  fn default() -> Color {
+    Color {
+      use_color: UseColor::Never,
+      atty: false,
+      style: Style::new(),
+    }
+  }
+}

src/common.rs

@@ -9,14 +9,14 @@ pub(crate) use std::{
   path::{Path, PathBuf},
   process,
   process::Command,
+  str::Chars,
   sync::{Mutex, MutexGuard},
   usize, vec,
 };
 
 pub(crate) use edit_distance::edit_distance;
 pub(crate) use libc::{EXIT_FAILURE, EXIT_SUCCESS};
 pub(crate) use log::warn;
-pub(crate) use regex::Regex;
 pub(crate) use tempdir::TempDir;
 pub(crate) use unicode_width::UnicodeWidthChar;
 
@@ -28,22 +28,31 @@ pub(crate) use crate::{
   color::Color,
   compilation_error::{CompilationError, CompilationErrorKind, CompilationResult},
   configuration::Configuration,
-  cooked_string::CookedString,
   expression::Expression,
   fragment::Fragment,
-  function::{evaluate_function, resolve_function, FunctionContext},
+  function::{evaluate_function, resolve_function},
+  function_context::FunctionContext,
+  functions::Functions,
+  interrupt_guard::InterruptGuard,
   interrupt_handler::InterruptHandler,
   justfile::Justfile,
   lexer::Lexer,
   load_dotenv::load_dotenv,
   misc::{default, empty},
   parameter::Parameter,
   parser::Parser,
-  recipe::{Recipe, RecipeContext},
+  position::Position,
+  recipe::Recipe,
+  recipe_context::RecipeContext,
   recipe_resolver::RecipeResolver,
   runtime_error::{RunResult, RuntimeError},
   shebang::Shebang,
-  token::{Token, TokenKind},
+  state::State,
+  string_literal::StringLiteral,
+  token::Token,
+  token_kind::TokenKind,
+  use_color::UseColor,
+  variables::Variables,
   verbosity::Verbosity,
 };