Add support for \ as token (#153)

* Add support for \ as token

Fixes #152

* Allow have - in the regex symbols list `[\\-]`

Sources/SynKit/Libraries/Lexer.Generator/RegExParser.cs

@@ -116,6 +116,7 @@ private char ParseLiteralRangeAtom()
         {
             var ch = this.Consume();
             var escaped = Escape(ch);
+            if (ch == '-') return ch;
             if (escaped == null) throw new FormatException($"invalid escape \\{ch} in grouping (position {this.index - 2})");
             return escaped.Value;
         }
@@ -183,7 +184,7 @@ private void Expect(char ch)
     /// </summary>
     /// <param name="ch">The character to check.</param>
     /// <returns>True, if the given character is special, false otherwise.</returns>
-    public static bool IsSpecial(char ch) => "()[]{}?*+|".Contains(ch);
+    public static bool IsSpecial(char ch) => "()[]{}?*+|\\".Contains(ch);
 
     /// <summary>
     /// Escapes the given string to be used as a literal in a regular expression.

Sources/SynKit/Tests/Lexer.Tests/AdvancedLexerGeneratorTests.cs

@@ -0,0 +1,35 @@
+// Copyright (c) 2021-2022 Yoakke.
+// Licensed under the Apache License, Version 2.0.
+// Source repository: https://github.com/LanguageDev/Yoakke
+
+using Xunit;
+using Yoakke.SynKit.Lexer.Attributes;
+using IgnoreAttribute = Yoakke.SynKit.Lexer.Attributes.IgnoreAttribute;
+
+namespace Yoakke.SynKit.Lexer.Tests;
+
+public partial class AdvancedLexerGeneratorTests : TestBase<AdvancedLexerGeneratorTests.TokenType>
+{
+    public enum TokenType
+    {
+        [Ignore][Regex(Regexes.Whitespace)] Whitespace,
+
+        [Error] Error,
+        [End] End,
+
+        [Regex("[\\-]")] MinusInRegexGroup,
+    }
+
+    [Lexer(typeof(TokenType))]
+    internal partial class AdvancedLexer
+    {
+    }
+
+    [Fact]
+    public void AdvancedRulesSequence()
+    {
+        var lexer = new AdvancedLexer("-");
+        Assert.Equal(Token("-", TokenType.MinusInRegexGroup, Range((0, 0), 1)), lexer.Next());
+        Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 1), 0)), lexer.Next());
+    }
+}

Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs

@@ -23,6 +23,7 @@ [Ignore] [Regex(Regexes.Whitespace)] Whitespace,
         [Token("+")] Plus,
         [Token("-")] Minus,
         [Regex(Regexes.IntLiteral)] Number,
+        [Token("\\")] Slash,
     }
 
     [Lexer(typeof(TokenType))]
@@ -47,13 +48,14 @@ public void EmptySpaces()
     [Fact]
     public void SimpleSequence()
     {
-        var lexer = new Lexer("if    asd+b 123");
+        var lexer = new Lexer("if    asd+b 123\\");
         Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next());
         Assert.Equal(Token("asd", TokenType.Identifier, Range((0, 6), 3)), lexer.Next());
         Assert.Equal(Token("+", TokenType.Plus, Range((0, 9), 1)), lexer.Next());
         Assert.Equal(Token("b", TokenType.Identifier, Range((0, 10), 1)), lexer.Next());
         Assert.Equal(Token("123", TokenType.Number, Range((0, 12), 3)), lexer.Next());
-        Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 15), 0)), lexer.Next());
+        Assert.Equal(Token("\\", TokenType.Slash, Range((0, 15), 1)), lexer.Next());
+        Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 16), 0)), lexer.Next());
     }
 
     [Fact]