From 1a65f57f5cf23635d0a4177876b89fd0164d7437 Mon Sep 17 00:00:00 2001 From: Andrii Kurdiumov Date: Fri, 26 Aug 2022 01:43:00 +0600 Subject: [PATCH 1/4] Assign location to tokens - That allow implement parsers which indirectly span across multiple files, like C/C++ Related: https://github.com/LanguageDev/Yoakke/issues/34#issuecomment-1003509503 --- .../SynKit/Examples/C.Syntax.Sample/Program.cs | 2 +- .../SynKit/Examples/Lexer.Sample/Program.cs | 2 +- .../SynKit/Examples/Parser.Sample/Program.cs | 2 +- Sources/SynKit/Libraries/C.Syntax/CLexer.cs | 6 +++--- Sources/SynKit/Libraries/C.Syntax/CToken.cs | 12 ++++++++---- Sources/SynKit/Libraries/C.Syntax/UserMacro.cs | 2 +- .../Lexer.Generator/Templates/lexer.sbncs | 4 ++-- .../Libraries/Lexer/CharStreamExtensions.cs | 6 +++--- Sources/SynKit/Libraries/Lexer/ICharStream.cs | 2 ++ Sources/SynKit/Libraries/Lexer/IToken.cs | 6 ++++++ .../Libraries/Lexer/TextReaderCharStream.cs | 10 +++++++--- Sources/SynKit/Libraries/Lexer/Token.cs | 13 ++++++++++++- Sources/SynKit/Libraries/Text/SourceFile.cs | 10 +++++----- .../SynKit/Tests/C.Syntax.Tests/CLexerTests.cs | 17 ++++++++++------- .../Tests/C.Syntax.Tests/CPreProcessorTests.cs | 4 ++-- .../Tests/Lexer.Tests/CharStreamTests.cs | 6 +++--- .../SynKit/Tests/Lexer.Tests/Issue140Tests.cs | 2 +- .../SynKit/Tests/Lexer.Tests/Issue17Tests.cs | 8 ++++---- .../Tests/Lexer.Tests/LexerGeneratorTests.cs | 18 ++++++++++-------- .../SynKit/Tests/Lexer.Tests/RegexesTests.cs | 2 +- Sources/SynKit/Tests/Lexer.Tests/TestBase.cs | 2 +- .../Parser.Tests/ExpressionParserTests.cs | 2 +- .../Parser.Tests/IndirectLeftRecursionTests.cs | 2 +- .../SynKit/Tests/Parser.Tests/Issue138Tests.cs | 2 +- .../SynKit/Tests/Parser.Tests/Issue59Tests.cs | 2 +- .../SynKit/Tests/Parser.Tests/Issue62Tests.cs | 2 +- .../Tests/Parser.Tests/LeftRecursionTests.cs | 2 +- .../Tests/Parser.Tests/PunctuatedTests.cs | 4 ++-- 28 files changed, 92 insertions(+), 60 deletions(-) diff --git a/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs b/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs index 4efba14a..ed02f6b3 100644 --- a/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs +++ b/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs @@ -14,7 +14,7 @@ static void Main(string[] args) #define FOO(x, y) x ## y FOO(L, ""asd"") "; - var lexer = new CLexer(sourceCode); + var lexer = new CLexer("sample.cs", sourceCode); var pp = new CPreProcessor(lexer); while (true) diff --git a/Sources/SynKit/Examples/Lexer.Sample/Program.cs b/Sources/SynKit/Examples/Lexer.Sample/Program.cs index bbd5c903..a79cabca 100644 --- a/Sources/SynKit/Examples/Lexer.Sample/Program.cs +++ b/Sources/SynKit/Examples/Lexer.Sample/Program.cs @@ -38,7 +38,7 @@ internal class Program { private static void Main(string[] args) { - var lexer = new Lexer(Console.In); + var lexer = new Lexer(new Text.SourceFile("console", Console.In)); while (true) { var t = lexer.Next(); diff --git a/Sources/SynKit/Examples/Parser.Sample/Program.cs b/Sources/SynKit/Examples/Parser.Sample/Program.cs index 4ca8865f..9de3482a 100644 --- a/Sources/SynKit/Examples/Parser.Sample/Program.cs +++ b/Sources/SynKit/Examples/Parser.Sample/Program.cs @@ -75,7 +75,7 @@ internal class Program { private static void Main(string[] args) { - var lexer = new Lexer(Console.In); + var lexer = new Lexer(new Text.SourceFile("console", Console.In)); var parser = new Parser(lexer); while (true) diff --git a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs index 6942c099..6d79ec82 100644 --- a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs +++ b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs @@ -64,7 +64,7 @@ public CLexer(ICharStream source) /// Initializes a new instance of the class. /// /// The to read the source from. - public CLexer(TextReader reader) + public CLexer(ISourceFile reader) : this(new TextReaderCharStream(reader)) { } @@ -73,8 +73,8 @@ public CLexer(TextReader reader) /// Initializes a new instance of the class. /// /// The text to read. - public CLexer(string source) - : this(new StringReader(source)) + public CLexer(string path, string source) + : this(new SourceFile(path, source)) { } diff --git a/Sources/SynKit/Libraries/C.Syntax/CToken.cs b/Sources/SynKit/Libraries/C.Syntax/CToken.cs index 2b43dd19..88f09b1f 100644 --- a/Sources/SynKit/Libraries/C.Syntax/CToken.cs +++ b/Sources/SynKit/Libraries/C.Syntax/CToken.cs @@ -13,7 +13,10 @@ namespace Yoakke.SynKit.C.Syntax; public sealed class CToken : IToken, IEquatable { /// - public Text.Range Range { get; } + public Text.Range Range => this.Location.Range; + + /// + public Text.Location Location { get; } /// public string Text { get; } @@ -46,9 +49,9 @@ public sealed class CToken : IToken, IEquatable /// The original text the was parsed from. /// The of the . /// The that this one was expanded from. - public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom) + public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom) { - this.Range = range; + this.Location = range; this.Text = text; this.Kind = kind; this.LogicalRange = logicalRange; @@ -64,7 +67,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log /// The logical of the in the source. /// The original text the was parsed from. /// The of the . - public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind) + public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind) : this(range, text, logicalRange, logicalText, kind, null) { } @@ -82,6 +85,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log public bool Equals(CToken? other) => other is not null && this.Range == other.Range + && this.Location.File.Path == other.Location.File.Path && this.Text == other.Text && this.Kind == other.Kind && this.LogicalRange == other.LogicalRange diff --git a/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs b/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs index dd2850c8..7c3ef499 100644 --- a/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs +++ b/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs @@ -139,7 +139,7 @@ private static IReadOnlyList Paste(IReadOnlyList left, IReadOnly .Append(rightFirst.LogicalText) .ToString(); // Try to lex a single token - var lexer = new CLexer(pastedTokenSource); + var lexer = new CLexer("usermacro.cs", pastedTokenSource); var token = lexer.Next(); if (token.Kind == CTokenType.End) { diff --git a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs index 839e25bf..9e1f6af6 100644 --- a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs +++ b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs @@ -41,8 +41,8 @@ partial {{ LexerType.Kind }} {{ LexerType.Name }} public ICharStream {{ SourceName }} { get; } public {{ LexerType.Name }}(ICharStream source) { this.{{ SourceName }} = source; } - public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(reader)) { } - public {{ LexerType.Name }}(string text) : this(new StringReader(text)) { } + public {{ LexerType.Name }}(SourceFile reader) : this(new TextReaderCharStream(reader)) { } + public {{ LexerType.Name }}(string path, string text) : this(new SourceFile(path, text)) { } {{ end }} {{ # Token parsing implementation }} diff --git a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs index d43273ab..306bed1b 100644 --- a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs +++ b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs @@ -110,12 +110,12 @@ public static string ConsumeText(this ICharStream stream, int length, out Text.R /// The factory function that receives the source of the skipped characters /// and the skipped characters themselves concatenated as a string, and produces an from them. /// The constructed returned from . - public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken) + public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken) { var start = stream.Position; var text = length == 0 ? string.Empty : stream.ConsumeText(length); var range = new Text.Range(start, stream.Position); - return makeToken(range, text); + return makeToken(new Text.Location(stream.SourceFile, range), text); } /// @@ -128,5 +128,5 @@ public static TToken ConsumeToken(this ICharStream stream, int length, F /// The constructed . public static Token ConsumeToken(this ICharStream stream, TKind kind, int length) where TKind : notnull => - stream.ConsumeToken(length, (range, text) => new Token(range, text, kind)); + stream.ConsumeToken(length, (location, text) => new Token(location.Range, location, text, kind)); } diff --git a/Sources/SynKit/Libraries/Lexer/ICharStream.cs b/Sources/SynKit/Libraries/Lexer/ICharStream.cs index d8db74aa..ab42589a 100644 --- a/Sources/SynKit/Libraries/Lexer/ICharStream.cs +++ b/Sources/SynKit/Libraries/Lexer/ICharStream.cs @@ -12,6 +12,8 @@ namespace Yoakke.SynKit.Lexer; /// public interface ICharStream : IPeekableStream { + public ISourceFile SourceFile { get; } + /// /// The current the stream is at. /// diff --git a/Sources/SynKit/Libraries/Lexer/IToken.cs b/Sources/SynKit/Libraries/Lexer/IToken.cs index 5501d908..98359a36 100644 --- a/Sources/SynKit/Libraries/Lexer/IToken.cs +++ b/Sources/SynKit/Libraries/Lexer/IToken.cs @@ -4,6 +4,7 @@ using System; using Range = Yoakke.SynKit.Text.Range; +using Location = Yoakke.SynKit.Text.Location; namespace Yoakke.SynKit.Lexer; @@ -18,6 +19,11 @@ public interface IToken : IEquatable /// public Range Range { get; } + /// + /// The that the token can be found at in the source. + /// + public Location Location { get; } + /// /// The text this token was parsed from. /// diff --git a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs index f4c3e291..9af4ff47 100644 --- a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs +++ b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs @@ -18,7 +18,7 @@ public class TextReaderCharStream : ICharStream /// /// The underlying . /// - public TextReader Underlying { get; } + public TextReader Underlying => this.SourceFile.Reader; /// public Position Position { get; private set; } @@ -26,16 +26,20 @@ public class TextReaderCharStream : ICharStream /// public bool IsEnd => !this.TryPeek(out _); + /// + public ISourceFile SourceFile => underlying; + private readonly RingBuffer peek = new(); private char prevChar; + private ISourceFile underlying; /// /// Initializes a new instance of the class. /// /// The unerlying to read from. - public TextReaderCharStream(TextReader underlying) + public TextReaderCharStream(ISourceFile underlying) { - this.Underlying = underlying; + this.underlying = underlying; } /// diff --git a/Sources/SynKit/Libraries/Lexer/Token.cs b/Sources/SynKit/Libraries/Lexer/Token.cs index 582c7ec7..f02e49e5 100644 --- a/Sources/SynKit/Libraries/Lexer/Token.cs +++ b/Sources/SynKit/Libraries/Lexer/Token.cs @@ -4,6 +4,7 @@ using System; using Range = Yoakke.SynKit.Text.Range; +using Location = Yoakke.SynKit.Text.Location; namespace Yoakke.SynKit.Lexer; @@ -11,7 +12,7 @@ namespace Yoakke.SynKit.Lexer; /// A default implementation for . /// /// The kind type this uses. Usually an enumeration type. -public sealed record Token(Range Range, string Text, TKind Kind) : IToken, IEquatable> +public sealed record Token(Range Range, Location Location, string Text, TKind Kind) : IToken, IEquatable> where TKind : notnull { /// @@ -19,4 +20,14 @@ public sealed record Token(Range Range, string Text, TKind Kind) : IToken /// public bool Equals(IToken? other) => this.Equals(other as Token); + public bool Equals(Token? other) => + other is not null + && this.Range == other.Range + && this.Location.File.Path == other.Location.File.Path + && this.Text == other.Text + && this.Kind.Equals(other.Kind); + + /// + public override int GetHashCode() => + HashCode.Combine(this.Range, this.Text, this.Kind, this.Location.File.Path); } diff --git a/Sources/SynKit/Libraries/Text/SourceFile.cs b/Sources/SynKit/Libraries/Text/SourceFile.cs index 68cb5bc6..ac6904ae 100644 --- a/Sources/SynKit/Libraries/Text/SourceFile.cs +++ b/Sources/SynKit/Libraries/Text/SourceFile.cs @@ -19,7 +19,7 @@ public class SourceFile : TextReader, ISourceFile public string Path { get; } /// - public TextReader Reader => this; + public TextReader Reader => this.underlying; /// public int AvailableLines => this.lineStarts.Count; @@ -56,10 +56,10 @@ public SourceFile(string path, TextReader underlying) public SourceFile(string path, string source) : this(path, new StringReader(source)) { - while (this.ReadNextLine()) - { - } - this.index = 0; + //while (this.ReadNextLine()) + //{ + //} + //this.index = 0; } /// diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs index 23734eb3..123d944b 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs @@ -11,6 +11,8 @@ namespace Yoakke.SynKit.C.Syntax.Tests; public class CLexerTests { + private static ISourceFile fakeLocation = new Text.SourceFile("clexer_test.cs", "1"); + private static Range Rn(int line, int column, int length) => new(new(line, column), length); private static Range Rn(Position from, Position to) => new(from, to); @@ -21,7 +23,7 @@ public class CLexerTests private static CToken Tok(Kind ty, Range r, string t) => Tok(ty, r, t, r, t); - private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(r, t, logicalR, logicalT, ty); + private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(new Location(fakeLocation, r), t, logicalR, logicalT, ty); [Theory] @@ -183,11 +185,12 @@ public class CLexerTests public void LexSingleToken(Kind kind, string text) { var expected = Tok(kind, text); - var lexer = new CLexer(text); + var lexer = new CLexer("clexer_test.cs", text); var token = lexer.Next(); var end = Tok(Kind.End, Rn(0, token.Text.Length, 0), string.Empty); Assert.Equal(expected, token); - Assert.Equal(end, lexer.Next()); + token = lexer.Next(); + Assert.Equal(end, token); } [Theory] @@ -198,7 +201,7 @@ public void LexSingleToken(Kind kind, string text) public void DisabledUnicodeCharacters(Kind kind, string text) { var expected = Tok(kind, text); - var lexer = new CLexer(text) { AllowUnicodeCharacters = false }; + var lexer = new CLexer("clexer_test.cs", text) { AllowUnicodeCharacters = false }; var token = lexer.Next(); Assert.NotEqual(expected, token); } @@ -207,7 +210,7 @@ public void DisabledUnicodeCharacters(Kind kind, string text) public void SimpleSequence() { var sourceCode = "int x = 2;"; - var lexer = new CLexer(sourceCode); + var lexer = new CLexer("clexer_test.cs", sourceCode); Assert.Equal(Tok(Kind.KeywordInt, Rn(0, 0, 3), "int"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 4, 1), "x"), lexer.Next()); Assert.Equal(Tok(Kind.Assign, Rn(0, 6, 1), "="), lexer.Next()); @@ -221,7 +224,7 @@ public void LineContinuatedSequence() { var sourceCode = @"char* x = ""ab\ cd"";"; - var lexer = new CLexer(sourceCode); + var lexer = new CLexer("clexer_test.cs", sourceCode); Assert.Equal(Tok(Kind.KeywordChar, Rn(0, 0, 4), "char", Rn(0, 0, 4), "char"), lexer.Next()); Assert.Equal(Tok(Kind.Multiply, Rn(0, 4, 1), "*", Rn(0, 4, 1), "*"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 6, 1), "x", Rn(0, 6, 1), "x"), lexer.Next()); @@ -237,7 +240,7 @@ public void TrigraphLineContinuatedSequence() { var sourceCode = @"char* x = ""ab??/ cd"";"; - var lexer = new CLexer(sourceCode); + var lexer = new CLexer("clexer_test.cs", sourceCode); Assert.Equal(Tok(Kind.KeywordChar, Rn(0, 0, 4), "char", Rn(0, 0, 4), "char"), lexer.Next()); Assert.Equal(Tok(Kind.Multiply, Rn(0, 4, 1), "*", Rn(0, 4, 1), "*"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 6, 1), "x", Rn(0, 6, 1), "x"), lexer.Next()); diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs index 8d713166..e874b4f3 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs @@ -84,8 +84,8 @@ FOO LP() RP()", "0 1 2")] public void TextEquals(string beforePP, string afterPP) { - var expectLexer = new CLexer(afterPP); - var pp = new CPreProcessor(new CLexer(beforePP)) + var expectLexer = new CLexer("cpreprocessor_test.cs", afterPP); + var pp = new CPreProcessor(new CLexer("cpreprocessor_test.cs", beforePP)) .DefineCounter(); while (true) { diff --git a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs index 223d00dd..19b5cd35 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs @@ -33,7 +33,7 @@ public class Lexer : ILexer> public Lexer(string source) { - this.charStream = new TextReaderCharStream(new StringReader(source)); + this.charStream = new TextReaderCharStream(new SourceFile("test", source)); } /// @@ -77,8 +77,8 @@ public Token Next() var result = this.charStream.ConsumeToken(TokenType.Identifier, length); return result.Text switch { - "if" => new Token(result.Range, result.Text, TokenType.KwIf), - "else" => new Token(result.Range, result.Text, TokenType.KwElse), + "if" => new Token(result.Range, result.Location, result.Text, TokenType.KwIf), + "else" => new Token(result.Range, result.Location, result.Text, TokenType.KwElse), _ => result, }; } diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs index 083a53e5..d97d5b84 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs @@ -30,7 +30,7 @@ internal partial class Lexer [InlineData(@"'hello \' bye'")] public void Tests(string input) { - var lexer = new Lexer(input); + var lexer = new Lexer("issue_140.cs", input); var t = lexer.Next(); Assert.Equal(input, t.Text); Assert.Equal(TokenType.Test, t.Kind); diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs index eb94346b..3a7d4aa4 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs @@ -30,7 +30,7 @@ internal partial class ExplicitCtorLexer public ExplicitCtorLexer(string text) { - this.source = new TextReaderCharStream(new StringReader(text)); + this.source = new TextReaderCharStream(new Text.SourceFile("issue17.cs", text)); } } @@ -38,8 +38,8 @@ public ExplicitCtorLexer(string text) public void ImplicitCtors() { Assert.Equal(3, typeof(ImplicitCtorLexer).GetConstructors().Length); - Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string) })); - Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string), typeof(string) })); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(ICharStream) })); } @@ -48,6 +48,6 @@ public void ExplicitCtors() { Assert.Single(typeof(ExplicitCtorLexer).GetConstructors()); Assert.NotNull(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(string) })); - Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); + Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); } } diff --git a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs index f666d737..9e5b1430 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs @@ -33,21 +33,23 @@ internal partial class Lexer [Fact] public void Empty() { - var lexer = new Lexer(string.Empty); - Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), lexer.Next()); + var lexer = new Lexer("test", string.Empty); + var expected = Token(string.Empty, TokenType.End, Range((0, 0), 0)); + var actual = lexer.Next(); + Assert.Equal(expected, actual); } [Fact] public void EmptySpaces() { - var lexer = new Lexer(" "); + var lexer = new Lexer("test", " "); Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 3), 0)), lexer.Next()); } [Fact] public void SimpleSequence() { - var lexer = new Lexer("if asd+b 123"); + var lexer = new Lexer("test", "if asd+b 123"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("asd", TokenType.Identifier, Range((0, 6), 3)), lexer.Next()); Assert.Equal(Token("+", TokenType.Plus, Range((0, 9), 1)), lexer.Next()); @@ -59,7 +61,7 @@ public void SimpleSequence() [Fact] public void SimpleSequenceWithAlias() { - var lexer = new Lexer("if asd+b 123 IF"); + var lexer = new Lexer("test", "if asd+b 123 IF"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("asd", TokenType.Identifier, Range((0, 6), 3)), lexer.Next()); Assert.Equal(Token("+", TokenType.Plus, Range((0, 9), 1)), lexer.Next()); @@ -72,7 +74,7 @@ public void SimpleSequenceWithAlias() [Fact] public void SimpleMultilineSequence() { - var lexer = new Lexer(@"if + var lexer = new Lexer("test", @"if asd+b 123 -2 b5"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); @@ -89,7 +91,7 @@ public void SimpleMultilineSequence() [Fact] public void SimpleMultilineSequenceWithNewline() { - var lexer = new Lexer(@"if + var lexer = new Lexer("test", @"if asd+b 123 -2 b5 "); @@ -107,7 +109,7 @@ public void SimpleMultilineSequenceWithNewline() [Fact] public void UnknownCharacter() { - var lexer = new Lexer(@"if $"); + var lexer = new Lexer("test", @"if $"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("$", TokenType.Error, Range((0, 3), 1)), lexer.Next()); Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 4), 0)), lexer.Next()); diff --git a/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs b/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs index 00fd8e09..5f53e912 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs @@ -202,7 +202,7 @@ internal partial class WhitespaceLexer [InlineData("a", typeof(Whitespace), false, typeof(WhitespaceLexer))] public void SingleTokenAcceptance(string input, Type enumType, bool shouldAccept, Type lexerType) { - dynamic lexer = Activator.CreateInstance(lexerType, input)!; + dynamic lexer = Activator.CreateInstance(lexerType, "test", input)!; dynamic token = lexer.Next(); var enumValues = Enum.GetValues(enumType); diff --git a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs index 925924a9..10b49c29 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs @@ -9,7 +9,7 @@ namespace Yoakke.SynKit.Lexer.Tests; public abstract class TestBase where TKind : notnull { - protected static Token Token(string value, TKind tt, Range r) => new(r, value, tt); + protected static Token Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("test", "test"), r), value, tt); protected static Range Range((int Line, int Column) p1, (int Line, int Column) p2) => new(new Position(p1.Line, p1.Column), new Position(p2.Line, p2.Column)); diff --git a/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs b/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs index 57e4d832..709b876d 100644 --- a/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs @@ -61,7 +61,7 @@ internal partial class Parser private static int Number(IToken tok) => int.Parse(tok.Text); } - private static int Eval(string s) => new Parser(new Lexer(s)).ParseTopExpression().Ok.Value; + private static int Eval(string s) => new Parser(new Lexer("expression_parser.cs", s)).ParseTopExpression().Ok.Value; [Theory] [InlineData(3, "1+2")] diff --git a/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs b/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs index 36b7217d..0849ff53 100644 --- a/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs @@ -40,7 +40,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer(source)).ParseGrouping().Ok.Value; + new Parser(new Lexer("indirect_left_recursion.cs", source)).ParseGrouping().Ok.Value; [Theory] [InlineData("a", "a")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs index f83bc5a3..6050d980 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs @@ -135,7 +135,7 @@ private static DirectDeclarator MakeDirectDeclarator(IToken identifier) => } private static FunctionDefinition Parse(string source) => - new Parser(new Lexer(source)).ParseFunctionDefinition().Ok.Value; + new Parser(new Lexer("issue_138.cs", source)).ParseFunctionDefinition().Ok.Value; [InlineData("int main", "Specifiers=[int], Declarator=main")] [InlineData("unsigned int main", "Specifiers=[unsigned, int], Declarator=main")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs index 16afab85..ae506bcc 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs @@ -50,7 +50,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer(source)).ParseCall().Ok.Value; + new Parser(new Lexer("issue_59.cs", source)).ParseCall().Ok.Value; [Theory] [InlineData("x", "x")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs index 7ab3493e..6b550d1a 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs @@ -40,7 +40,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer(source)).ParseProgram().Ok.Value; + new Parser(new Lexer("issue_62.cs", source)).ParseProgram().Ok.Value; [Theory] [InlineData("", "; ^")] diff --git a/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs b/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs index 616d5ab3..49c76f88 100644 --- a/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs @@ -43,7 +43,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer(source)).ParseGrouping().Ok.Value; + new Parser(new Lexer("left_recursion.cs", source)).ParseGrouping().Ok.Value; [Theory] [InlineData("a", "a")] diff --git a/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs b/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs index f7739e63..7ec1e9c7 100644 --- a/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs @@ -45,9 +45,9 @@ private static List OneOrMoreNoTrailing(IToken _lp, Punctuated t.Text).ToList(); } - private static List Any0NoTrailing(string source) => new Parser(new Lexer(source)).ParseAny0NoTrailing().Ok.Value; + private static List Any0NoTrailing(string source) => new Parser(new Lexer("punctuated.cs", source)).ParseAny0NoTrailing().Ok.Value; - private static List Any1NoTrailing(string source) => new Parser(new Lexer(source)).ParseAny1NoTrailing().Ok.Value; + private static List Any1NoTrailing(string source) => new Parser(new Lexer("punctuated.cs", source)).ParseAny1NoTrailing().Ok.Value; [Fact] public void Empty0NoTrailing() => Assert.True(Any0NoTrailing("()").SequenceEqual(Array.Empty())); From b60432706702126981ee8cc16b3452b15695b5c5 Mon Sep 17 00:00:00 2001 From: Andrii Kurdiumov Date: Mon, 1 Jan 2024 18:24:02 +0600 Subject: [PATCH 2/4] Make less breaking changes. Restore backward compatible constructors for generated lexers and restore changes in tests. --- .../Examples/C.Syntax.Sample/Program.cs | 2 +- .../SynKit/Examples/Lexer.Sample/Program.cs | 2 +- .../SynKit/Examples/Parser.Sample/Program.cs | 2 +- Sources/SynKit/Libraries/C.Syntax/CLexer.cs | 32 +++++++++++++++++-- .../SynKit/Libraries/C.Syntax/UserMacro.cs | 2 +- .../Lexer.Generator/Templates/lexer.sbncs | 2 ++ .../Libraries/Lexer/TextReaderCharStream.cs | 22 +++++++++++-- Sources/SynKit/Libraries/Text/SourceFile.cs | 12 ++++--- .../Tests/Lexer.Tests/CharStreamTests.cs | 5 +-- .../SynKit/Tests/Lexer.Tests/Issue140Tests.cs | 2 +- .../SynKit/Tests/Lexer.Tests/Issue17Tests.cs | 5 ++- .../Tests/Lexer.Tests/LexerGeneratorTests.cs | 14 ++++---- .../SynKit/Tests/Lexer.Tests/RegexesTests.cs | 2 +- Sources/SynKit/Tests/Lexer.Tests/TestBase.cs | 2 +- 14 files changed, 81 insertions(+), 25 deletions(-) diff --git a/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs b/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs index ed02f6b3..4efba14a 100644 --- a/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs +++ b/Sources/SynKit/Examples/C.Syntax.Sample/Program.cs @@ -14,7 +14,7 @@ static void Main(string[] args) #define FOO(x, y) x ## y FOO(L, ""asd"") "; - var lexer = new CLexer("sample.cs", sourceCode); + var lexer = new CLexer(sourceCode); var pp = new CPreProcessor(lexer); while (true) diff --git a/Sources/SynKit/Examples/Lexer.Sample/Program.cs b/Sources/SynKit/Examples/Lexer.Sample/Program.cs index a79cabca..bbd5c903 100644 --- a/Sources/SynKit/Examples/Lexer.Sample/Program.cs +++ b/Sources/SynKit/Examples/Lexer.Sample/Program.cs @@ -38,7 +38,7 @@ internal class Program { private static void Main(string[] args) { - var lexer = new Lexer(new Text.SourceFile("console", Console.In)); + var lexer = new Lexer(Console.In); while (true) { var t = lexer.Next(); diff --git a/Sources/SynKit/Examples/Parser.Sample/Program.cs b/Sources/SynKit/Examples/Parser.Sample/Program.cs index 9de3482a..4ca8865f 100644 --- a/Sources/SynKit/Examples/Parser.Sample/Program.cs +++ b/Sources/SynKit/Examples/Parser.Sample/Program.cs @@ -75,7 +75,7 @@ internal class Program { private static void Main(string[] args) { - var lexer = new Lexer(new Text.SourceFile("console", Console.In)); + var lexer = new Lexer(Console.In); var parser = new Parser(lexer); while (true) diff --git a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs index 6d79ec82..be37d4a8 100644 --- a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs +++ b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs @@ -60,18 +60,46 @@ public CLexer(ICharStream source) this.source = source; } + /// + /// Initializes a new instance of the class. + /// + /// The to read the source from. + public CLexer(ISourceFile sourceFile) + : this(new TextReaderCharStream(sourceFile)) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The to read the source from. + public CLexer(SourceFile sourceFile) + : this(new TextReaderCharStream(sourceFile)) + { + } + /// /// Initializes a new instance of the class. /// /// The to read the source from. - public CLexer(ISourceFile reader) - : this(new TextReaderCharStream(reader)) + public CLexer(TextReader reader) + : this(new TextReaderCharStream(new SourceFile("", reader))) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The text to read. + public CLexer(string source) + : this((ISourceFile)new SourceFile("", source)) { } /// /// Initializes a new instance of the class. /// + /// Path to source location. /// The text to read. public CLexer(string path, string source) : this(new SourceFile(path, source)) diff --git a/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs b/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs index 7c3ef499..dd2850c8 100644 --- a/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs +++ b/Sources/SynKit/Libraries/C.Syntax/UserMacro.cs @@ -139,7 +139,7 @@ private static IReadOnlyList Paste(IReadOnlyList left, IReadOnly .Append(rightFirst.LogicalText) .ToString(); // Try to lex a single token - var lexer = new CLexer("usermacro.cs", pastedTokenSource); + var lexer = new CLexer(pastedTokenSource); var token = lexer.Next(); if (token.Kind == CTokenType.End) { diff --git a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs index 9e1f6af6..13aceb22 100644 --- a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs +++ b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs @@ -41,8 +41,10 @@ partial {{ LexerType.Kind }} {{ LexerType.Name }} public ICharStream {{ SourceName }} { get; } public {{ LexerType.Name }}(ICharStream source) { this.{{ SourceName }} = source; } + public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(new SourceFile("", reader))) { } public {{ LexerType.Name }}(SourceFile reader) : this(new TextReaderCharStream(reader)) { } public {{ LexerType.Name }}(string path, string text) : this(new SourceFile(path, text)) { } + public {{ LexerType.Name }}(string text) : this(new SourceFile("", text)) { } {{ end }} {{ # Token parsing implementation }} diff --git a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs index 9af4ff47..c637a37d 100644 --- a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs +++ b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs @@ -27,7 +27,7 @@ public class TextReaderCharStream : ICharStream public bool IsEnd => !this.TryPeek(out _); /// - public ISourceFile SourceFile => underlying; + public ISourceFile SourceFile => this.underlying; private readonly RingBuffer peek = new(); private char prevChar; @@ -36,12 +36,30 @@ public class TextReaderCharStream : ICharStream /// /// Initializes a new instance of the class. /// - /// The unerlying to read from. + /// The unerlying to read from. public TextReaderCharStream(ISourceFile underlying) { this.underlying = underlying; } + /// + /// Initializes a new instance of the class. + /// + /// The unerlying to read from. + public TextReaderCharStream(SourceFile underlying) + { + this.underlying = underlying; + } + + /// + /// Initializes a new instance of the class. + /// + /// The unerlying to read from. + public TextReaderCharStream(TextReader underlying) + { + this.underlying = new SourceFile("", underlying); + } + /// public bool TryPeek(out char ch) => this.TryLookAhead(0, out ch); diff --git a/Sources/SynKit/Libraries/Text/SourceFile.cs b/Sources/SynKit/Libraries/Text/SourceFile.cs index ac6904ae..e9e168b1 100644 --- a/Sources/SynKit/Libraries/Text/SourceFile.cs +++ b/Sources/SynKit/Libraries/Text/SourceFile.cs @@ -56,10 +56,14 @@ public SourceFile(string path, TextReader underlying) public SourceFile(string path, string source) : this(path, new StringReader(source)) { - //while (this.ReadNextLine()) - //{ - //} - //this.index = 0; + while (this.ReadNextLine()) + { + } + this.index = 0; + + // this is to reset state of the reader to the "initial". + // I resort to this hack, since constructor from string pre-scan lines, but did not reset. + this.underlying = new StringReader(source); } /// diff --git a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs index 19b5cd35..c8d8dc3b 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs @@ -33,7 +33,7 @@ public class Lexer : ILexer> public Lexer(string source) { - this.charStream = new TextReaderCharStream(new SourceFile("test", source)); + this.charStream = new TextReaderCharStream(new StringReader(source)); } /// @@ -90,7 +90,8 @@ public Token Next() public void Empty() { var lexer = new Lexer(string.Empty); - Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), lexer.Next()); + var actual = lexer.Next(); + Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), actual); } [Fact] diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs index d97d5b84..083a53e5 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue140Tests.cs @@ -30,7 +30,7 @@ internal partial class Lexer [InlineData(@"'hello \' bye'")] public void Tests(string input) { - var lexer = new Lexer("issue_140.cs", input); + var lexer = new Lexer(input); var t = lexer.Next(); Assert.Equal(input, t.Text); Assert.Equal(TokenType.Test, t.Kind); diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs index 3a7d4aa4..d1cb4b39 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs @@ -37,8 +37,10 @@ public ExplicitCtorLexer(string text) [Fact] public void ImplicitCtors() { - Assert.Equal(3, typeof(ImplicitCtorLexer).GetConstructors().Length); + Assert.Equal(5, typeof(ImplicitCtorLexer).GetConstructors().Length); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string), typeof(string) })); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(ICharStream) })); } @@ -49,5 +51,6 @@ public void ExplicitCtors() Assert.Single(typeof(ExplicitCtorLexer).GetConstructors()); Assert.NotNull(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(string) })); Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); + Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); } } diff --git a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs index 9e5b1430..a3ab272a 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs @@ -33,7 +33,7 @@ internal partial class Lexer [Fact] public void Empty() { - var lexer = new Lexer("test", string.Empty); + var lexer = new Lexer(string.Empty); var expected = Token(string.Empty, TokenType.End, Range((0, 0), 0)); var actual = lexer.Next(); Assert.Equal(expected, actual); @@ -42,14 +42,14 @@ public void Empty() [Fact] public void EmptySpaces() { - var lexer = new Lexer("test", " "); + var lexer = new Lexer(" "); Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 3), 0)), lexer.Next()); } [Fact] public void SimpleSequence() { - var lexer = new Lexer("test", "if asd+b 123"); + var lexer = new Lexer("if asd+b 123"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("asd", TokenType.Identifier, Range((0, 6), 3)), lexer.Next()); Assert.Equal(Token("+", TokenType.Plus, Range((0, 9), 1)), lexer.Next()); @@ -61,7 +61,7 @@ public void SimpleSequence() [Fact] public void SimpleSequenceWithAlias() { - var lexer = new Lexer("test", "if asd+b 123 IF"); + var lexer = new Lexer("if asd+b 123 IF"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("asd", TokenType.Identifier, Range((0, 6), 3)), lexer.Next()); Assert.Equal(Token("+", TokenType.Plus, Range((0, 9), 1)), lexer.Next()); @@ -74,7 +74,7 @@ public void SimpleSequenceWithAlias() [Fact] public void SimpleMultilineSequence() { - var lexer = new Lexer("test", @"if + var lexer = new Lexer(@"if asd+b 123 -2 b5"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); @@ -91,7 +91,7 @@ public void SimpleMultilineSequence() [Fact] public void SimpleMultilineSequenceWithNewline() { - var lexer = new Lexer("test", @"if + var lexer = new Lexer(@"if asd+b 123 -2 b5 "); @@ -109,7 +109,7 @@ public void SimpleMultilineSequenceWithNewline() [Fact] public void UnknownCharacter() { - var lexer = new Lexer("test", @"if $"); + var lexer = new Lexer(@"if $"); Assert.Equal(Token("if", TokenType.KwIf, Range((0, 0), 2)), lexer.Next()); Assert.Equal(Token("$", TokenType.Error, Range((0, 3), 1)), lexer.Next()); Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 4), 0)), lexer.Next()); diff --git a/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs b/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs index 5f53e912..00fd8e09 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/RegexesTests.cs @@ -202,7 +202,7 @@ internal partial class WhitespaceLexer [InlineData("a", typeof(Whitespace), false, typeof(WhitespaceLexer))] public void SingleTokenAcceptance(string input, Type enumType, bool shouldAccept, Type lexerType) { - dynamic lexer = Activator.CreateInstance(lexerType, "test", input)!; + dynamic lexer = Activator.CreateInstance(lexerType, input)!; dynamic token = lexer.Next(); var enumValues = Enum.GetValues(enumType); diff --git a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs index 10b49c29..8e765cf8 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs @@ -9,7 +9,7 @@ namespace Yoakke.SynKit.Lexer.Tests; public abstract class TestBase where TKind : notnull { - protected static Token Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("test", "test"), r), value, tt); + protected static Token Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("", "test"), r), value, tt); protected static Range Range((int Line, int Column) p1, (int Line, int Column) p2) => new(new Position(p1.Line, p1.Column), new Position(p2.Line, p2.Column)); From be528877d487fc54cd61fb558669e8990e338e0e Mon Sep 17 00:00:00 2001 From: Andrii Kurdiumov Date: Mon, 1 Jan 2024 18:34:55 +0600 Subject: [PATCH 3/4] Restore more tests to it's pristine state --- Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs | 12 ++++++------ .../Tests/C.Syntax.Tests/CPreProcessorTests.cs | 4 ++-- Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs | 2 +- .../SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs | 4 +--- .../Tests/Parser.Tests/ExpressionParserTests.cs | 2 +- .../Tests/Parser.Tests/IndirectLeftRecursionTests.cs | 2 +- Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs | 2 +- Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs | 2 +- Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs | 2 +- .../SynKit/Tests/Parser.Tests/LeftRecursionTests.cs | 2 +- Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs | 4 ++-- 11 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs index 123d944b..a9b68df9 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs @@ -11,7 +11,7 @@ namespace Yoakke.SynKit.C.Syntax.Tests; public class CLexerTests { - private static ISourceFile fakeLocation = new Text.SourceFile("clexer_test.cs", "1"); + private static ISourceFile fakeLocation = new Text.SourceFile("", "1"); private static Range Rn(int line, int column, int length) => new(new(line, column), length); @@ -185,7 +185,7 @@ public class CLexerTests public void LexSingleToken(Kind kind, string text) { var expected = Tok(kind, text); - var lexer = new CLexer("clexer_test.cs", text); + var lexer = new CLexer(text); var token = lexer.Next(); var end = Tok(Kind.End, Rn(0, token.Text.Length, 0), string.Empty); Assert.Equal(expected, token); @@ -201,7 +201,7 @@ public void LexSingleToken(Kind kind, string text) public void DisabledUnicodeCharacters(Kind kind, string text) { var expected = Tok(kind, text); - var lexer = new CLexer("clexer_test.cs", text) { AllowUnicodeCharacters = false }; + var lexer = new CLexer(text) { AllowUnicodeCharacters = false }; var token = lexer.Next(); Assert.NotEqual(expected, token); } @@ -210,7 +210,7 @@ public void DisabledUnicodeCharacters(Kind kind, string text) public void SimpleSequence() { var sourceCode = "int x = 2;"; - var lexer = new CLexer("clexer_test.cs", sourceCode); + var lexer = new CLexer(sourceCode); Assert.Equal(Tok(Kind.KeywordInt, Rn(0, 0, 3), "int"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 4, 1), "x"), lexer.Next()); Assert.Equal(Tok(Kind.Assign, Rn(0, 6, 1), "="), lexer.Next()); @@ -224,7 +224,7 @@ public void LineContinuatedSequence() { var sourceCode = @"char* x = ""ab\ cd"";"; - var lexer = new CLexer("clexer_test.cs", sourceCode); + var lexer = new CLexer(sourceCode); Assert.Equal(Tok(Kind.KeywordChar, Rn(0, 0, 4), "char", Rn(0, 0, 4), "char"), lexer.Next()); Assert.Equal(Tok(Kind.Multiply, Rn(0, 4, 1), "*", Rn(0, 4, 1), "*"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 6, 1), "x", Rn(0, 6, 1), "x"), lexer.Next()); @@ -240,7 +240,7 @@ public void TrigraphLineContinuatedSequence() { var sourceCode = @"char* x = ""ab??/ cd"";"; - var lexer = new CLexer("clexer_test.cs", sourceCode); + var lexer = new CLexer(sourceCode); Assert.Equal(Tok(Kind.KeywordChar, Rn(0, 0, 4), "char", Rn(0, 0, 4), "char"), lexer.Next()); Assert.Equal(Tok(Kind.Multiply, Rn(0, 4, 1), "*", Rn(0, 4, 1), "*"), lexer.Next()); Assert.Equal(Tok(Kind.Identifier, Rn(0, 6, 1), "x", Rn(0, 6, 1), "x"), lexer.Next()); diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs index e874b4f3..8d713166 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CPreProcessorTests.cs @@ -84,8 +84,8 @@ FOO LP() RP()", "0 1 2")] public void TextEquals(string beforePP, string afterPP) { - var expectLexer = new CLexer("cpreprocessor_test.cs", afterPP); - var pp = new CPreProcessor(new CLexer("cpreprocessor_test.cs", beforePP)) + var expectLexer = new CLexer(afterPP); + var pp = new CPreProcessor(new CLexer(beforePP)) .DefineCounter(); while (true) { diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs index d1cb4b39..6a5d3599 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs @@ -30,7 +30,7 @@ internal partial class ExplicitCtorLexer public ExplicitCtorLexer(string text) { - this.source = new TextReaderCharStream(new Text.SourceFile("issue17.cs", text)); + this.source = new TextReaderCharStream(new StringReader(text)); } } diff --git a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs index a3ab272a..f666d737 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/LexerGeneratorTests.cs @@ -34,9 +34,7 @@ internal partial class Lexer public void Empty() { var lexer = new Lexer(string.Empty); - var expected = Token(string.Empty, TokenType.End, Range((0, 0), 0)); - var actual = lexer.Next(); - Assert.Equal(expected, actual); + Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), lexer.Next()); } [Fact] diff --git a/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs b/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs index 709b876d..57e4d832 100644 --- a/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/ExpressionParserTests.cs @@ -61,7 +61,7 @@ internal partial class Parser private static int Number(IToken tok) => int.Parse(tok.Text); } - private static int Eval(string s) => new Parser(new Lexer("expression_parser.cs", s)).ParseTopExpression().Ok.Value; + private static int Eval(string s) => new Parser(new Lexer(s)).ParseTopExpression().Ok.Value; [Theory] [InlineData(3, "1+2")] diff --git a/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs b/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs index 0849ff53..36b7217d 100644 --- a/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/IndirectLeftRecursionTests.cs @@ -40,7 +40,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer("indirect_left_recursion.cs", source)).ParseGrouping().Ok.Value; + new Parser(new Lexer(source)).ParseGrouping().Ok.Value; [Theory] [InlineData("a", "a")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs index 6050d980..f83bc5a3 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue138Tests.cs @@ -135,7 +135,7 @@ private static DirectDeclarator MakeDirectDeclarator(IToken identifier) => } private static FunctionDefinition Parse(string source) => - new Parser(new Lexer("issue_138.cs", source)).ParseFunctionDefinition().Ok.Value; + new Parser(new Lexer(source)).ParseFunctionDefinition().Ok.Value; [InlineData("int main", "Specifiers=[int], Declarator=main")] [InlineData("unsigned int main", "Specifiers=[unsigned, int], Declarator=main")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs index ae506bcc..16afab85 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue59Tests.cs @@ -50,7 +50,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer("issue_59.cs", source)).ParseCall().Ok.Value; + new Parser(new Lexer(source)).ParseCall().Ok.Value; [Theory] [InlineData("x", "x")] diff --git a/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs b/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs index 6b550d1a..7ab3493e 100644 --- a/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/Issue62Tests.cs @@ -40,7 +40,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer("issue_62.cs", source)).ParseProgram().Ok.Value; + new Parser(new Lexer(source)).ParseProgram().Ok.Value; [Theory] [InlineData("", "; ^")] diff --git a/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs b/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs index 49c76f88..616d5ab3 100644 --- a/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/LeftRecursionTests.cs @@ -43,7 +43,7 @@ internal partial class Parser } private static string Parse(string source) => - new Parser(new Lexer("left_recursion.cs", source)).ParseGrouping().Ok.Value; + new Parser(new Lexer(source)).ParseGrouping().Ok.Value; [Theory] [InlineData("a", "a")] diff --git a/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs b/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs index 7ec1e9c7..f7739e63 100644 --- a/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs +++ b/Sources/SynKit/Tests/Parser.Tests/PunctuatedTests.cs @@ -45,9 +45,9 @@ private static List OneOrMoreNoTrailing(IToken _lp, Punctuated t.Text).ToList(); } - private static List Any0NoTrailing(string source) => new Parser(new Lexer("punctuated.cs", source)).ParseAny0NoTrailing().Ok.Value; + private static List Any0NoTrailing(string source) => new Parser(new Lexer(source)).ParseAny0NoTrailing().Ok.Value; - private static List Any1NoTrailing(string source) => new Parser(new Lexer("punctuated.cs", source)).ParseAny1NoTrailing().Ok.Value; + private static List Any1NoTrailing(string source) => new Parser(new Lexer(source)).ParseAny1NoTrailing().Ok.Value; [Fact] public void Empty0NoTrailing() => Assert.True(Any0NoTrailing("()").SequenceEqual(Array.Empty())); From 21a9ceba80df3d148200cb7b36b012e5fd42c7a6 Mon Sep 17 00:00:00 2001 From: Andrii Kurdiumov Date: Mon, 1 Jan 2024 18:42:41 +0600 Subject: [PATCH 4/4] Reduce changes in tests even more --- Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs | 3 +-- Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs index a9b68df9..98e594a8 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs @@ -189,8 +189,7 @@ public void LexSingleToken(Kind kind, string text) var token = lexer.Next(); var end = Tok(Kind.End, Rn(0, token.Text.Length, 0), string.Empty); Assert.Equal(expected, token); - token = lexer.Next(); - Assert.Equal(end, token); + Assert.Equal(end, lexer.Next()); } [Theory] diff --git a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs index c8d8dc3b..15458332 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs @@ -90,8 +90,7 @@ public Token Next() public void Empty() { var lexer = new Lexer(string.Empty); - var actual = lexer.Next(); - Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), actual); + Assert.Equal(Token(string.Empty, TokenType.End, Range((0, 0), 0)), lexer.Next()); } [Fact]