Assign location to tokens (#150)

- That allow implement parsers which indirectly span across multiple files, like C/C++

Sources/SynKit/Libraries/C.Syntax/CLexer.cs

@@ -60,12 +60,30 @@ public CLexer(ICharStream source)
         this.source = source;
     }
 
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CLexer"/> class.
+    /// </summary>
+    /// <param name="sourceFile">The <see cref="ISourceFile"/> to read the source from.</param>
+    public CLexer(ISourceFile sourceFile)
+        : this(new TextReaderCharStream(sourceFile))
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CLexer"/> class.
+    /// </summary>
+    /// <param name="sourceFile">The <see cref="ISourceFile"/> to read the source from.</param>
+    public CLexer(SourceFile sourceFile)
+        : this(new TextReaderCharStream(sourceFile))
+    {
+    }
+
     /// <summary>
     /// Initializes a new instance of the <see cref="CLexer"/> class.
     /// </summary>
     /// <param name="reader">The <see cref="TextReader"/> to read the source from.</param>
     public CLexer(TextReader reader)
-        : this(new TextReaderCharStream(reader))
+        : this(new TextReaderCharStream(new SourceFile("<no-location>", reader)))
     {
     }
 
@@ -74,7 +92,17 @@ public CLexer(TextReader reader)
     /// </summary>
     /// <param name="source">The text to read.</param>
     public CLexer(string source)
-        : this(new StringReader(source))
+        : this((ISourceFile)new SourceFile("<no-location>", source))
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="CLexer"/> class.
+    /// </summary>
+    /// <param name="path">Path to source location.</param>
+    /// <param name="source">The text to read.</param>
+    public CLexer(string path, string source)
+        : this(new SourceFile(path, source))
     {
     }
 

Sources/SynKit/Libraries/C.Syntax/CToken.cs

@@ -13,7 +13,10 @@ namespace Yoakke.SynKit.C.Syntax;
 public sealed class CToken : IToken<CTokenType>, IEquatable<CToken>
 {
     /// <inheritdoc/>
-    public Text.Range Range { get; }
+    public Text.Range Range => this.Location.Range;
+
+    /// <inheritdoc/>
+    public Text.Location Location { get; }
 
     /// <inheritdoc/>
     public string Text { get; }
@@ -46,9 +49,9 @@ public sealed class CToken : IToken<CTokenType>, IEquatable<CToken>
     /// <param name="logicalText">The original text the <see cref="CToken"/> was parsed from.</param>
     /// <param name="kind">The <see cref="CTokenType"/> of the <see cref="CToken"/>.</param>
     /// <param name="expandedFrom">The <see cref="CToken"/> that this one was expanded from.</param>
-    public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom)
+    public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom)
     {
-        this.Range = range;
+        this.Location = range;
         this.Text = text;
         this.Kind = kind;
         this.LogicalRange = logicalRange;
@@ -64,7 +67,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log
     /// <param name="logicalRange">The logical <see cref="Text.Range"/> of the <see cref="CToken"/> in the source.</param>
     /// <param name="logicalText">The original text the <see cref="CToken"/> was parsed from.</param>
     /// <param name="kind">The <see cref="CTokenType"/> of the <see cref="CToken"/>.</param>
-    public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind)
+    public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind)
         : this(range, text, logicalRange, logicalText, kind, null)
     {
     }
@@ -82,6 +85,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log
     public bool Equals(CToken? other) =>
            other is not null
         && this.Range == other.Range
+        && this.Location.File.Path == other.Location.File.Path
         && this.Text == other.Text
         && this.Kind == other.Kind
         && this.LogicalRange == other.LogicalRange

Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs

@@ -41,8 +41,10 @@ partial {{ LexerType.Kind }} {{ LexerType.Name }}
     public ICharStream {{ SourceName }} { get; }
 
     public {{ LexerType.Name }}(ICharStream source) { this.{{ SourceName }} = source; }
-    public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(reader)) { }
-    public {{ LexerType.Name }}(string text) : this(new StringReader(text)) { }
+    public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(new SourceFile("<no-location>", reader))) { }
+    public {{ LexerType.Name }}(SourceFile reader) : this(new TextReaderCharStream(reader)) { }
+    public {{ LexerType.Name }}(string path, string text) : this(new SourceFile(path, text)) { }
+    public {{ LexerType.Name }}(string text) : this(new SourceFile("<no-location>", text)) { }
     {{ end }}
 
     {{ # Token parsing implementation }}

Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs

@@ -110,12 +110,12 @@ public static string ConsumeText(this ICharStream stream, int length, out Text.R
     /// <param name="makeToken">The factory function that receives the source <see cref="Text.Range"/> of the skipped characters
     /// and the skipped characters themselves concatenated as a string, and produces an <see cref="IToken"/> from them.</param>
     /// <returns>The constructed <see cref="IToken{TKind}"/> returned from <paramref name="makeToken"/>.</returns>
-    public static TToken ConsumeToken<TToken>(this ICharStream stream, int length, Func<Text.Range, string, TToken> makeToken)
+    public static TToken ConsumeToken<TToken>(this ICharStream stream, int length, Func<Text.Location, string, TToken> makeToken)
     {
         var start = stream.Position;
         var text = length == 0 ? string.Empty : stream.ConsumeText(length);
         var range = new Text.Range(start, stream.Position);
-        return makeToken(range, text);
+        return makeToken(new Text.Location(stream.SourceFile, range), text);
     }
 
     /// <summary>
@@ -128,5 +128,5 @@ public static TToken ConsumeToken<TToken>(this ICharStream stream, int length, F
     /// <returns>The constructed <see cref="Token{TKind}"/>.</returns>
     public static Token<TKind> ConsumeToken<TKind>(this ICharStream stream, TKind kind, int length)
         where TKind : notnull =>
-        stream.ConsumeToken(length, (range, text) => new Token<TKind>(range, text, kind));
+        stream.ConsumeToken(length, (location, text) => new Token<TKind>(location.Range, location, text, kind));
 }

Sources/SynKit/Libraries/Lexer/ICharStream.cs

@@ -12,6 +12,8 @@ namespace Yoakke.SynKit.Lexer;
 /// </summary>
 public interface ICharStream : IPeekableStream<char>
 {
+    public ISourceFile SourceFile { get; }
+
     /// <summary>
     /// The current <see cref="Text.Position"/> the stream is at.
     /// </summary>

Sources/SynKit/Libraries/Lexer/IToken.cs

@@ -4,6 +4,7 @@
 
 using System;
 using Range = Yoakke.SynKit.Text.Range;
+using Location = Yoakke.SynKit.Text.Location;
 
 namespace Yoakke.SynKit.Lexer;
 
@@ -18,6 +19,11 @@ public interface IToken : IEquatable<IToken>
     /// </summary>
     public Range Range { get; }
 
+    /// <summary>
+    /// The <see cref="Text.Location"/> that the token can be found at in the source.
+    /// </summary>
+    public Location Location { get; }
+
     /// <summary>
     /// The text this token was parsed from.
     /// </summary>

Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs

@@ -18,24 +18,46 @@ public class TextReaderCharStream : ICharStream
     /// <summary>
     /// The underlying <see cref="TextReader"/>.
     /// </summary>
-    public TextReader Underlying { get; }
+    public TextReader Underlying => this.SourceFile.Reader;
 
     /// <inheritdoc/>
     public Position Position { get; private set; }
 
     /// <inheritdoc/>
     public bool IsEnd => !this.TryPeek(out _);
 
+    /// <inheritdoc/>
+    public ISourceFile SourceFile => this.underlying;
+
     private readonly RingBuffer<char> peek = new();
     private char prevChar;
+    private ISourceFile underlying;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TextReaderCharStream"/> class.
+    /// </summary>
+    /// <param name="underlying">The unerlying <see cref="ISourceFile"/> to read from.</param>
+    public TextReaderCharStream(ISourceFile underlying)
+    {
+        this.underlying = underlying;
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TextReaderCharStream"/> class.
+    /// </summary>
+    /// <param name="underlying">The unerlying <see cref="SourceFile"/> to read from.</param>
+    public TextReaderCharStream(SourceFile underlying)
+    {
+        this.underlying = underlying;
+    }
 
     /// <summary>
     /// Initializes a new instance of the <see cref="TextReaderCharStream"/> class.
     /// </summary>
     /// <param name="underlying">The unerlying <see cref="TextReader"/> to read from.</param>
     public TextReaderCharStream(TextReader underlying)
     {
-        this.Underlying = underlying;
+        this.underlying = new SourceFile("<no-location>", underlying);
     }
 
     /// <inheritdoc/>

Sources/SynKit/Libraries/Lexer/Token.cs

@@ -4,19 +4,30 @@
 
 using System;
 using Range = Yoakke.SynKit.Text.Range;
+using Location = Yoakke.SynKit.Text.Location;
 
 namespace Yoakke.SynKit.Lexer;
 
 /// <summary>
 /// A default implementation for <see cref="IToken{TKind}"/>.
 /// </summary>
 /// <typeparam name="TKind">The kind type this <see cref="Token{TKind}"/> uses. Usually an enumeration type.</typeparam>
-public sealed record Token<TKind>(Range Range, string Text, TKind Kind) : IToken<TKind>, IEquatable<Token<TKind>>
+public sealed record Token<TKind>(Range Range, Location Location, string Text, TKind Kind) : IToken<TKind>, IEquatable<Token<TKind>>
     where TKind : notnull
 {
     /// <inheritdoc/>
     public bool Equals(IToken? other) => this.Equals(other as Token<TKind>);
 
     /// <inheritdoc/>
     public bool Equals(IToken<TKind>? other) => this.Equals(other as Token<TKind>);
+    public bool Equals(Token<TKind>? other) =>
+           other is not null
+        && this.Range == other.Range
+        && this.Location.File.Path == other.Location.File.Path
+        && this.Text == other.Text
+        && this.Kind.Equals(other.Kind);
+
+    /// <inheritdoc/>
+    public override int GetHashCode() =>
+        HashCode.Combine(this.Range, this.Text, this.Kind, this.Location.File.Path);
 }

Sources/SynKit/Libraries/Text/SourceFile.cs

@@ -19,7 +19,7 @@ public class SourceFile : TextReader, ISourceFile
     public string Path { get; }
 
     /// <inheritdoc/>
-    public TextReader Reader => this;
+    public TextReader Reader => this.underlying;
 
     /// <inheritdoc/>
     public int AvailableLines => this.lineStarts.Count;
@@ -60,6 +60,10 @@ public SourceFile(string path, string source)
         {
         }
         this.index = 0;
+
+        // this is to reset state of the reader to the "initial".
+        // I resort to this hack, since constructor from string pre-scan lines, but did not reset.
+        this.underlying = new StringReader(source);
     }
 
     /// <inheritdoc/>

Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs

@@ -11,6 +11,8 @@ namespace Yoakke.SynKit.C.Syntax.Tests;
 
 public class CLexerTests
 {
+    private static ISourceFile fakeLocation = new Text.SourceFile("<no-location>", "1");
+
     private static Range Rn(int line, int column, int length) => new(new(line, column), length);
 
     private static Range Rn(Position from, Position to) => new(from, to);
@@ -21,7 +23,7 @@ public class CLexerTests
 
     private static CToken Tok(Kind ty, Range r, string t) => Tok(ty, r, t, r, t);
 
-    private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(r, t, logicalR, logicalT, ty);
+    private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(new Location(fakeLocation, r), t, logicalR, logicalT, ty);
 
     [Theory]
 

Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs

@@ -77,8 +77,8 @@ public Token<TokenType> Next()
                 var result = this.charStream.ConsumeToken(TokenType.Identifier, length);
                 return result.Text switch
                 {
-                    "if" => new Token<TokenType>(result.Range, result.Text, TokenType.KwIf),
-                    "else" => new Token<TokenType>(result.Range, result.Text, TokenType.KwElse),
+                    "if" => new Token<TokenType>(result.Range, result.Location, result.Text, TokenType.KwIf),
+                    "else" => new Token<TokenType>(result.Range, result.Location, result.Text, TokenType.KwElse),
                     _ => result,
                 };
             }

Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs

@@ -37,9 +37,11 @@ public ExplicitCtorLexer(string text)
     [Fact]
     public void ImplicitCtors()
     {
-        Assert.Equal(3, typeof(ImplicitCtorLexer).GetConstructors().Length);
+        Assert.Equal(5, typeof(ImplicitCtorLexer).GetConstructors().Length);
         Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string) }));
+        Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string), typeof(string) }));
         Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) }));
+        Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) }));
         Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(ICharStream) }));
     }
 
@@ -48,6 +50,7 @@ public void ExplicitCtors()
     {
         Assert.Single(typeof(ExplicitCtorLexer).GetConstructors());
         Assert.NotNull(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(string) }));
+        Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) }));
         Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) }));
     }
 }

Sources/SynKit/Tests/Lexer.Tests/TestBase.cs

@@ -9,7 +9,7 @@ namespace Yoakke.SynKit.Lexer.Tests;
 public abstract class TestBase<TKind>
     where TKind : notnull
 {
-    protected static Token<TKind> Token(string value, TKind tt, Range r) => new(r, value, tt);
+    protected static Token<TKind> Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("<no-location>", "test"), r), value, tt);
 
     protected static Range Range((int Line, int Column) p1, (int Line, int Column) p2) =>
         new(new Position(p1.Line, p1.Column), new Position(p2.Line, p2.Column));