diff --git a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs index 6942c099..be37d4a8 100644 --- a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs +++ b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs @@ -60,12 +60,30 @@ public CLexer(ICharStream source) this.source = source; } + /// + /// Initializes a new instance of the class. + /// + /// The to read the source from. + public CLexer(ISourceFile sourceFile) + : this(new TextReaderCharStream(sourceFile)) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The to read the source from. + public CLexer(SourceFile sourceFile) + : this(new TextReaderCharStream(sourceFile)) + { + } + /// /// Initializes a new instance of the class. /// /// The to read the source from. public CLexer(TextReader reader) - : this(new TextReaderCharStream(reader)) + : this(new TextReaderCharStream(new SourceFile("", reader))) { } @@ -74,7 +92,17 @@ public CLexer(TextReader reader) /// /// The text to read. public CLexer(string source) - : this(new StringReader(source)) + : this((ISourceFile)new SourceFile("", source)) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Path to source location. + /// The text to read. + public CLexer(string path, string source) + : this(new SourceFile(path, source)) { } diff --git a/Sources/SynKit/Libraries/C.Syntax/CToken.cs b/Sources/SynKit/Libraries/C.Syntax/CToken.cs index 2b43dd19..88f09b1f 100644 --- a/Sources/SynKit/Libraries/C.Syntax/CToken.cs +++ b/Sources/SynKit/Libraries/C.Syntax/CToken.cs @@ -13,7 +13,10 @@ namespace Yoakke.SynKit.C.Syntax; public sealed class CToken : IToken, IEquatable { /// - public Text.Range Range { get; } + public Text.Range Range => this.Location.Range; + + /// + public Text.Location Location { get; } /// public string Text { get; } @@ -46,9 +49,9 @@ public sealed class CToken : IToken, IEquatable /// The original text the was parsed from. /// The of the . /// The that this one was expanded from. - public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom) + public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom) { - this.Range = range; + this.Location = range; this.Text = text; this.Kind = kind; this.LogicalRange = logicalRange; @@ -64,7 +67,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log /// The logical of the in the source. /// The original text the was parsed from. /// The of the . - public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind) + public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind) : this(range, text, logicalRange, logicalText, kind, null) { } @@ -82,6 +85,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log public bool Equals(CToken? other) => other is not null && this.Range == other.Range + && this.Location.File.Path == other.Location.File.Path && this.Text == other.Text && this.Kind == other.Kind && this.LogicalRange == other.LogicalRange diff --git a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs index 839e25bf..13aceb22 100644 --- a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs +++ b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs @@ -41,8 +41,10 @@ partial {{ LexerType.Kind }} {{ LexerType.Name }} public ICharStream {{ SourceName }} { get; } public {{ LexerType.Name }}(ICharStream source) { this.{{ SourceName }} = source; } - public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(reader)) { } - public {{ LexerType.Name }}(string text) : this(new StringReader(text)) { } + public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(new SourceFile("", reader))) { } + public {{ LexerType.Name }}(SourceFile reader) : this(new TextReaderCharStream(reader)) { } + public {{ LexerType.Name }}(string path, string text) : this(new SourceFile(path, text)) { } + public {{ LexerType.Name }}(string text) : this(new SourceFile("", text)) { } {{ end }} {{ # Token parsing implementation }} diff --git a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs index d43273ab..306bed1b 100644 --- a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs +++ b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs @@ -110,12 +110,12 @@ public static string ConsumeText(this ICharStream stream, int length, out Text.R /// The factory function that receives the source of the skipped characters /// and the skipped characters themselves concatenated as a string, and produces an from them. /// The constructed returned from . - public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken) + public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken) { var start = stream.Position; var text = length == 0 ? string.Empty : stream.ConsumeText(length); var range = new Text.Range(start, stream.Position); - return makeToken(range, text); + return makeToken(new Text.Location(stream.SourceFile, range), text); } /// @@ -128,5 +128,5 @@ public static TToken ConsumeToken(this ICharStream stream, int length, F /// The constructed . public static Token ConsumeToken(this ICharStream stream, TKind kind, int length) where TKind : notnull => - stream.ConsumeToken(length, (range, text) => new Token(range, text, kind)); + stream.ConsumeToken(length, (location, text) => new Token(location.Range, location, text, kind)); } diff --git a/Sources/SynKit/Libraries/Lexer/ICharStream.cs b/Sources/SynKit/Libraries/Lexer/ICharStream.cs index d8db74aa..ab42589a 100644 --- a/Sources/SynKit/Libraries/Lexer/ICharStream.cs +++ b/Sources/SynKit/Libraries/Lexer/ICharStream.cs @@ -12,6 +12,8 @@ namespace Yoakke.SynKit.Lexer; /// public interface ICharStream : IPeekableStream { + public ISourceFile SourceFile { get; } + /// /// The current the stream is at. /// diff --git a/Sources/SynKit/Libraries/Lexer/IToken.cs b/Sources/SynKit/Libraries/Lexer/IToken.cs index 5501d908..98359a36 100644 --- a/Sources/SynKit/Libraries/Lexer/IToken.cs +++ b/Sources/SynKit/Libraries/Lexer/IToken.cs @@ -4,6 +4,7 @@ using System; using Range = Yoakke.SynKit.Text.Range; +using Location = Yoakke.SynKit.Text.Location; namespace Yoakke.SynKit.Lexer; @@ -18,6 +19,11 @@ public interface IToken : IEquatable /// public Range Range { get; } + /// + /// The that the token can be found at in the source. + /// + public Location Location { get; } + /// /// The text this token was parsed from. /// diff --git a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs index f4c3e291..c637a37d 100644 --- a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs +++ b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs @@ -18,7 +18,7 @@ public class TextReaderCharStream : ICharStream /// /// The underlying . /// - public TextReader Underlying { get; } + public TextReader Underlying => this.SourceFile.Reader; /// public Position Position { get; private set; } @@ -26,8 +26,30 @@ public class TextReaderCharStream : ICharStream /// public bool IsEnd => !this.TryPeek(out _); + /// + public ISourceFile SourceFile => this.underlying; + private readonly RingBuffer peek = new(); private char prevChar; + private ISourceFile underlying; + + /// + /// Initializes a new instance of the class. + /// + /// The unerlying to read from. + public TextReaderCharStream(ISourceFile underlying) + { + this.underlying = underlying; + } + + /// + /// Initializes a new instance of the class. + /// + /// The unerlying to read from. + public TextReaderCharStream(SourceFile underlying) + { + this.underlying = underlying; + } /// /// Initializes a new instance of the class. @@ -35,7 +57,7 @@ public class TextReaderCharStream : ICharStream /// The unerlying to read from. public TextReaderCharStream(TextReader underlying) { - this.Underlying = underlying; + this.underlying = new SourceFile("", underlying); } /// diff --git a/Sources/SynKit/Libraries/Lexer/Token.cs b/Sources/SynKit/Libraries/Lexer/Token.cs index 582c7ec7..f02e49e5 100644 --- a/Sources/SynKit/Libraries/Lexer/Token.cs +++ b/Sources/SynKit/Libraries/Lexer/Token.cs @@ -4,6 +4,7 @@ using System; using Range = Yoakke.SynKit.Text.Range; +using Location = Yoakke.SynKit.Text.Location; namespace Yoakke.SynKit.Lexer; @@ -11,7 +12,7 @@ namespace Yoakke.SynKit.Lexer; /// A default implementation for . /// /// The kind type this uses. Usually an enumeration type. -public sealed record Token(Range Range, string Text, TKind Kind) : IToken, IEquatable> +public sealed record Token(Range Range, Location Location, string Text, TKind Kind) : IToken, IEquatable> where TKind : notnull { /// @@ -19,4 +20,14 @@ public sealed record Token(Range Range, string Text, TKind Kind) : IToken /// public bool Equals(IToken? other) => this.Equals(other as Token); + public bool Equals(Token? other) => + other is not null + && this.Range == other.Range + && this.Location.File.Path == other.Location.File.Path + && this.Text == other.Text + && this.Kind.Equals(other.Kind); + + /// + public override int GetHashCode() => + HashCode.Combine(this.Range, this.Text, this.Kind, this.Location.File.Path); } diff --git a/Sources/SynKit/Libraries/Text/SourceFile.cs b/Sources/SynKit/Libraries/Text/SourceFile.cs index 68cb5bc6..e9e168b1 100644 --- a/Sources/SynKit/Libraries/Text/SourceFile.cs +++ b/Sources/SynKit/Libraries/Text/SourceFile.cs @@ -19,7 +19,7 @@ public class SourceFile : TextReader, ISourceFile public string Path { get; } /// - public TextReader Reader => this; + public TextReader Reader => this.underlying; /// public int AvailableLines => this.lineStarts.Count; @@ -60,6 +60,10 @@ public SourceFile(string path, string source) { } this.index = 0; + + // this is to reset state of the reader to the "initial". + // I resort to this hack, since constructor from string pre-scan lines, but did not reset. + this.underlying = new StringReader(source); } /// diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs index 23734eb3..98e594a8 100644 --- a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs +++ b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs @@ -11,6 +11,8 @@ namespace Yoakke.SynKit.C.Syntax.Tests; public class CLexerTests { + private static ISourceFile fakeLocation = new Text.SourceFile("", "1"); + private static Range Rn(int line, int column, int length) => new(new(line, column), length); private static Range Rn(Position from, Position to) => new(from, to); @@ -21,7 +23,7 @@ public class CLexerTests private static CToken Tok(Kind ty, Range r, string t) => Tok(ty, r, t, r, t); - private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(r, t, logicalR, logicalT, ty); + private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(new Location(fakeLocation, r), t, logicalR, logicalT, ty); [Theory] diff --git a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs index 223d00dd..15458332 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs @@ -77,8 +77,8 @@ public Token Next() var result = this.charStream.ConsumeToken(TokenType.Identifier, length); return result.Text switch { - "if" => new Token(result.Range, result.Text, TokenType.KwIf), - "else" => new Token(result.Range, result.Text, TokenType.KwElse), + "if" => new Token(result.Range, result.Location, result.Text, TokenType.KwIf), + "else" => new Token(result.Range, result.Location, result.Text, TokenType.KwElse), _ => result, }; } diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs index eb94346b..6a5d3599 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs @@ -37,9 +37,11 @@ public ExplicitCtorLexer(string text) [Fact] public void ImplicitCtors() { - Assert.Equal(3, typeof(ImplicitCtorLexer).GetConstructors().Length); + Assert.Equal(5, typeof(ImplicitCtorLexer).GetConstructors().Length); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string) })); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string), typeof(string) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); + Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(ICharStream) })); } @@ -48,6 +50,7 @@ public void ExplicitCtors() { Assert.Single(typeof(ExplicitCtorLexer).GetConstructors()); Assert.NotNull(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(string) })); + Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) })); Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) })); } } diff --git a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs index 925924a9..8e765cf8 100644 --- a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs +++ b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs @@ -9,7 +9,7 @@ namespace Yoakke.SynKit.Lexer.Tests; public abstract class TestBase where TKind : notnull { - protected static Token Token(string value, TKind tt, Range r) => new(r, value, tt); + protected static Token Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("", "test"), r), value, tt); protected static Range Range((int Line, int Column) p1, (int Line, int Column) p2) => new(new Position(p1.Line, p1.Column), new Position(p2.Line, p2.Column));