diff --git a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs
index 6942c099..be37d4a8 100644
--- a/Sources/SynKit/Libraries/C.Syntax/CLexer.cs
+++ b/Sources/SynKit/Libraries/C.Syntax/CLexer.cs
@@ -60,12 +60,30 @@ public CLexer(ICharStream source)
this.source = source;
}
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The to read the source from.
+ public CLexer(ISourceFile sourceFile)
+ : this(new TextReaderCharStream(sourceFile))
+ {
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The to read the source from.
+ public CLexer(SourceFile sourceFile)
+ : this(new TextReaderCharStream(sourceFile))
+ {
+ }
+
///
/// Initializes a new instance of the class.
///
/// The to read the source from.
public CLexer(TextReader reader)
- : this(new TextReaderCharStream(reader))
+ : this(new TextReaderCharStream(new SourceFile("", reader)))
{
}
@@ -74,7 +92,17 @@ public CLexer(TextReader reader)
///
/// The text to read.
public CLexer(string source)
- : this(new StringReader(source))
+ : this((ISourceFile)new SourceFile("", source))
+ {
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// Path to source location.
+ /// The text to read.
+ public CLexer(string path, string source)
+ : this(new SourceFile(path, source))
{
}
diff --git a/Sources/SynKit/Libraries/C.Syntax/CToken.cs b/Sources/SynKit/Libraries/C.Syntax/CToken.cs
index 2b43dd19..88f09b1f 100644
--- a/Sources/SynKit/Libraries/C.Syntax/CToken.cs
+++ b/Sources/SynKit/Libraries/C.Syntax/CToken.cs
@@ -13,7 +13,10 @@ namespace Yoakke.SynKit.C.Syntax;
public sealed class CToken : IToken, IEquatable
{
///
- public Text.Range Range { get; }
+ public Text.Range Range => this.Location.Range;
+
+ ///
+ public Text.Location Location { get; }
///
public string Text { get; }
@@ -46,9 +49,9 @@ public sealed class CToken : IToken, IEquatable
/// The original text the was parsed from.
/// The of the .
/// The that this one was expanded from.
- public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom)
+ public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind, CToken? expandedFrom)
{
- this.Range = range;
+ this.Location = range;
this.Text = text;
this.Kind = kind;
this.LogicalRange = logicalRange;
@@ -64,7 +67,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log
/// The logical of the in the source.
/// The original text the was parsed from.
/// The of the .
- public CToken(Text.Range range, string text, Text.Range logicalRange, string logicalText, CTokenType kind)
+ public CToken(Text.Location range, string text, Text.Range logicalRange, string logicalText, CTokenType kind)
: this(range, text, logicalRange, logicalText, kind, null)
{
}
@@ -82,6 +85,7 @@ public CToken(Text.Range range, string text, Text.Range logicalRange, string log
public bool Equals(CToken? other) =>
other is not null
&& this.Range == other.Range
+ && this.Location.File.Path == other.Location.File.Path
&& this.Text == other.Text
&& this.Kind == other.Kind
&& this.LogicalRange == other.LogicalRange
diff --git a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs
index 839e25bf..13aceb22 100644
--- a/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs
+++ b/Sources/SynKit/Libraries/Lexer.Generator/Templates/lexer.sbncs
@@ -41,8 +41,10 @@ partial {{ LexerType.Kind }} {{ LexerType.Name }}
public ICharStream {{ SourceName }} { get; }
public {{ LexerType.Name }}(ICharStream source) { this.{{ SourceName }} = source; }
- public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(reader)) { }
- public {{ LexerType.Name }}(string text) : this(new StringReader(text)) { }
+ public {{ LexerType.Name }}(TextReader reader) : this(new TextReaderCharStream(new SourceFile("", reader))) { }
+ public {{ LexerType.Name }}(SourceFile reader) : this(new TextReaderCharStream(reader)) { }
+ public {{ LexerType.Name }}(string path, string text) : this(new SourceFile(path, text)) { }
+ public {{ LexerType.Name }}(string text) : this(new SourceFile("", text)) { }
{{ end }}
{{ # Token parsing implementation }}
diff --git a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs
index d43273ab..306bed1b 100644
--- a/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs
+++ b/Sources/SynKit/Libraries/Lexer/CharStreamExtensions.cs
@@ -110,12 +110,12 @@ public static string ConsumeText(this ICharStream stream, int length, out Text.R
/// The factory function that receives the source of the skipped characters
/// and the skipped characters themselves concatenated as a string, and produces an from them.
/// The constructed returned from .
- public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken)
+ public static TToken ConsumeToken(this ICharStream stream, int length, Func makeToken)
{
var start = stream.Position;
var text = length == 0 ? string.Empty : stream.ConsumeText(length);
var range = new Text.Range(start, stream.Position);
- return makeToken(range, text);
+ return makeToken(new Text.Location(stream.SourceFile, range), text);
}
///
@@ -128,5 +128,5 @@ public static TToken ConsumeToken(this ICharStream stream, int length, F
/// The constructed .
public static Token ConsumeToken(this ICharStream stream, TKind kind, int length)
where TKind : notnull =>
- stream.ConsumeToken(length, (range, text) => new Token(range, text, kind));
+ stream.ConsumeToken(length, (location, text) => new Token(location.Range, location, text, kind));
}
diff --git a/Sources/SynKit/Libraries/Lexer/ICharStream.cs b/Sources/SynKit/Libraries/Lexer/ICharStream.cs
index d8db74aa..ab42589a 100644
--- a/Sources/SynKit/Libraries/Lexer/ICharStream.cs
+++ b/Sources/SynKit/Libraries/Lexer/ICharStream.cs
@@ -12,6 +12,8 @@ namespace Yoakke.SynKit.Lexer;
///
public interface ICharStream : IPeekableStream
{
+ public ISourceFile SourceFile { get; }
+
///
/// The current the stream is at.
///
diff --git a/Sources/SynKit/Libraries/Lexer/IToken.cs b/Sources/SynKit/Libraries/Lexer/IToken.cs
index 5501d908..98359a36 100644
--- a/Sources/SynKit/Libraries/Lexer/IToken.cs
+++ b/Sources/SynKit/Libraries/Lexer/IToken.cs
@@ -4,6 +4,7 @@
using System;
using Range = Yoakke.SynKit.Text.Range;
+using Location = Yoakke.SynKit.Text.Location;
namespace Yoakke.SynKit.Lexer;
@@ -18,6 +19,11 @@ public interface IToken : IEquatable
///
public Range Range { get; }
+ ///
+ /// The that the token can be found at in the source.
+ ///
+ public Location Location { get; }
+
///
/// The text this token was parsed from.
///
diff --git a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs
index f4c3e291..c637a37d 100644
--- a/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs
+++ b/Sources/SynKit/Libraries/Lexer/TextReaderCharStream.cs
@@ -18,7 +18,7 @@ public class TextReaderCharStream : ICharStream
///
/// The underlying .
///
- public TextReader Underlying { get; }
+ public TextReader Underlying => this.SourceFile.Reader;
///
public Position Position { get; private set; }
@@ -26,8 +26,30 @@ public class TextReaderCharStream : ICharStream
///
public bool IsEnd => !this.TryPeek(out _);
+ ///
+ public ISourceFile SourceFile => this.underlying;
+
private readonly RingBuffer peek = new();
private char prevChar;
+ private ISourceFile underlying;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The unerlying to read from.
+ public TextReaderCharStream(ISourceFile underlying)
+ {
+ this.underlying = underlying;
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The unerlying to read from.
+ public TextReaderCharStream(SourceFile underlying)
+ {
+ this.underlying = underlying;
+ }
///
/// Initializes a new instance of the class.
@@ -35,7 +57,7 @@ public class TextReaderCharStream : ICharStream
/// The unerlying to read from.
public TextReaderCharStream(TextReader underlying)
{
- this.Underlying = underlying;
+ this.underlying = new SourceFile("", underlying);
}
///
diff --git a/Sources/SynKit/Libraries/Lexer/Token.cs b/Sources/SynKit/Libraries/Lexer/Token.cs
index 582c7ec7..f02e49e5 100644
--- a/Sources/SynKit/Libraries/Lexer/Token.cs
+++ b/Sources/SynKit/Libraries/Lexer/Token.cs
@@ -4,6 +4,7 @@
using System;
using Range = Yoakke.SynKit.Text.Range;
+using Location = Yoakke.SynKit.Text.Location;
namespace Yoakke.SynKit.Lexer;
@@ -11,7 +12,7 @@ namespace Yoakke.SynKit.Lexer;
/// A default implementation for .
///
/// The kind type this uses. Usually an enumeration type.
-public sealed record Token(Range Range, string Text, TKind Kind) : IToken, IEquatable>
+public sealed record Token(Range Range, Location Location, string Text, TKind Kind) : IToken, IEquatable>
where TKind : notnull
{
///
@@ -19,4 +20,14 @@ public sealed record Token(Range Range, string Text, TKind Kind) : IToken
///
public bool Equals(IToken? other) => this.Equals(other as Token);
+ public bool Equals(Token? other) =>
+ other is not null
+ && this.Range == other.Range
+ && this.Location.File.Path == other.Location.File.Path
+ && this.Text == other.Text
+ && this.Kind.Equals(other.Kind);
+
+ ///
+ public override int GetHashCode() =>
+ HashCode.Combine(this.Range, this.Text, this.Kind, this.Location.File.Path);
}
diff --git a/Sources/SynKit/Libraries/Text/SourceFile.cs b/Sources/SynKit/Libraries/Text/SourceFile.cs
index 68cb5bc6..e9e168b1 100644
--- a/Sources/SynKit/Libraries/Text/SourceFile.cs
+++ b/Sources/SynKit/Libraries/Text/SourceFile.cs
@@ -19,7 +19,7 @@ public class SourceFile : TextReader, ISourceFile
public string Path { get; }
///
- public TextReader Reader => this;
+ public TextReader Reader => this.underlying;
///
public int AvailableLines => this.lineStarts.Count;
@@ -60,6 +60,10 @@ public SourceFile(string path, string source)
{
}
this.index = 0;
+
+ // this is to reset state of the reader to the "initial".
+ // I resort to this hack, since constructor from string pre-scan lines, but did not reset.
+ this.underlying = new StringReader(source);
}
///
diff --git a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs
index 23734eb3..98e594a8 100644
--- a/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs
+++ b/Sources/SynKit/Tests/C.Syntax.Tests/CLexerTests.cs
@@ -11,6 +11,8 @@ namespace Yoakke.SynKit.C.Syntax.Tests;
public class CLexerTests
{
+ private static ISourceFile fakeLocation = new Text.SourceFile("", "1");
+
private static Range Rn(int line, int column, int length) => new(new(line, column), length);
private static Range Rn(Position from, Position to) => new(from, to);
@@ -21,7 +23,7 @@ public class CLexerTests
private static CToken Tok(Kind ty, Range r, string t) => Tok(ty, r, t, r, t);
- private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(r, t, logicalR, logicalT, ty);
+ private static CToken Tok(Kind ty, Range r, string t, Range logicalR, string logicalT) => new(new Location(fakeLocation, r), t, logicalR, logicalT, ty);
[Theory]
diff --git a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs
index 223d00dd..15458332 100644
--- a/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs
+++ b/Sources/SynKit/Tests/Lexer.Tests/CharStreamTests.cs
@@ -77,8 +77,8 @@ public Token Next()
var result = this.charStream.ConsumeToken(TokenType.Identifier, length);
return result.Text switch
{
- "if" => new Token(result.Range, result.Text, TokenType.KwIf),
- "else" => new Token(result.Range, result.Text, TokenType.KwElse),
+ "if" => new Token(result.Range, result.Location, result.Text, TokenType.KwIf),
+ "else" => new Token(result.Range, result.Location, result.Text, TokenType.KwElse),
_ => result,
};
}
diff --git a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs
index eb94346b..6a5d3599 100644
--- a/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs
+++ b/Sources/SynKit/Tests/Lexer.Tests/Issue17Tests.cs
@@ -37,9 +37,11 @@ public ExplicitCtorLexer(string text)
[Fact]
public void ImplicitCtors()
{
- Assert.Equal(3, typeof(ImplicitCtorLexer).GetConstructors().Length);
+ Assert.Equal(5, typeof(ImplicitCtorLexer).GetConstructors().Length);
Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string) }));
+ Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(string), typeof(string) }));
Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) }));
+ Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) }));
Assert.NotNull(typeof(ImplicitCtorLexer).GetConstructor(new[] { typeof(ICharStream) }));
}
@@ -48,6 +50,7 @@ public void ExplicitCtors()
{
Assert.Single(typeof(ExplicitCtorLexer).GetConstructors());
Assert.NotNull(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(string) }));
+ Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(Text.SourceFile) }));
Assert.Null(typeof(ExplicitCtorLexer).GetConstructor(new[] { typeof(TextReader) }));
}
}
diff --git a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs
index 925924a9..8e765cf8 100644
--- a/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs
+++ b/Sources/SynKit/Tests/Lexer.Tests/TestBase.cs
@@ -9,7 +9,7 @@ namespace Yoakke.SynKit.Lexer.Tests;
public abstract class TestBase
where TKind : notnull
{
- protected static Token Token(string value, TKind tt, Range r) => new(r, value, tt);
+ protected static Token Token(string value, TKind tt, Range r) => new(r, new Location(new SourceFile("", "test"), r), value, tt);
protected static Range Range((int Line, int Column) p1, (int Line, int Column) p2) =>
new(new Position(p1.Line, p1.Column), new Position(p2.Line, p2.Column));