From 7ff6ca583c2c7c085d5105bca0b33624fe03c612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9F=B3=E5=8D=9A=E6=96=87?= Date: Sat, 23 Nov 2024 22:35:39 +0800 Subject: [PATCH] feat: Add location info in hand written parser --- lib/src/ast/statement.rs | 10 ++-- lib/src/parser/default_impl/mod.rs | 65 ++++++++++++++++++++------ lib/src/parser/lalrpop_impl/st.lalrpop | 2 +- lsp/Cargo.toml | 2 +- viewer/Cargo.toml | 2 +- 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/lib/src/ast/statement.rs b/lib/src/ast/statement.rs index 96edcd6..983a526 100644 --- a/lib/src/ast/statement.rs +++ b/lib/src/ast/statement.rs @@ -44,7 +44,11 @@ impl Statement { } #[inline] - pub fn expr(expr: Box, start: Option, end: Option) -> Self { + pub fn expr_stmt( + expr: Box, + start: Option, + end: Option, + ) -> Self { Self { kind: StmtKind::Expr(expr), start_pos: start, @@ -53,8 +57,8 @@ impl Statement { } #[inline] - pub fn new_expr(expr: Expression) -> Self { - Self::expr(Box::new(ExprStatement::new(expr)), None, None) + pub fn expr(expr: Expression, start: Option, end: Option) -> Self { + Self::expr_stmt(Box::new(ExprStatement::new(expr)), start, end) } #[inline] diff --git a/lib/src/parser/default_impl/mod.rs b/lib/src/parser/default_impl/mod.rs index 651227a..6c57584 100644 --- a/lib/src/parser/default_impl/mod.rs +++ b/lib/src/parser/default_impl/mod.rs @@ -2,7 +2,7 @@ use crate::ast::*; use crate::parser::token::Token; use crate::parser::*; -use smallvec::smallvec; +use smallvec::{smallvec, SmallVec}; use std::sync::Arc; /// @@ -26,6 +26,7 @@ impl DefaultParser { } impl ParserTrait for DefaultParser { + #[inline] fn name(&self) -> String { "DefaultParser".to_string() } @@ -70,7 +71,7 @@ struct DefaultParserImpl> { /// next token index next: usize, /// save all tokens, because we need to backtracking - tokens: Vec, + tokens: SmallVec<[Token; 1024]>, } impl> DefaultParserImpl { @@ -78,10 +79,38 @@ impl> DefaultParserImpl { Self { lexer, next: 0, - tokens: vec![], + tokens: smallvec![], } } + /// Clear backtracking tokens + fn clear_tokens(&mut self, pos: usize) { + self.tokens.truncate(pos); + self.next = pos; + } + + /// Get start location for Token `tok_pos` + #[inline] + fn start_location(&self, tok_pos: usize) -> Option { + Some(self.tokens[tok_pos].location) + } + + /// Get end location for Token `tok_pos` + #[inline] + fn end_location(&self, tok_pos: usize) -> Option { + let tok = &self.tokens[tok_pos]; + let mut start_loc = tok.location; + start_loc.offset += tok.length; + + Some(start_loc) + } + + /// Get the end location for self.tokens[self.next - 1] + #[inline] + fn current_end_location(&self) -> Option { + self.end_location(self.next - 1) + } + /// get next token from self.tokens[self.next], if self.next is not exist, get token from lexer. fn next(&mut self) -> Result, ParseError> { while self.next >= self.tokens.len() { @@ -543,13 +572,13 @@ impl> DefaultParserImpl { let mut statements: Vec<_> = vec![]; loop { - let p = self.next; + let pos = self.next; match self.next_token() { Err(ParseError::UnexpectedEnd) => { break; } _ => { - self.next = p; + self.next = pos; } } @@ -559,6 +588,9 @@ impl> DefaultParserImpl { } None => break, } + + // Clear backtracking tokens + self.clear_tokens(0); } // extract statement list to single statement @@ -612,6 +644,7 @@ impl> DefaultParserImpl { // 'IF' token already taken fn expect_if_statement(&mut self) -> Result { + let if_position = self.start_location(self.next - 1); let cond = match self.parse_expression()? { Some(expr) => expr, // TODO: error type @@ -632,8 +665,8 @@ impl> DefaultParserImpl { TokenKind::EndIf => { return Ok(Statement::if_stmt( Box::new(IfStatement::from_then(cond, then_ctrl)), - None, - None, + if_position, + self.end_location(self.next - 1), )); } // IF .. THEN .. ELSE .. END_IF @@ -647,8 +680,8 @@ impl> DefaultParserImpl { return Ok(Statement::if_stmt( Box::new(IfStatement::from_then_else(cond, then_ctrl, else_ctrl)), - None, - None, + if_position, + self.current_end_location(), )); } _ => self.next = pos, @@ -666,6 +699,7 @@ impl> DefaultParserImpl { fn parse_expr_statement(&mut self) -> ParseResult { let pos = self.next; + let expr_start = self.start_location(self.next); let expr = match self.parse_expression()? { Some(expr) => expr, None => { @@ -679,7 +713,11 @@ impl> DefaultParserImpl { return Ok(None); } - Ok(Some(Statement::new_expr(expr))) + Ok(Some(Statement::expr( + expr, + expr_start, + self.current_end_location(), + ))) } fn parse_op_expression(&mut self) -> ParseResult> { @@ -693,10 +731,11 @@ impl> DefaultParserImpl { fn parse_expression(&mut self) -> ParseResult { let pos = self.next; if let Some(bitor) = self.parse_bitor_expression()? { - return match self.parse_expression_fix()? { - Some(fix) => Ok(Some(Expression::new_assign(bitor, fix))), - None => Ok(Some(bitor)), + let r = match self.parse_expression_fix()? { + Some(fix) => Expression::new_assign(bitor, fix), + None => bitor, }; + return Ok(Some(r)); } self.next = pos; diff --git a/lib/src/parser/lalrpop_impl/st.lalrpop b/lib/src/parser/lalrpop_impl/st.lalrpop index 9e26590..913da97 100644 --- a/lib/src/parser/lalrpop_impl/st.lalrpop +++ b/lib/src/parser/lalrpop_impl/st.lalrpop @@ -123,7 +123,7 @@ StatementList: Statement = { /// Single statement Statement: Statement = { - ";" => Statement::expr(Box::new(ExprStatement::new(e)), Some(start), Some(end)), + ";" => Statement::expr(e, Some(start), Some(end)), => Statement::if_stmt(Box::new(e), Some(start), Some(end)), } diff --git a/lsp/Cargo.toml b/lsp/Cargo.toml index 4cf4de3..d801cb3 100644 --- a/lsp/Cargo.toml +++ b/lsp/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -stc-rs = { path = "../lib" } +stc-rs = { path = "../lib", default-features = false } pretty_env_logger = "*" log = "*" tower-lsp = { version = "*" } diff --git a/viewer/Cargo.toml b/viewer/Cargo.toml index a91ac67..f6e2e83 100644 --- a/viewer/Cargo.toml +++ b/viewer/Cargo.toml @@ -8,7 +8,7 @@ name = "viewer" path = "src/bin.rs" [dependencies] -stc-rs = { path = "../lib" } +stc-rs = { path = "../lib", default-features = false, features = ["lua_backend"] } pretty_env_logger = "*" log = "*" once_cell = "*"