feat: let parser use standalone lexer

sbwtw · Aug 26, 2024 · e25f947 · e25f947
1 parent 022cb50
commit e25f947
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 32 deletions.
diff --git a/lib/src/backend/lua/test.rs b/lib/src/backend/lua/test.rs
@@ -8,8 +8,8 @@ use crate::{parser::*, prelude::*};
 fn generate_module<S1: AsRef<str>, S2: AsRef<str>>(decl: S1, body: S2, writer: &mut dyn Write) {
     let mgr = UnitsManager::new();
     let ctx = ModuleContext::new(ModuleKind::Application);
-    let lexer = StLexerBuilder::new().build_str(decl.as_ref());
-    let decl = ParserBuilder::default().build().parse(lexer).unwrap();
+    let mut lexer = StLexerBuilder::new().build_str(decl.as_ref());
+    let decl = ParserBuilder::default().build().parse(&mut lexer).unwrap();
     let fun_id = ctx.write().add_declaration(decl, Uuid::nil());
 
     let lexer = StLexerBuilder::new().build_str(body.as_ref());

diff --git a/lib/src/parser/buffer.rs b/lib/src/parser/buffer.rs
@@ -15,10 +15,6 @@ pub trait Buffer {
     fn peek(&mut self, n: usize) -> Option<char>;
     fn current_line(&self) -> usize;
     fn current_offset(&self) -> usize;
-    /// At end of buffer
-    fn eof(&mut self) -> bool {
-        self.peek1().is_some()
-    }
 }
 
 pub struct StringBuffer<'input> {

diff --git a/lib/src/parser/default_impl/mod.rs b/lib/src/parser/default_impl/mod.rs
@@ -90,6 +90,19 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
         r
     }
 
+    /// test has next token
+    fn has_next(&mut self) -> bool {
+        // fill buff
+        while self.next >= self.tokens.len() {
+            match self.lexer.next() {
+                Some(Ok(item)) => self.tokens.push(item),
+                _ => return false,
+            }
+        }
+
+        true
+    }
+
     #[inline]
     fn next_token(&mut self) -> Result<&Token, ParseError> {
         match self.next()? {
@@ -122,6 +135,7 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
 
     /// parse a declaration
     fn parse_declaration(&mut self) -> Result<Declaration, ParseError> {
+        let pos = self.next;
         let except_token = self.except_one_of(&[
             TokenKind::Type,
             TokenKind::Function,
@@ -132,6 +146,7 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
         match except_token.kind.clone() {
             // pure global variables declare
             TokenKind::VarGlobal => {
+                self.next = pos;
                 let global_vars = self
                     .parse_global_variable_declare_factor()?
                     .unwrap_or(smallvec![]);
@@ -211,9 +226,11 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
         let token = self.next_token()?;
 
         match &token.kind {
+            TokenKind::Bit => Ok(Some(BitType::new_type())),
+            TokenKind::Bool => Ok(Some(BoolType::new_type())),
             TokenKind::Byte => Ok(Some(ByteType::new_type())),
             TokenKind::Int => Ok(Some(IntType::new_type())),
-            TokenKind::Bool => Ok(Some(BoolType::new_type())),
+            TokenKind::Real => Ok(Some(RealType::new_type())),
             TokenKind::Identifier(ident) => Ok(Some(UserType::from_name(ident.clone()).into())),
             _ => {
                 self.next = pos;
@@ -303,10 +320,19 @@ impl<I: Iterator<Item = LexerResult>> DefaultParserImpl<I> {
 
     fn parse_global_variable_declare_factor(&mut self) -> ParseResult<SmallVec8<Rc<Variable>>> {
         let mut v = smallvec![];
-        while let Some(mut x) = self.parse_global_variable_group()? {
-            v.append(&mut x);
+        loop {
+            if !self.has_next() {
+                break;
+            }
+
+            if let Some(mut x) = self.parse_global_variable_group()? {
+                v.append(&mut x);
+            } else {
+                panic!()
+            }
         }
 
+        // TODO: should be fail if 'v' is empty
         Ok(Some(v))
     }
 

diff --git a/lib/src/parser/lexer.rs b/lib/src/parser/lexer.rs
@@ -226,8 +226,14 @@ impl StLexerBuilder {
 }
 
 impl<'input> StLexer<'input> {
-    fn eof(&mut self) -> bool {
-        self.buffer.eof()
+    pub fn eof(&mut self) -> bool {
+        loop {
+            match self.buffer.peek1() {
+                Some(' ') | Some('\r') | Some('\n') => self.buffer.consume1(),
+                None => return true,
+                _ => return false,
+            }
+        }
     }
 
     fn parse_number(&mut self, mut tok: Token, ch: char) -> Option<LexerResult> {

diff --git a/lib/src/parser/mod.rs b/lib/src/parser/mod.rs
@@ -59,15 +59,14 @@ impl From<lalrpop_util::ParseError<usize, TokenKind, LexicalError>> for ParseErr
     }
 }
 
-pub struct Parser<'a, T: ParserTrait> {
+pub struct Parser<T: ParserTrait> {
     inner: T,
-    lexer: Option<StLexer<'a>>,
 }
 
-impl<T: ParserTrait> Parser<'_, T> {
+impl<T: ParserTrait> Parser<T> {
     #[inline]
-    pub fn parse(&self, mut lexer: StLexer) -> Result<Declaration, ParseError> {
-        self.inner.parse_decl(&mut lexer)
+    pub fn parse(&self, lexer: &mut StLexer) -> Result<Declaration, ParseError> {
+        self.inner.parse_decl(lexer)
     }
 
     #[inline]
@@ -95,18 +94,16 @@ pub struct ParserBuilder {}
 
 impl ParserBuilder {
     #[cfg(feature = "lalrpop_parser")]
-    pub fn build(self) -> Parser<'static, LalrpopParser> {
+    pub fn build(self) -> Parser<LalrpopParser> {
         Parser {
             inner: LalrpopParser::new(),
-            lexer: None,
         }
     }
 
     #[cfg(not(feature = "lalrpop_parser"))]
-    pub fn build(self) -> Parser<'static, DefaultParser> {
+    pub fn build(self) -> Parser<DefaultParser> {
         Parser {
             inner: DefaultParser::new(),
-            lexer: None,
         }
     }
 }

diff --git a/lib/src/test/mod.rs b/lib/src/test/mod.rs
@@ -10,9 +10,9 @@ fn test_decl_parse() {
         let f = entry.unwrap().path();
         let code = fs::read_to_string(&f).unwrap();
 
-        let lexer = StLexerBuilder::new().build_str(&code);
-        match parser.parse(lexer) {
-            Ok(_) => {}
+        let mut lexer = StLexerBuilder::new().build_str(&code);
+        match parser.parse(&mut lexer) {
+            Ok(_) => assert!(lexer.eof(), "{}", f.display()),
             Err(e) => panic!("{}: {:?}", f.display(), e),
         }
     }
@@ -53,13 +53,17 @@ fn test_scope_lookup() {
     mgr.write().set_active_application(Some(app_id));
 
     let app_ctx = mgr.write().get_context(app_id).unwrap();
-    let global = StLexerBuilder::new().build_str("VAR_GLOBAL END_VAR VAR_GLOBAL g1: REAL; END_VAR");
-    let global = ParserBuilder::default().build().parse(global).unwrap();
+    let mut global =
+        StLexerBuilder::new().build_str("VAR_GLOBAL END_VAR VAR_GLOBAL g1: REAL; END_VAR");
+    let global = ParserBuilder::default().build().parse(&mut global).unwrap();
     let _global_id = app_ctx.write().add_declaration(global, Uuid::nil());
 
-    let test_func =
+    let mut test_func =
         StLexerBuilder::new().build_str("program prg: int VAR g1: BYTE; END_VAR end_program");
-    let test_fun_decl = ParserBuilder::default().build().parse(test_func).unwrap();
+    let test_fun_decl = ParserBuilder::default()
+        .build()
+        .parse(&mut test_func)
+        .unwrap();
     let test_fun_decl_id = app_ctx.write().add_declaration(test_fun_decl, Uuid::nil());
 
     let scope = Scope::new(Some(mgr.clone()), Some(app_id), Some(test_fun_decl_id));

diff --git a/lib/src/test_file.rs b/lib/src/test_file.rs
@@ -1,11 +1,11 @@
 use stc::parser::{ParserBuilder, StLexerBuilder};
 
 fn main() {
-    let f = "lib/src/test/test_decl_parse/test_parse_prg.st";
-    let lexer = StLexerBuilder::new().build_file(&f).unwrap();
+    let f = "lib/src/test/test_decl_parse/test_base_types.st";
+    let mut lexer = StLexerBuilder::new().build_file(f).unwrap();
 
     let parser = ParserBuilder::default().build();
-    match parser.parse(lexer) {
+    match parser.parse(&mut lexer) {
         Ok(r) => {
             println!("{:?}", r);
         }

diff --git a/viewer/src/storage.rs b/viewer/src/storage.rs
@@ -36,8 +36,8 @@ impl From<Application> for ModuleContext {
         let mut ctx_write = ctx.write();
 
         for pou in app.pou_list.pou {
-            let lexer = StLexerBuilder::new().build_str(&pou.interface.content);
-            let decl = ParserBuilder::default().build().parse(lexer).unwrap();
+            let mut lexer = StLexerBuilder::new().build_str(&pou.interface.content);
+            let decl = ParserBuilder::default().build().parse(&mut lexer).unwrap();
             let func = ctx_write.add_declaration(
                 decl,
                 pou.uuid_text