From 4b8fc23840e52a81f1c62c48e4e83d04b700b392 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9ctor=20Ram=C3=B3n=20Jim=C3=A9nez?= Date: Fri, 31 Jan 2025 20:37:07 +0100 Subject: [PATCH] Implement `markdown` incremental code highlighting --- examples/markdown/src/main.rs | 81 +++++++++------- highlighter/src/lib.rs | 112 +++++++++++++++++----- widget/src/markdown.rs | 175 +++++++++++++++++++++++++--------- 3 files changed, 264 insertions(+), 104 deletions(-) diff --git a/examples/markdown/src/main.rs b/examples/markdown/src/main.rs index a55e91d2ab..2361b7b770 100644 --- a/examples/markdown/src/main.rs +++ b/examples/markdown/src/main.rs @@ -19,7 +19,7 @@ struct Markdown { } enum Mode { - Oneshot(Vec), + Preview(Vec), Stream { pending: String, parsed: markdown::Content, @@ -43,14 +43,14 @@ impl Markdown { ( Self { content: text_editor::Content::with_text(INITIAL_CONTENT), - mode: Mode::Oneshot(markdown::parse(INITIAL_CONTENT).collect()), + mode: Mode::Preview(markdown::parse(INITIAL_CONTENT).collect()), theme, }, widget::focus_next(), ) } - fn update(&mut self, message: Message) { + fn update(&mut self, message: Message) -> Task { match message { Message::Edit(action) => { let is_edit = action.is_edit(); @@ -58,48 +58,57 @@ impl Markdown { self.content.perform(action); if is_edit { - self.mode = match self.mode { - Mode::Oneshot(_) => Mode::Oneshot( - markdown::parse(&self.content.text()).collect(), - ), - Mode::Stream { .. } => Mode::Stream { - pending: self.content.text(), - parsed: markdown::Content::parse(""), - }, - } + self.mode = Mode::Preview( + markdown::parse(&self.content.text()).collect(), + ); } + + Task::none() } Message::LinkClicked(link) => { let _ = open::that_in_background(link.to_string()); + + Task::none() } Message::ToggleStream(enable_stream) => { - self.mode = if enable_stream { - Mode::Stream { + if enable_stream { + self.mode = Mode::Stream { pending: self.content.text(), parsed: markdown::Content::parse(""), - } + }; + + scrollable::snap_to( + "preview", + scrollable::RelativeOffset::END, + ) } else { - Mode::Oneshot( + self.mode = Mode::Preview( markdown::parse(&self.content.text()).collect(), - ) - }; + ); + + Task::none() + } } - Message::NextToken => match &mut self.mode { - Mode::Oneshot(_) => {} - Mode::Stream { pending, parsed } => { - if pending.is_empty() { - self.mode = Mode::Oneshot(parsed.items().to_vec()); - } else { - let mut tokens = pending.split(' '); - - if let Some(token) = tokens.next() { - parsed.push_str(&format!("{token} ")); + Message::NextToken => { + match &mut self.mode { + Mode::Preview(_) => {} + Mode::Stream { pending, parsed } => { + if pending.is_empty() { + self.mode = Mode::Preview(parsed.items().to_vec()); + } else { + let mut tokens = pending.split(' '); + + if let Some(token) = tokens.next() { + parsed.push_str(&format!("{token} ")); + } + + *pending = tokens.collect::>().join(" "); } - - *pending = tokens.collect::>().join(" "); } } - }, + + Task::none() + } } } @@ -113,7 +122,7 @@ impl Markdown { .highlight("markdown", highlighter::Theme::Base16Ocean); let items = match &self.mode { - Mode::Oneshot(items) => items.as_slice(), + Mode::Preview(items) => items.as_slice(), Mode::Stream { parsed, .. } => parsed.items(), }; @@ -127,7 +136,11 @@ impl Markdown { row![ editor, hover( - scrollable(preview).spacing(10).width(Fill).height(Fill), + scrollable(preview) + .spacing(10) + .width(Fill) + .height(Fill) + .id("preview"), right( toggler(matches!(self.mode, Mode::Stream { .. })) .label("Stream") @@ -147,7 +160,7 @@ impl Markdown { fn subscription(&self) -> Subscription { match self.mode { - Mode::Oneshot(_) => Subscription::none(), + Mode::Preview(_) => Subscription::none(), Mode::Stream { .. } => { time::every(milliseconds(20)).map(|_| Message::NextToken) } diff --git a/highlighter/src/lib.rs b/highlighter/src/lib.rs index d2abc6b12a..2d0ac2e434 100644 --- a/highlighter/src/lib.rs +++ b/highlighter/src/lib.rs @@ -7,6 +7,7 @@ use crate::core::Color; use std::ops::Range; use std::sync::LazyLock; + use syntect::highlighting; use syntect::parsing; @@ -104,30 +105,7 @@ impl highlighter::Highlighter for Highlighter { let ops = parser.parse_line(line, &SYNTAXES).unwrap_or_default(); - let highlighter = &self.highlighter; - - Box::new( - ScopeRangeIterator { - ops, - line_length: line.len(), - index: 0, - last_str_index: 0, - } - .filter_map(move |(range, scope)| { - let _ = stack.apply(&scope); - - if range.is_empty() { - None - } else { - Some(( - range, - Highlight( - highlighter.style_mod_for_stack(&stack.scopes), - ), - )) - } - }), - ) + Box::new(scope_iterator(ops, line, stack, &self.highlighter)) } fn current_line(&self) -> usize { @@ -135,6 +113,92 @@ impl highlighter::Highlighter for Highlighter { } } +fn scope_iterator<'a>( + ops: Vec<(usize, parsing::ScopeStackOp)>, + line: &str, + stack: &'a mut parsing::ScopeStack, + highlighter: &'a highlighting::Highlighter<'static>, +) -> impl Iterator, Highlight)> + 'a { + ScopeRangeIterator { + ops, + line_length: line.len(), + index: 0, + last_str_index: 0, + } + .filter_map(move |(range, scope)| { + let _ = stack.apply(&scope); + + if range.is_empty() { + None + } else { + Some(( + range, + Highlight(highlighter.style_mod_for_stack(&stack.scopes)), + )) + } + }) +} + +/// A streaming syntax highlighter. +/// +/// It can efficiently highlight an immutable stream of tokens. +#[derive(Debug)] +pub struct Stream { + syntax: &'static parsing::SyntaxReference, + highlighter: highlighting::Highlighter<'static>, + commit: (parsing::ParseState, parsing::ScopeStack), + state: parsing::ParseState, + stack: parsing::ScopeStack, +} + +impl Stream { + /// Creates a new [`Stream`] highlighter. + pub fn new(settings: &Settings) -> Self { + let syntax = SYNTAXES + .find_syntax_by_token(&settings.token) + .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text()); + + let highlighter = highlighting::Highlighter::new( + &THEMES.themes[settings.theme.key()], + ); + + let state = parsing::ParseState::new(syntax); + let stack = parsing::ScopeStack::new(); + + Self { + syntax, + highlighter, + commit: (state.clone(), stack.clone()), + state, + stack, + } + } + + /// Highlights the given line from the last commit. + pub fn highlight_line( + &mut self, + line: &str, + ) -> impl Iterator, Highlight)> + '_ { + self.state = self.commit.0.clone(); + self.stack = self.commit.1.clone(); + + let ops = self.state.parse_line(line, &SYNTAXES).unwrap_or_default(); + scope_iterator(ops, line, &mut self.stack, &self.highlighter) + } + + /// Commits the last highlighted line. + pub fn commit(&mut self) { + self.commit = (self.state.clone(), self.stack.clone()); + } + + /// Resets the [`Stream`] highlighter. + pub fn reset(&mut self) { + self.state = parsing::ParseState::new(self.syntax); + self.stack = parsing::ScopeStack::new(); + self.commit = (self.state.clone(), self.stack.clone()); + } +} + /// The settings of a [`Highlighter`]. #[derive(Debug, Clone, PartialEq)] pub struct Settings { diff --git a/widget/src/markdown.rs b/widget/src/markdown.rs index 0365dee8a0..7f6965e59c 100644 --- a/widget/src/markdown.rs +++ b/widget/src/markdown.rs @@ -57,6 +57,7 @@ use crate::core::{ }; use crate::{column, container, rich_text, row, scrollable, span, text}; +use std::borrow::BorrowMut; use std::cell::{Cell, RefCell}; use std::ops::Range; use std::sync::Arc; @@ -65,7 +66,7 @@ pub use core::text::Highlight; pub use pulldown_cmark::HeadingLevel; pub use url::Url; -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Content { items: Vec, state: State, @@ -80,6 +81,10 @@ impl Content { } pub fn push_str(&mut self, markdown: &str) { + if markdown.is_empty() { + return; + } + // Append to last leftover text let mut leftover = std::mem::take(&mut self.state.leftover); leftover.push_str(markdown); @@ -90,8 +95,6 @@ impl Content { // Re-parse last item and new text let new_items = parse_with(&mut self.state, &leftover); self.items.extend(new_items); - - dbg!(&self.state); } pub fn items(&self) -> &[Item] { @@ -271,19 +274,91 @@ pub fn parse(markdown: &str) -> impl Iterator + '_ { parse_with(State::default(), markdown) } -#[derive(Debug, Clone, Default)] -pub struct State { +#[derive(Debug, Default)] +struct State { leftover: String, + #[cfg(feature = "highlighter")] + highlighter: Option, +} + +#[cfg(feature = "highlighter")] +#[derive(Debug)] +struct Highlighter { + lines: Vec<(String, Vec)>, + parser: iced_highlighter::Stream, + current: usize, } -impl AsMut for State { - fn as_mut(&mut self) -> &mut Self { - self +#[cfg(feature = "highlighter")] +impl Highlighter { + pub fn new(language: &str) -> Self { + Self { + lines: Vec::new(), + parser: iced_highlighter::Stream::new( + &iced_highlighter::Settings { + theme: iced_highlighter::Theme::Base16Ocean, + token: language.to_string(), + }, + ), + current: 0, + } + } + + pub fn prepare(&mut self) { + self.current = 0; + } + + pub fn highlight_line(&mut self, text: &str) -> &[Span] { + match self.lines.get(self.current) { + Some(line) if line.0 == text => {} + _ => { + if self.current + 1 < self.lines.len() { + println!("Resetting..."); + self.parser.reset(); + self.lines.truncate(self.current); + + for line in &self.lines { + println!("Refeeding {n} lines", n = self.lines.len()); + + let _ = self.parser.highlight_line(&line.0); + } + } + + println!("Parsing: {text}", text = text.trim_end()); + if self.current + 1 < self.lines.len() { + self.parser.commit(); + } + + let mut spans = Vec::new(); + + for (range, highlight) in self.parser.highlight_line(text) { + spans.push(Span::Highlight { + text: text[range].to_owned(), + color: highlight.color(), + font: highlight.font(), + }); + } + + if self.current + 1 == self.lines.len() { + let _ = self.lines.pop(); + } + + self.lines.push((text.to_owned(), spans)); + } + } + + self.current += 1; + + &self + .lines + .get(self.current - 1) + .expect("Line must be parsed") + .1 } } fn parse_with<'a>( - mut state: impl AsMut + 'a, + mut state: impl BorrowMut + 'a, markdown: &'a str, ) -> impl Iterator + 'a { struct List { @@ -312,24 +387,26 @@ fn parse_with<'a>( ) .into_offset_iter(); - let mut produce = - move |lists: &mut Vec, item, source: Range| { - if lists.is_empty() { - state.as_mut().leftover = markdown[source.start..].to_owned(); - - Some(item) - } else { - lists - .last_mut() - .expect("list context") - .items - .last_mut() - .expect("item context") - .push(item); + let produce = move |state: &mut State, + lists: &mut Vec, + item, + source: Range| { + if lists.is_empty() { + state.leftover = markdown[source.start..].to_owned(); + + Some(item) + } else { + lists + .last_mut() + .expect("list context") + .items + .last_mut() + .expect("item context") + .push(item); - None - } - }; + None + } + }; // We want to keep the `spans` capacity #[allow(clippy::drain_collect)] @@ -367,6 +444,7 @@ fn parse_with<'a>( None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -393,20 +471,24 @@ fn parse_with<'a>( ) if !metadata && !table => { #[cfg(feature = "highlighter")] { - use iced_highlighter::Highlighter; - use text::Highlighter as _; - - highlighter = - Some(Highlighter::new(&iced_highlighter::Settings { - theme: iced_highlighter::Theme::Base16Ocean, - token: _language.to_string(), - })); + highlighter = Some({ + let mut highlighter = state + .borrow_mut() + .highlighter + .take() + .unwrap_or_else(|| Highlighter::new(&_language)); + + highlighter.prepare(); + + highlighter + }); } let prev = if spans.is_empty() { None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -428,6 +510,7 @@ fn parse_with<'a>( pulldown_cmark::Event::End(tag) => match tag { pulldown_cmark::TagEnd::Heading(level) if !metadata && !table => { produce( + state.borrow_mut(), &mut lists, Item::Heading(level, Text::new(spans.drain(..).collect())), source, @@ -451,6 +534,7 @@ fn parse_with<'a>( } pulldown_cmark::TagEnd::Paragraph if !metadata && !table => { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -461,6 +545,7 @@ fn parse_with<'a>( None } else { produce( + state.borrow_mut(), &mut lists, Item::Paragraph(Text::new(spans.drain(..).collect())), source, @@ -471,6 +556,7 @@ fn parse_with<'a>( let list = lists.pop().expect("list context"); produce( + state.borrow_mut(), &mut lists, Item::List { start: list.start, @@ -482,10 +568,11 @@ fn parse_with<'a>( pulldown_cmark::TagEnd::CodeBlock if !metadata && !table => { #[cfg(feature = "highlighter")] { - highlighter = None; + state.borrow_mut().highlighter = highlighter.take(); } produce( + state.borrow_mut(), &mut lists, Item::CodeBlock(Text::new(spans.drain(..).collect())), source, @@ -504,20 +591,16 @@ fn parse_with<'a>( pulldown_cmark::Event::Text(text) if !metadata && !table => { #[cfg(feature = "highlighter")] if let Some(highlighter) = &mut highlighter { - use text::Highlighter as _; + let start = std::time::Instant::now(); - for (range, highlight) in - highlighter.highlight_line(text.as_ref()) - { - let span = Span::Highlight { - text: text[range].to_owned(), - color: highlight.color(), - font: highlight.font(), - }; - - spans.push(span); + for line in text.lines() { + spans.extend_from_slice( + highlighter.highlight_line(&format!("{line}\n")), + ); } + dbg!(start.elapsed()); + return None; }