From ca7fb10d424ecf6e7a262c02f7da51e52de619bb Mon Sep 17 00:00:00 2001 From: Lucas Satabin Date: Mon, 29 Jan 2024 19:46:47 +0100 Subject: [PATCH] Add streaming pretty printer utilities It is inspired by the paper _Lazy v. Yield: Incremental, Linear Pretty-printing_ by Oleg Kiselyov, Simon Peyton-Jones, and Amr Sabry. It adds indentation and alignment features to be useful in the context of streaming tree structure pretty printing (XML, JSON, ...). Any type that has an instance of `Renderable` can be printed. The printer guarantees that the data is printed as early as possible and the it is correct up to the first malformed tree structure. --- build.sbt | 3 + .../scala/fs2/data/text/render/DocEvent.scala | 49 ++++ .../data/text/render/RenderException.scala | 19 ++ .../fs2/data/text/render/Renderable.scala | 28 ++ .../scala/fs2/data/text/render/Renderer.scala | 28 ++ .../data/text/render/internal/Annotated.scala | 30 +++ .../data/text/render/internal/Position.scala | 23 ++ .../text/render/internal/StreamPrinter.scala | 250 ++++++++++++++++++ .../scala/fs2/data/text/render/package.scala | 29 ++ 9 files changed, 459 insertions(+) create mode 100644 text/shared/src/main/scala/fs2/data/text/render/DocEvent.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/RenderException.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/Renderable.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/Renderer.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/internal/Annotated.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/internal/Position.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala create mode 100644 text/shared/src/main/scala/fs2/data/text/render/package.scala diff --git a/build.sbt b/build.sbt index 47d8411e..7946ff56 100644 --- a/build.sbt +++ b/build.sbt @@ -123,6 +123,9 @@ lazy val text = crossProject(JVMPlatform, JSPlatform, NativePlatform) .settings( name := "fs2-data-text", description := "Utilities for textual data format", + libraryDependencies ++= List( + "org.typelevel" %%% "cats-collections-core" % "0.9.8" + ), mimaBinaryIssueFilters ++= List( // private class ProblemFilters.exclude[IncompatibleResultTypeProblem]("fs2.data.text.CharLikeCharChunks.create"), diff --git a/text/shared/src/main/scala/fs2/data/text/render/DocEvent.scala b/text/shared/src/main/scala/fs2/data/text/render/DocEvent.scala new file mode 100644 index 00000000..15369d3f --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/DocEvent.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render + +sealed trait DocEvent + +object DocEvent { + + /** Renders the text _as is_. */ + case class Text(text: String) extends DocEvent + + /** Adds a new line and indent, or a space if undone by a group */ + case object Line extends DocEvent + + /** Adds a new line and indent, or a empty if undone by a group */ + case object LineBreak extends DocEvent + + /** Begins a new group */ + case object GroupBegin extends DocEvent + + /** Ends a group */ + case object GroupEnd extends DocEvent + + /** Begins a new indent, incrementing the current indentation level */ + case object IndentBegin extends DocEvent + + /** Ends an indent , decrementing the current indentation level */ + case object IndentEnd extends DocEvent + + /** Begins a new alignment, setting the current indentation level to the current column */ + case object AlignBegin extends DocEvent + + /** Ends an alignment, setting the current indentation level back to what it was before this alignment */ + case object AlignEnd extends DocEvent +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/RenderException.scala b/text/shared/src/main/scala/fs2/data/text/render/RenderException.scala new file mode 100644 index 00000000..9d6a25fc --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/RenderException.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render + +final case class RenderException(msg: String) extends Exception(msg) diff --git a/text/shared/src/main/scala/fs2/data/text/render/Renderable.scala b/text/shared/src/main/scala/fs2/data/text/render/Renderable.scala new file mode 100644 index 00000000..a7bc806b --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/Renderable.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render + +/** Typeclass indicating how to render a given event type. */ +trait Renderable[Event] { + + /** Creates a new instance of a renderer. + * This allows for renderer with mutable state. + * If the renderer has no state, it can return the same instance every time. + */ + def newRenderer(): Renderer[Event] + +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/Renderer.scala b/text/shared/src/main/scala/fs2/data/text/render/Renderer.scala new file mode 100644 index 00000000..3e087ee4 --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/Renderer.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render + +import fs2.{Stream, Pure} + +trait Renderer[Event] { + + /** Transforms the event into a stream of document events. + * The stream may be partial (e.g. opening a group when the event describes a new tree node). + */ + def doc(evt: Event): Stream[Pure, DocEvent] + +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/internal/Annotated.scala b/text/shared/src/main/scala/fs2/data/text/render/internal/Annotated.scala new file mode 100644 index 00000000..5a81353c --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/internal/Annotated.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render.internal + +private sealed trait Annotated +private object Annotated { + case class Text(text: String, hp: Int) extends Annotated + case class Line(hp: Int) extends Annotated + case class LineBreak(hp: Int) extends Annotated + case class GroupBegin(hpl: Position) extends Annotated + case class GroupEnd(hp: Int) extends Annotated + case class IndentBegin(hp: Int) extends Annotated + case class IndentEnd(hp: Int) extends Annotated + case class AlignBegin(hp: Int) extends Annotated + case class AlignEnd(hp: Int) extends Annotated +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/internal/Position.scala b/text/shared/src/main/scala/fs2/data/text/render/internal/Position.scala new file mode 100644 index 00000000..40cb6ce8 --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/internal/Position.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render.internal + +private sealed trait Position +private object Position { + case object TooFar extends Position + case class Small(pos: Int) extends Position +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala new file mode 100644 index 00000000..93319c0e --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala @@ -0,0 +1,250 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text.render +package internal + +import cats.collections.Dequeue +import cats.data.{Chain, NonEmptyList} +import fs2.{Chunk, Pipe, Pull, RaiseThrowable, Stream} + +private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(implicit + F: RaiseThrowable[F], + render: Renderable[Event]) + extends Pipe[F, Event, String] { + + private val emptyGroups = (0, Dequeue.empty[(Int, Chain[Annotated])]) + + private def push(groups: Dequeue[(Int, Chain[Annotated])], evt: Annotated): Dequeue[(Int, Chain[Annotated])] = + groups.unsnoc match { + case Some(((ghpl, group), groups)) => groups.snoc((ghpl, group.append(evt))) + case None => Dequeue.empty // should never happen + } + + private def pop(hpl: Int, + groups: Dequeue[(Int, Chain[Annotated])], + buffer: Chain[Annotated]): Pull[F, Annotated, (Int, Dequeue[(Int, Chain[Annotated])])] = + groups.unsnoc match { + case Some(((ghpl, group), groups)) => + Pull.pure((hpl, groups.snoc((ghpl, group.concat(buffer))))) + case None => + Pull.output(Chunk.chain(buffer)).as(emptyGroups) + } + + private def check(hpl: Int, + groups: Dequeue[(Int, Chain[Annotated])], + ghpl: Int): Pull[F, Annotated, (Int, Dequeue[(Int, Chain[Annotated])])] = + if (ghpl <= hpl && groups.size <= width) { + // groups still fits + Pull.pure((hpl, groups)) + } else { + // group does not fit, uncons first buffer + groups.uncons match { + case Some(((_, buffer), groups)) => + Pull.output(Chunk.chain(buffer.prepend(Annotated.GroupBegin(Position.TooFar)))) >> (groups.uncons match { + case Some(((hpl, _), _)) => check(hpl, groups, ghpl) // check inner groups recursively + case None => Pull.pure(emptyGroups) + }) + case None => + Pull.pure(emptyGroups) // should never happen + + } + } + + private def annotate(chunk: Chunk[DocEvent], + idx: Int, + rest: Stream[F, DocEvent], + pos: Int, + aligns: NonEmptyList[Int], + hpl: Int, + groups: Dequeue[(Int, Chain[Annotated])]): Pull[F, Annotated, Unit] = + if (idx >= chunk.size) { + rest.pull.uncons.flatMap { + case Some((hd, tl)) => annotate(hd, 0, tl, pos, aligns, hpl, groups) + case None => + if (!groups.isEmpty) { + Pull.raiseError(RenderException("Document stream malformed (unclosed group)")) + } else { + Pull.done + } + } + } else { + val evt = chunk(idx) + evt match { + case DocEvent.Text(text) => + val size = text.size + val pos1 = pos + size + if (groups.isEmpty) { + // no open group we can emit immediately + Pull.output1(Annotated.Text(text, pos1)) >> annotate(chunk, idx + 1, rest, pos1, aligns, hpl, groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.Text(text, pos1)), pos1).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos1, aligns, hpl, groups) + } + } + + case DocEvent.Line => + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.Line(pos + 1)) >> annotate(chunk, idx + 1, rest, pos + 1, aligns, hpl, groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.Line(pos + 1)), pos + 1).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos + 1, aligns, hpl, groups) + } + } + + case DocEvent.LineBreak => + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.LineBreak(pos)) >> annotate(chunk, idx + 1, rest, pos, aligns, hpl, groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.LineBreak(pos)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, aligns, hpl, groups) + } + } + + case DocEvent.GroupBegin => + val hpl1 = pos + width + aligns.head + if (groups.isEmpty) + // this is the top-level group, turn on the buffer mechanism + annotate(chunk, idx + 1, rest, pos, aligns, hpl1, groups.snoc((hpl1, Chain.empty))) + else + // starting a new group, puts a new empty buffer in the group dequeue, and check for overflow + check(hpl, groups.snoc((hpl1, Chain.empty)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, aligns, hpl, groups) + } + + case DocEvent.GroupEnd => + groups.unsnoc match { + case None => + // closing unknown group, just ignore it + annotate(chunk, idx + 1, rest, pos, aligns, hpl, groups) + + case Some(((_, group), groups)) => + // closing a group, pop it from the buffer dequeue, and continue + pop(hpl, groups, group.prepend(Annotated.GroupBegin(Position.Small(pos))).append(Annotated.GroupEnd(pos))) + .flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, aligns, hpl, groups) + } + + } + + case DocEvent.IndentBegin => + // increment the current indentation level + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.IndentBegin(pos)) >> annotate(chunk, + idx + 1, + rest, + pos, + NonEmptyList(aligns.head + 1, aligns.tail), + hpl, + groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.IndentBegin(pos)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, NonEmptyList(aligns.head + 1, aligns.tail), hpl, groups) + } + } + + case DocEvent.IndentEnd => + // decrement the current indentation level + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.IndentEnd(pos)) >> annotate(chunk, + idx + 1, + rest, + pos, + NonEmptyList(aligns.head - 1, aligns.tail), + hpl, + groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.IndentEnd(pos)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, NonEmptyList(aligns.head - 1, aligns.tail), hpl, groups) + } + } + + case DocEvent.AlignBegin => + // push new indentation level + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.AlignBegin(pos)) >> annotate(chunk, idx + 1, rest, pos, pos :: aligns, hpl, groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.AlignBegin(pos)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, pos :: aligns, hpl, groups) + } + } + + case DocEvent.AlignEnd => + // restore to previous indentation level + val aligns1 = + aligns match { + case NonEmptyList(_, i :: is) => NonEmptyList(i, is) + case NonEmptyList(_, Nil) => NonEmptyList.one(0) + } + if (groups.isEmpty) { + // no open group we can emit immediately a new line + Pull.output1(Annotated.AlignEnd(pos)) >> annotate(chunk, idx + 1, rest, pos, aligns1, hpl, groups) + } else { + // there is an open group, append the event to the current group + check(hpl, push(groups, Annotated.AlignEnd(pos)), pos).flatMap { case (hpl, groups) => + annotate(chunk, idx + 1, rest, pos, aligns1, hpl, groups) + } + } + } + } + + def apply(events: Stream[F, Event]): Stream[F, String] = + annotate( + Chunk.empty, + 0, + Stream.suspend(Stream.emit(render.newRenderer())).flatMap(renderer => events.flatMap(renderer.doc(_))), + 0, + NonEmptyList.one(0), + 0, + Dequeue.empty + ).stream + .mapAccumulate((0, width, NonEmptyList.one(""))) { case (acc @ (fit, hpl, lines), evt) => + evt match { + case Annotated.Text(text, _) => (acc, Some(text)) + case Annotated.Line(pos) if fit == 0 => ((fit, pos + width, lines), Some("\n" + lines.head)) + case Annotated.Line(_) => (acc, Some(" ")) + case Annotated.LineBreak(pos) if fit == 0 => ((fit, pos + width, lines), Some("\n" + lines.head)) + case Annotated.LineBreak(_) => (acc, None) + case Annotated.GroupBegin(Position.TooFar) if fit == 0 => ((0, hpl, lines), None) + case Annotated.GroupBegin(Position.Small(pos)) if fit == 0 => ((if (pos <= hpl) 1 else 0, hpl, lines), None) + case Annotated.GroupBegin(_) => ((fit + 1, hpl, lines), None) + case Annotated.GroupEnd(_) if fit == 0 => (acc, None) + case Annotated.GroupEnd(_) => ((fit - 1, hpl, lines), None) + case Annotated.IndentBegin(_) => + ((fit, hpl, NonEmptyList(lines.head + (" " * indentSize), lines.tail)), None) + case Annotated.IndentEnd(_) => + ((fit, hpl, NonEmptyList(lines.head.drop(indentSize), lines.tail)), None) + case Annotated.AlignBegin(pos) => + ((fit, hpl, (" " * pos) :: lines), None) + case Annotated.AlignEnd(_) => + ((fit, hpl, NonEmptyList.fromList(lines.tail).getOrElse(NonEmptyList.one(""))), None) + } + + } + .map(_._2) + .unNone +} diff --git a/text/shared/src/main/scala/fs2/data/text/render/package.scala b/text/shared/src/main/scala/fs2/data/text/render/package.scala new file mode 100644 index 00000000..4cdb82e0 --- /dev/null +++ b/text/shared/src/main/scala/fs2/data/text/render/package.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2024 fs2-data Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fs2.data.text + +import fs2.data.text.render.internal.StreamPrinter +import fs2.{Pipe, RaiseThrowable} + +package object render { + + def pretty[F[_], Event](width: Int = 100, indent: Int = 2)(implicit + F: RaiseThrowable[F], + render: Renderable[Event]): Pipe[F, Event, String] = + new StreamPrinter[F, Event](width, indent) + +}