From 49da1445fe1d9d88eec56a0cc526fe3089333b05 Mon Sep 17 00:00:00 2001 From: Lucas Satabin Date: Mon, 29 Jan 2024 20:22:13 +0100 Subject: [PATCH] Add renderer for JSON --- .../main/scala/fs2/data/json/package.scala | 24 +- .../src/main/scala/fs2/data/json/tokens.scala | 300 +++++++++++++++++- .../test/scala/fs2/data/json/RenderSpec.scala | 67 +++- .../text/render/internal/StreamPrinter.scala | 13 +- .../scala/fs2/data/text/render/package.scala | 3 +- 5 files changed, 383 insertions(+), 24 deletions(-) diff --git a/json/src/main/scala/fs2/data/json/package.scala b/json/src/main/scala/fs2/data/json/package.scala index cf5f05e2..308df4ba 100644 --- a/json/src/main/scala/fs2/data/json/package.scala +++ b/json/src/main/scala/fs2/data/json/package.scala @@ -22,6 +22,7 @@ import json.ast._ import json.internals._ import cats._ +import fs2.data.text.render.Renderable /** Handles stream parsing and traversing of json documents. */ @@ -153,7 +154,7 @@ package object json { * You can use this to write the Json stream to a file. */ def compact[F[_]]: Pipe[F, Token, String] = - Renderer.pipe[F](false, "") + _.through(fs2.data.text.render.pretty(width = Int.MaxValue)(Token.compact)) /** Renders a pretty-printed representation of the token stream with the given * indentation size. @@ -163,8 +164,25 @@ package object json { * * You can use this to write the Json stream to a file. */ + @deprecated(since = "fs2-data 1.11.0", message = "Consider using `fs2.data.json.render.prettyPrint` instead.") def pretty[F[_]](indent: String = " "): Pipe[F, Token, String] = - Renderer.pipe[F](true, indent) + _.through(fs2.data.text.render.pretty(width = 0)(Token.compact)) + + /** Renders a pretty-printed representation of the token stream with the given + * indentation size and page width. + * + * Chunks can be concatenated to render all values in the stream, + * separated by new lines. + * + * You can use this to write the Json stream to a file. + * + * You can configure how the stream is rendered by providing an instance of + * [[fs2.data.text.render.Renderable Renderable]] in scope. A default one is automatically + * provided if you do not need specific formatting. + */ + def prettyPrint[F[_]](width: Int = 100, indent: Int = 2)(implicit + renderable: Renderable[Token]): Pipe[F, Token, String] = + _.through(fs2.data.text.render.pretty(width = width, indent = indent)) } @@ -187,6 +205,8 @@ package object json { * * Top-level values are separated by new lines. */ + @deprecated(since = "fs2-data 1.11.0", + message = "Consider using `.through(fs2.data.json.render.prettyPrint()).compile.string` instead.") def pretty(indent: String = " "): Collector.Aux[Token, String] = new Collector[Token] { type Out = String diff --git a/json/src/main/scala/fs2/data/json/tokens.scala b/json/src/main/scala/fs2/data/json/tokens.scala index 42a9006b..05ecdf62 100644 --- a/json/src/main/scala/fs2/data/json/tokens.scala +++ b/json/src/main/scala/fs2/data/json/tokens.scala @@ -18,9 +18,16 @@ package fs2 package data package json +import fs2.Pure +import fs2.data.text.render.{DocEvent, Renderable, Renderer} + +import scala.annotation.switch +import scala.annotation.tailrec + sealed abstract class Token(val kind: String) { def jsonRepr: String } + object Token { case object StartObject extends Token("object") { @@ -55,7 +62,298 @@ object Token { def jsonRepr: String = value } case class StringValue(value: String) extends Token("string") { - def jsonRepr: String = s""""$value"""" + def jsonRepr: String = { + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + rendered.result() + } + } + + private final val hex = "0123456789abcdef" + + @tailrec + def renderString(s: String, idx: Int, builder: StringBuilder): Unit = + if (idx < s.length) { + val nextEscape = s.indexWhere(c => c > 127 || Character.isISOControl(c) || "\\/\b\f\n\r\t\"".contains(c), idx) + if (nextEscape >= 0) { + if (nextEscape > 0) { + builder.append(s.substring(idx, nextEscape)) + } + val c = s(nextEscape) + (c: @switch) match { + case '\\' => + builder.append("\\\\") + case '/' => + builder.append("\\/") + case '\b' => + builder.append("\\b") + case '\f' => + builder.append("\\f") + case '\n' => + builder.append("\\n") + case '\r' => + builder.append("\\r") + case '\t' => + builder.append("\\t") + case '"' => + builder.append("\\\"") + case _ => + // escape non ascii or control characters + builder + .append("\\u") + .append(hex((c >> 12) & 0x0f)) + .append(hex((c >> 8) & 0x0f)) + .append(hex((c >> 4) & 0x0f)) + .append(hex(c & 0x0f)) + } + renderString(s, nextEscape + 1, builder) + } else { + // append the rest of the string and we are done + builder.append(s.substring(idx)) + } + } + + private val nullValue = + Stream.emit(DocEvent.Text("null")) + + private val trueValue = + Stream.emit(DocEvent.Text("true")) + + private val falseValue = + Stream.emit(DocEvent.Text("false")) + + private final val FirstObjectKey = 0 + private final val ObjectKey = 1 + private final val ObjectValue = 2 + private final val FirstArrayValue = 3 + private final val ArrayValue = 4 + + implicit object renderable extends Renderable[Token] { + + private val startObject = + Stream.emits(DocEvent.GroupBegin :: DocEvent.Text("{") :: DocEvent.IndentBegin :: DocEvent.LineBreak :: Nil) + + private val endEmptyObject = + Stream.emits(DocEvent.IndentEnd :: DocEvent.LineBreak :: DocEvent.Text("}") :: DocEvent.GroupEnd :: Nil) + + private val endObject = + Stream.emits(DocEvent.GroupEnd :: Nil) ++ endEmptyObject + + private val startArray = + Stream.emits(DocEvent.GroupBegin :: DocEvent.Text("[") :: DocEvent.IndentBegin :: DocEvent.LineBreak :: Nil) + + private val endEmptyArray = + Stream.emits(DocEvent.IndentEnd :: DocEvent.LineBreak :: DocEvent.Text("]") :: DocEvent.GroupEnd :: Nil) + + private val endArray = + Stream.emit(DocEvent.GroupEnd) ++ endEmptyArray + + private val objectSep = + Stream.emits(DocEvent.Text(",") :: DocEvent.GroupEnd :: DocEvent.Line :: Nil) + + private val arraySep = + Stream.emits(DocEvent.Text(",") :: DocEvent.GroupEnd :: DocEvent.Line :: DocEvent.GroupBegin :: Nil) + + override def newRenderer(): Renderer[Token] = new Renderer[Token] { + + // the current stack of states, helping to deal with comma and indentation + // states are described right above + private[this] var states = List.empty[Int] + + private def separator(): Stream[Pure, DocEvent] = + states match { + case Nil => + Stream.empty + case state :: rest => + (state: @switch) match { + case FirstObjectKey => + states = ObjectValue :: rest + Stream.empty + case ObjectKey => + states = ObjectValue :: rest + objectSep + case ObjectValue => + states = ObjectKey :: rest + Stream.emit(DocEvent.GroupBegin) + case FirstArrayValue => + states = ArrayValue :: rest + Stream.emit(DocEvent.GroupBegin) + case ArrayValue => + states = ArrayValue :: rest + arraySep + } + } + + private def closeObject(): Stream[Pure, DocEvent] = + states match { + case Nil => endEmptyObject + case state :: rest => + states = rest + (state: @switch) match { + case FirstObjectKey => endEmptyObject + case ObjectKey => endObject + } + } + + private def closeArray(): Stream[Pure, DocEvent] = + states match { + case Nil => endEmptyArray + case state :: rest => + states = rest + (state: @switch) match { + case FirstArrayValue => endEmptyArray + case ArrayValue => endArray + } + } + + override def doc(token: Token): Stream[Pure, DocEvent] = + token match { + case StartObject => + val res = separator() ++ startObject + states = FirstObjectKey :: states + res + case EndObject => + closeObject() + case StartArray => + val res = separator() ++ startArray + states = FirstArrayValue :: states + res + case EndArray => + closeArray() + case Key(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append("\": ") + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + case NullValue => + separator() ++ nullValue + case TrueValue => + separator() ++ trueValue + case FalseValue => + separator() ++ falseValue + case NumberValue(value) => + separator() ++ Stream.emit(DocEvent.Text(value)) + case StringValue(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + } + } + + } + + /** A compact version of the JSON rendering with no space or new lines. */ + object compact extends Renderable[Token] { + + private val startObject = + Stream.emit(DocEvent.Text("{")) + + private val endObject = + Stream.emit(DocEvent.Text("}")) + + private val startArray = + Stream.emit(DocEvent.Text("[")) + + private val endArray = + Stream.emit(DocEvent.Text("]")) + + private val sep = + Stream.emit(DocEvent.Text(",")) + + override def newRenderer(): Renderer[Token] = new Renderer[Token] { + + // the current stack of states, helping to deal with comma and indentation + // states are described right above + private[this] var states = List.empty[Int] + + private def separator(): Stream[Pure, DocEvent] = + states match { + case Nil => + Stream.empty + case state :: rest => + (state: @switch) match { + case FirstObjectKey => + states = ObjectValue :: rest + Stream.empty + case ObjectKey => + states = ObjectValue :: rest + sep + case ObjectValue => + states = ObjectKey :: rest + Stream.empty + case FirstArrayValue => + states = ArrayValue :: rest + Stream.empty + case ArrayValue => + states = ArrayValue :: rest + sep + } + } + + private def closeObject(): Stream[Pure, DocEvent] = + states match { + case Nil => endObject + case state :: rest => + states = rest + (state: @switch) match { + case FirstObjectKey => endObject + case ObjectKey => endObject + } + } + + private def closeArray(): Stream[Pure, DocEvent] = + states match { + case Nil => endArray + case state :: rest => + states = rest + (state: @switch) match { + case FirstArrayValue => endArray + case ArrayValue => endArray + } + } + + override def doc(token: Token): Stream[Pure, DocEvent] = + token match { + case StartObject => + val res = separator() ++ startObject + states = FirstObjectKey :: states + res + case EndObject => + closeObject() + case StartArray => + val res = separator() ++ startArray + states = FirstArrayValue :: states + res + case EndArray => + closeArray() + case Key(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append("\":") + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + case NullValue => + separator() ++ nullValue + case TrueValue => + separator() ++ trueValue + case FalseValue => + separator() ++ falseValue + case NumberValue(value) => + separator() ++ Stream.emit(DocEvent.Text(value)) + case StringValue(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + } + } + } } diff --git a/json/src/test/scala/fs2/data/json/RenderSpec.scala b/json/src/test/scala/fs2/data/json/RenderSpec.scala index ae7a721d..31852584 100644 --- a/json/src/test/scala/fs2/data/json/RenderSpec.scala +++ b/json/src/test/scala/fs2/data/json/RenderSpec.scala @@ -16,10 +16,7 @@ package fs2.data.json -import internals._ - import fs2._ - import weaver._ object RenderSpec extends SimpleIOSuite { @@ -30,7 +27,7 @@ object RenderSpec extends SimpleIOSuite { val toks = input.through(tokens[Fallible, Char]) - val roundtrip = toks.through(render.compact).flatMap(Stream.emits(_)).through(tokens) + val roundtrip = toks.through(render.compact).through(tokens) expect(toks.compile.toList == roundtrip.compile.toList) @@ -42,20 +39,72 @@ object RenderSpec extends SimpleIOSuite { val toks = input.through(tokens[Fallible, Char]) - val roundtrip = toks.through(render.pretty()).flatMap(Stream.emits(_)).through(tokens) + val roundtrip = toks.through(render.prettyPrint()).through(tokens) expect(toks.compile.toList == roundtrip.compile.toList) } pureTest("a Renderer should properly escape what needs to be escaped") { - val renderer = new Renderer(true, true, "") + val input = Stream.emit(Token.StringValue("some\ncharacters must\\be\"escaped\" like ß")).covaryOutput[Token] + + expect.same("\"some\\ncharacters must\\\\be\\\"escaped\\\" like \\u00df\"", + input.through(render.prettyPrint()).compile.string) + + } + + pureTest("An object should be properly pretty renderer with line width of 10") { + val input = Stream.emits("""{"field1": "test", "field2": [23, [true, null]]}""") + + expect.same( + Right("""{ + | "field1": "test", + | "field2": [ + | 23, + | [ + | true, + | null + | ] + | ] + |}""".stripMargin), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 10, indent = 2)).compile.string + ) + + } + + pureTest("An object should be properly pretty renderer with line width of 32") { + val input = Stream.emits("""{"field1": "test", "field2": [23, [true, null]]}""") - renderer += Chunk.singleton(Token.StringValue("some\ncharacters must\\be\"escaped\" like ß")) + expect.same( + Right("""{ + | "field1": "test", + | "field2": [23, [true, null]] + |}""".stripMargin), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 32, indent = 2)).compile.string + ) + + } + + pureTest("An object should be properly pretty renderer with line width of 80") { + val input = Stream.emits("""{ + | "field1": "test", + | "field2": [23, [true, null]]}""".stripMargin) + + expect.same( + Right("""{"field1": "test", "field2": [23, [true, null]]}"""), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 80, indent = 2)).compile.string + ) + } - val res = renderer.result + pureTest("An object should be properly pretty renderer with line width of 80") { + val input = Stream.emits("""{ + | "field1": "test", + | "field2": [23, [true, null]]}""".stripMargin) - expect(res == "\"some\\ncharacters must\\\\be\\\"escaped\\\" like \\u00df\"") + expect.same( + Right("""{"field1": "test", "field2": [23, [true, null]]}"""), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 80, indent = 2)).compile.string + ) } diff --git a/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala index 93319c0e..bb5a552a 100644 --- a/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala +++ b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala @@ -19,11 +19,9 @@ package internal import cats.collections.Dequeue import cats.data.{Chain, NonEmptyList} -import fs2.{Chunk, Pipe, Pull, RaiseThrowable, Stream} +import fs2.{Chunk, Pipe, Pull, Stream} -private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(implicit - F: RaiseThrowable[F], - render: Renderable[Event]) +private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(implicit render: Renderable[Event]) extends Pipe[F, Event, String] { private val emptyGroups = (0, Dequeue.empty[(Int, Chain[Annotated])]) @@ -74,12 +72,7 @@ private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(im if (idx >= chunk.size) { rest.pull.uncons.flatMap { case Some((hd, tl)) => annotate(hd, 0, tl, pos, aligns, hpl, groups) - case None => - if (!groups.isEmpty) { - Pull.raiseError(RenderException("Document stream malformed (unclosed group)")) - } else { - Pull.done - } + case None => Pull.done } } else { val evt = chunk(idx) diff --git a/text/shared/src/main/scala/fs2/data/text/render/package.scala b/text/shared/src/main/scala/fs2/data/text/render/package.scala index 4cdb82e0..317b2bef 100644 --- a/text/shared/src/main/scala/fs2/data/text/render/package.scala +++ b/text/shared/src/main/scala/fs2/data/text/render/package.scala @@ -16,13 +16,12 @@ package fs2.data.text +import fs2.Pipe import fs2.data.text.render.internal.StreamPrinter -import fs2.{Pipe, RaiseThrowable} package object render { def pretty[F[_], Event](width: Int = 100, indent: Int = 2)(implicit - F: RaiseThrowable[F], render: Renderable[Event]): Pipe[F, Event, String] = new StreamPrinter[F, Event](width, indent)