diff --git a/examples/jqlike/src/main/scala/fs2/data/example/jqlike/JqLike.scala b/examples/jqlike/src/main/scala/fs2/data/example/jqlike/JqLike.scala index c4334fa9..7ca40612 100644 --- a/examples/jqlike/src/main/scala/fs2/data/example/jqlike/JqLike.scala +++ b/examples/jqlike/src/main/scala/fs2/data/example/jqlike/JqLike.scala @@ -60,7 +60,7 @@ object JqLike extends CommandIOApp(name = "fs2-jq", header = "A streaming implem // execute the compiled query on the input .through(compiled) // render the query result - .through(render.pretty()) + .through(render.prettyPrint()) // encode the result .through(fs2.text.utf8.encode[IO]) // and save it to the output diff --git a/json/src/main/scala/fs2/data/json/package.scala b/json/src/main/scala/fs2/data/json/package.scala index cf5f05e2..135ea595 100644 --- a/json/src/main/scala/fs2/data/json/package.scala +++ b/json/src/main/scala/fs2/data/json/package.scala @@ -22,6 +22,7 @@ import json.ast._ import json.internals._ import cats._ +import fs2.data.text.render.Renderable /** Handles stream parsing and traversing of json documents. */ @@ -153,7 +154,7 @@ package object json { * You can use this to write the Json stream to a file. */ def compact[F[_]]: Pipe[F, Token, String] = - Renderer.pipe[F](false, "") + _.through(fs2.data.text.render.pretty(width = Int.MaxValue)(Token.compact)) /** Renders a pretty-printed representation of the token stream with the given * indentation size. @@ -163,8 +164,25 @@ package object json { * * You can use this to write the Json stream to a file. */ + @deprecated(message = "Consider using `fs2.data.json.render.prettyPrint` instead.", since = "fs2-data 1.11.0") def pretty[F[_]](indent: String = " "): Pipe[F, Token, String] = - Renderer.pipe[F](true, indent) + _.through(fs2.data.text.render.pretty(width = 0)(Token.compact)) + + /** Renders a pretty-printed representation of the token stream with the given + * indentation size and page width. + * + * Chunks can be concatenated to render all values in the stream, + * separated by new lines. + * + * You can use this to write the Json stream to a file. + * + * You can configure how the stream is rendered by providing an instance of + * [[fs2.data.text.render.Renderable Renderable]] in scope. A default one is automatically + * provided if you do not need specific formatting. + */ + def prettyPrint[F[_]](width: Int = 100, indent: Int = 2)(implicit + renderable: Renderable[Token]): Pipe[F, Token, String] = + _.through(fs2.data.text.render.pretty(width = width, indent = indent)) } @@ -187,6 +205,8 @@ package object json { * * Top-level values are separated by new lines. */ + @deprecated(message = "Consider using `.through(fs2.data.json.render.prettyPrint()).compile.string` instead.", + since = "fs2-data 1.11.0") def pretty(indent: String = " "): Collector.Aux[Token, String] = new Collector[Token] { type Out = String diff --git a/json/src/main/scala/fs2/data/json/tokens.scala b/json/src/main/scala/fs2/data/json/tokens.scala index 42a9006b..4f253e32 100644 --- a/json/src/main/scala/fs2/data/json/tokens.scala +++ b/json/src/main/scala/fs2/data/json/tokens.scala @@ -18,9 +18,16 @@ package fs2 package data package json +import fs2.Pure +import fs2.data.text.render.{DocEvent, Renderable, Renderer} + +import scala.annotation.switch +import scala.annotation.tailrec + sealed abstract class Token(val kind: String) { def jsonRepr: String } + object Token { case object StartObject extends Token("object") { @@ -55,7 +62,298 @@ object Token { def jsonRepr: String = value } case class StringValue(value: String) extends Token("string") { - def jsonRepr: String = s""""$value"""" + def jsonRepr: String = { + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + rendered.result() + } + } + + private final val hex = "0123456789abcdef" + + @tailrec + def renderString(s: String, idx: Int, builder: StringBuilder): Unit = + if (idx < s.length) { + val nextEscape = s.indexWhere(c => c > 127 || Character.isISOControl(c) || "\\/\b\f\n\r\t\"".contains(c), idx) + if (nextEscape >= 0) { + if (nextEscape > 0) { + builder.append(s.substring(idx, nextEscape)) + } + val c = s(nextEscape) + (c: @switch) match { + case '\\' => + builder.append("\\\\") + case '/' => + builder.append("\\/") + case '\b' => + builder.append("\\b") + case '\f' => + builder.append("\\f") + case '\n' => + builder.append("\\n") + case '\r' => + builder.append("\\r") + case '\t' => + builder.append("\\t") + case '"' => + builder.append("\\\"") + case _ => + // escape non ascii or control characters + builder + .append("\\u") + .append(hex((c >> 12) & 0x0f)) + .append(hex((c >> 8) & 0x0f)) + .append(hex((c >> 4) & 0x0f)) + .append(hex(c & 0x0f)) + } + renderString(s, nextEscape + 1, builder) + } else { + // append the rest of the string and we are done + builder.append(s.substring(idx)) + } + } + + private val nullValue = + Stream.emit(DocEvent.Text("null")) + + private val trueValue = + Stream.emit(DocEvent.Text("true")) + + private val falseValue = + Stream.emit(DocEvent.Text("false")) + + private final val FirstObjectKey = 0 + private final val ObjectKey = 1 + private final val ObjectValue = 2 + private final val FirstArrayValue = 3 + private final val ArrayValue = 4 + + implicit object renderable extends Renderable[Token] { + + private val startObject = + Stream.emits(DocEvent.GroupBegin :: DocEvent.Text("{") :: Nil) + + private val endEmptyObject = + Stream.emits(DocEvent.Text("}") :: DocEvent.GroupEnd :: Nil) + + private val endObject = + Stream.emits(DocEvent.GroupEnd :: DocEvent.IndentEnd :: DocEvent.LineBreak :: Nil) ++ endEmptyObject + + private val startArray = + Stream.emits(DocEvent.GroupBegin :: DocEvent.Text("[") :: Nil) + + private val endEmptyArray = + Stream.emits(DocEvent.Text("]") :: DocEvent.GroupEnd :: Nil) + + private val endArray = + Stream.emits(DocEvent.GroupEnd :: DocEvent.IndentEnd :: DocEvent.LineBreak :: Nil) ++ endEmptyArray + + private val objectSep = + Stream.emits(DocEvent.Text(",") :: DocEvent.GroupEnd :: DocEvent.Line :: Nil) + + private val arraySep = + Stream.emits(DocEvent.Text(",") :: DocEvent.GroupEnd :: DocEvent.Line :: DocEvent.GroupBegin :: Nil) + + override def newRenderer(): Renderer[Token] = new Renderer[Token] { + + // the current stack of states, helping to deal with comma and indentation + // states are described right above + private[this] var states = List.empty[Int] + + private def separator(): Stream[Pure, DocEvent] = + states match { + case Nil => + Stream.empty + case state :: rest => + (state: @switch) match { + case FirstObjectKey => + states = ObjectValue :: rest + Stream.emits(DocEvent.IndentBegin :: DocEvent.LineBreak :: Nil) + case ObjectKey => + states = ObjectValue :: rest + objectSep + case ObjectValue => + states = ObjectKey :: rest + Stream.emit(DocEvent.GroupBegin) + case FirstArrayValue => + states = ArrayValue :: rest + Stream.emits(DocEvent.IndentBegin :: DocEvent.LineBreak :: DocEvent.GroupBegin :: Nil) + case ArrayValue => + states = ArrayValue :: rest + arraySep + } + } + + private def closeObject(): Stream[Pure, DocEvent] = + states match { + case Nil => endEmptyObject + case state :: rest => + states = rest + (state: @switch) match { + case FirstObjectKey => endEmptyObject + case ObjectKey => endObject + } + } + + private def closeArray(): Stream[Pure, DocEvent] = + states match { + case Nil => endEmptyArray + case state :: rest => + states = rest + (state: @switch) match { + case FirstArrayValue => endEmptyArray + case ArrayValue => endArray + } + } + + override def doc(token: Token): Stream[Pure, DocEvent] = + token match { + case StartObject => + val res = separator() ++ startObject + states = FirstObjectKey :: states + res + case EndObject => + closeObject() + case StartArray => + val res = separator() ++ startArray + states = FirstArrayValue :: states + res + case EndArray => + closeArray() + case Key(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append("\": ") + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + case NullValue => + separator() ++ nullValue + case TrueValue => + separator() ++ trueValue + case FalseValue => + separator() ++ falseValue + case NumberValue(value) => + separator() ++ Stream.emit(DocEvent.Text(value)) + case StringValue(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + } + } + + } + + /** A compact version of the JSON rendering with no space or new lines. */ + object compact extends Renderable[Token] { + + private val startObject = + Stream.emit(DocEvent.Text("{")) + + private val endObject = + Stream.emit(DocEvent.Text("}")) + + private val startArray = + Stream.emit(DocEvent.Text("[")) + + private val endArray = + Stream.emit(DocEvent.Text("]")) + + private val sep = + Stream.emit(DocEvent.Text(",")) + + override def newRenderer(): Renderer[Token] = new Renderer[Token] { + + // the current stack of states, helping to deal with comma and indentation + // states are described right above + private[this] var states = List.empty[Int] + + private def separator(): Stream[Pure, DocEvent] = + states match { + case Nil => + Stream.empty + case state :: rest => + (state: @switch) match { + case FirstObjectKey => + states = ObjectValue :: rest + Stream.empty + case ObjectKey => + states = ObjectValue :: rest + sep + case ObjectValue => + states = ObjectKey :: rest + Stream.empty + case FirstArrayValue => + states = ArrayValue :: rest + Stream.empty + case ArrayValue => + states = ArrayValue :: rest + sep + } + } + + private def closeObject(): Stream[Pure, DocEvent] = + states match { + case Nil => endObject + case state :: rest => + states = rest + (state: @switch) match { + case FirstObjectKey => endObject + case ObjectKey => endObject + } + } + + private def closeArray(): Stream[Pure, DocEvent] = + states match { + case Nil => endArray + case state :: rest => + states = rest + (state: @switch) match { + case FirstArrayValue => endArray + case ArrayValue => endArray + } + } + + override def doc(token: Token): Stream[Pure, DocEvent] = + token match { + case StartObject => + val res = separator() ++ startObject + states = FirstObjectKey :: states + res + case EndObject => + closeObject() + case StartArray => + val res = separator() ++ startArray + states = FirstArrayValue :: states + res + case EndArray => + closeArray() + case Key(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append("\":") + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + case NullValue => + separator() ++ nullValue + case TrueValue => + separator() ++ trueValue + case FalseValue => + separator() ++ falseValue + case NumberValue(value) => + separator() ++ Stream.emit(DocEvent.Text(value)) + case StringValue(value) => + val rendered = new StringBuilder + rendered.append('"') + renderString(value, 0, rendered) + rendered.append('"') + separator() ++ Stream.emit(DocEvent.Text(rendered.result())) + } + } + } } diff --git a/json/src/test/scala/fs2/data/json/RenderSpec.scala b/json/src/test/scala/fs2/data/json/RenderSpec.scala index ae7a721d..31852584 100644 --- a/json/src/test/scala/fs2/data/json/RenderSpec.scala +++ b/json/src/test/scala/fs2/data/json/RenderSpec.scala @@ -16,10 +16,7 @@ package fs2.data.json -import internals._ - import fs2._ - import weaver._ object RenderSpec extends SimpleIOSuite { @@ -30,7 +27,7 @@ object RenderSpec extends SimpleIOSuite { val toks = input.through(tokens[Fallible, Char]) - val roundtrip = toks.through(render.compact).flatMap(Stream.emits(_)).through(tokens) + val roundtrip = toks.through(render.compact).through(tokens) expect(toks.compile.toList == roundtrip.compile.toList) @@ -42,20 +39,72 @@ object RenderSpec extends SimpleIOSuite { val toks = input.through(tokens[Fallible, Char]) - val roundtrip = toks.through(render.pretty()).flatMap(Stream.emits(_)).through(tokens) + val roundtrip = toks.through(render.prettyPrint()).through(tokens) expect(toks.compile.toList == roundtrip.compile.toList) } pureTest("a Renderer should properly escape what needs to be escaped") { - val renderer = new Renderer(true, true, "") + val input = Stream.emit(Token.StringValue("some\ncharacters must\\be\"escaped\" like ß")).covaryOutput[Token] + + expect.same("\"some\\ncharacters must\\\\be\\\"escaped\\\" like \\u00df\"", + input.through(render.prettyPrint()).compile.string) + + } + + pureTest("An object should be properly pretty renderer with line width of 10") { + val input = Stream.emits("""{"field1": "test", "field2": [23, [true, null]]}""") + + expect.same( + Right("""{ + | "field1": "test", + | "field2": [ + | 23, + | [ + | true, + | null + | ] + | ] + |}""".stripMargin), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 10, indent = 2)).compile.string + ) + + } + + pureTest("An object should be properly pretty renderer with line width of 32") { + val input = Stream.emits("""{"field1": "test", "field2": [23, [true, null]]}""") - renderer += Chunk.singleton(Token.StringValue("some\ncharacters must\\be\"escaped\" like ß")) + expect.same( + Right("""{ + | "field1": "test", + | "field2": [23, [true, null]] + |}""".stripMargin), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 32, indent = 2)).compile.string + ) + + } + + pureTest("An object should be properly pretty renderer with line width of 80") { + val input = Stream.emits("""{ + | "field1": "test", + | "field2": [23, [true, null]]}""".stripMargin) + + expect.same( + Right("""{"field1": "test", "field2": [23, [true, null]]}"""), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 80, indent = 2)).compile.string + ) + } - val res = renderer.result + pureTest("An object should be properly pretty renderer with line width of 80") { + val input = Stream.emits("""{ + | "field1": "test", + | "field2": [23, [true, null]]}""".stripMargin) - expect(res == "\"some\\ncharacters must\\\\be\\\"escaped\\\" like \\u00df\"") + expect.same( + Right("""{"field1": "test", "field2": [23, [true, null]]}"""), + input.through(tokens[Fallible, Char]).through(fs2.data.text.render.pretty(width = 80, indent = 2)).compile.string + ) } diff --git a/site/cookbooks/jq.md b/site/cookbooks/jq.md index 93bfe049..e0f485a4 100644 --- a/site/cookbooks/jq.md +++ b/site/cookbooks/jq.md @@ -15,7 +15,7 @@ The _Reading_ and _Writing_ steps are not specific to `fs2-data` but rely on pur - The `tokens` pipe to parse the input stream into JSON @:api(fs2.data.json.Token)s (see [the documentation][json-doc] for more details). - The @:api(fs2.data.json.jq.Compiler) class to compile a query into a pipe (see [the documentation][jq-doc] for more details). - - The `render.pretty` pipe to render the query result into a pretty-printed JSON string (see [the documentation][render-doc] for more details). + - The `render.prettyPrint` pipe to render the query result into a pretty-printed JSON string (see [the documentation][render-doc] for more details). In general the _Transforming_ step can use whatever operator fits your purpose, from `fs2` or any other `fs2`-based library. But in our case the only transformation will be performed by the query. @@ -55,7 +55,7 @@ import fs2.data.json Files[IO] .readUtf8(Path("site/cookbooks/data/json/sample.json")) .through(json.tokens) // parsing JSON input - .through(json.render.pretty()) // pretty printing JSON stream + .through(json.render.prettyPrint()) // pretty printing JSON stream .through(utf8.encode[IO]) .through(stdout) .compile @@ -95,7 +95,7 @@ Files[IO] .readUtf8(Path("site/cookbooks/data/json/sample.json")) .through(json.tokens) .through(queryPipe) // the transformation using the query pipe - .through(json.render.pretty()) + .through(json.render.prettyPrint()) .through(utf8.encode[IO]) .through(stdout) .compile diff --git a/site/documentation/json/index.md b/site/documentation/json/index.md index 7ae3e00b..e74f6e66 100644 --- a/site/documentation/json/index.md +++ b/site/documentation/json/index.md @@ -128,17 +128,17 @@ stream .drain ``` -There exists also a `pretty()` renderer, that indents inner elements by the given indent string. +There exists also a `prettyPrint()` renderer, that indents inner elements by the given indent size (in spaces) and for a given page width. -If you are interested in the String rendering as a value, the library also provides `Collector`s: +If you are interested in the String rendering as a value, you can use the `string` `Collector`: ```scala mdoc -stream.compile.to(collector.compact) +stream.through(render.compact).compile.string // default indentation is 2 spaces -stream.compile.to(collector.pretty()) -// if you are more into tabs (or any other indentation size) you can change the indentation string -stream.compile.to(collector.pretty("\t")) +stream.through(render.prettyPrint(width = 10)).compile.string +// if you are more into 4 spaces (or any other indentation size) you can change the indentation size +stream.through(render.prettyPrint(indent = 4, width = 10)).compile.string ``` ## Generating JSON streams diff --git a/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala index 93319c0e..bb5a552a 100644 --- a/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala +++ b/text/shared/src/main/scala/fs2/data/text/render/internal/StreamPrinter.scala @@ -19,11 +19,9 @@ package internal import cats.collections.Dequeue import cats.data.{Chain, NonEmptyList} -import fs2.{Chunk, Pipe, Pull, RaiseThrowable, Stream} +import fs2.{Chunk, Pipe, Pull, Stream} -private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(implicit - F: RaiseThrowable[F], - render: Renderable[Event]) +private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(implicit render: Renderable[Event]) extends Pipe[F, Event, String] { private val emptyGroups = (0, Dequeue.empty[(Int, Chain[Annotated])]) @@ -74,12 +72,7 @@ private[render] class StreamPrinter[F[_], Event](width: Int, indentSize: Int)(im if (idx >= chunk.size) { rest.pull.uncons.flatMap { case Some((hd, tl)) => annotate(hd, 0, tl, pos, aligns, hpl, groups) - case None => - if (!groups.isEmpty) { - Pull.raiseError(RenderException("Document stream malformed (unclosed group)")) - } else { - Pull.done - } + case None => Pull.done } } else { val evt = chunk(idx) diff --git a/text/shared/src/main/scala/fs2/data/text/render/package.scala b/text/shared/src/main/scala/fs2/data/text/render/package.scala index 4cdb82e0..317b2bef 100644 --- a/text/shared/src/main/scala/fs2/data/text/render/package.scala +++ b/text/shared/src/main/scala/fs2/data/text/render/package.scala @@ -16,13 +16,12 @@ package fs2.data.text +import fs2.Pipe import fs2.data.text.render.internal.StreamPrinter -import fs2.{Pipe, RaiseThrowable} package object render { def pretty[F[_], Event](width: Int = 100, indent: Int = 2)(implicit - F: RaiseThrowable[F], render: Renderable[Event]): Pipe[F, Event, String] = new StreamPrinter[F, Event](width, indent)