From f744c36066365ad69b527c38f5d8e3dab7167c53 Mon Sep 17 00:00:00 2001 From: Alexander Ioffe Date: Wed, 22 Dec 2021 01:03:19 -0500 Subject: [PATCH] Better ExpandJoin phase using flat-joins --- README.md | 44 ++++++++- .../io/getquill/norm/SheathLeafClauses.scala | 79 ++++++++++------ .../io/getquill/sql/norm/ExpandJoin.scala | 94 ++++++++----------- .../io/getquill/sql/norm/SqlNormalize.scala | 1 + .../context/sql/norm/ExpandJoinSpec.scala | 2 +- 5 files changed, 134 insertions(+), 86 deletions(-) diff --git a/README.md b/README.md index 038799d3a3..9f430c3c43 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Quill provides a Quoted Domain Specific Language ([QDSL](http://homepages.inf.ed.ac.uk/wadler/papers/qdsl/qdsl.pdf)) to express queries in Scala and execute them in a target language. The library's core is designed to support multiple target languages, currently featuring specializations for Structured Query Language ([SQL](https://en.wikipedia.org/wiki/SQL)) and Cassandra Query Language ([CQL](https://cassandra.apache.org/doc/latest/cql/)). +> ### [Scastie](https://scastie.scala-lang.org/) is a great tool to try out Quill without having to prepare a local environment. It works with [mirror contexts](#mirror-context), see [this](https://scastie.scala-lang.org/QwOewNEiR3mFlKIM7v900A) snippet as an example. + ![example](https://raw.githubusercontent.com/getquill/quill/master/example.gif) 1. **Boilerplate-free mapping**: The database schema is mapped using simple case classes. @@ -24,6 +26,46 @@ Quill provides a Quoted Domain Specific Language ([QDSL](http://homepages.inf.ed Note: The GIF example uses Eclipse, which shows compilation messages to the user. +# Getting Started + +Quill has integrations with many modules and libraries. If you are using a regular RDBMs database e.g. PostgreSQL +and want to use Quill to query it an asychronous, no-blocking, reactive library, the easiest way to get +started is by using an awesome library called ZIO. + +Create a project directory `intro-to-quill` + +Create a `intro-to-quill/build.sbt` +```scala +``` + +// TODO Need to test this!!! +Create a `intro-to-quill/src/main/scala/intro/Module.scala` +```scala +object QuillContext extends PostgresZioJdbcContext(SnakeCase) { + val dataSourceLayer: ULayer[Has[DataSource]] = + DataSourceLayer.fromPrefix("database").orDie +} + +object DataService { + val live = (DataServiceLive.apply _).toLayer[DataService] +} + +final case class DataServiceLive(dataSource: DataSource) extends DataService { + val env = Has(dataSource) + def getPeople: IO[SQLException, List[Person]] = run(query[Person]).provide(env) + def getPeopleOlderThan(age: Int): IO[SQLException, List[Person]] = run(query[Person].filter(p => p.age > lift(age))).provide(env) +} + +object Main extends zio.App { + override def run(args: List[String]): URIO[zio.ZEnv, ExitCode] = + DataService.getPeople + .inject(dataSourceLayer,DataService.live) + .debug("Results") + .exitCode +} +``` + + # Quotation ## Introduction @@ -38,8 +80,6 @@ import io.getquill._ val ctx = new SqlMirrorContext(MirrorSqlDialect, Literal) ``` -> ### **Note:** [Scastie](https://scastie.scala-lang.org/) is a great tool to try out Quill without having to prepare a local environment. It works with [mirror contexts](#mirror-context), see [this](https://scastie.scala-lang.org/QwOewNEiR3mFlKIM7v900A) snippet as an example. - The context instance provides all the types, methods, and encoders/decoders needed for quotations: ```scala diff --git a/quill-core-portable/src/main/scala/io/getquill/norm/SheathLeafClauses.scala b/quill-core-portable/src/main/scala/io/getquill/norm/SheathLeafClauses.scala index 59209e5b34..ff4b67434e 100644 --- a/quill-core-portable/src/main/scala/io/getquill/norm/SheathLeafClauses.scala +++ b/quill-core-portable/src/main/scala/io/getquill/norm/SheathLeafClauses.scala @@ -7,12 +7,18 @@ import io.getquill.util.Interpolator import io.getquill.util.Messages.TraceType /** + * The state produced in some child clause by the `sheathLeaf` function is essentially "consumed" by the + * `elaborateSheath` function in the parent. + * * Note that in the documentation is use a couple of shorthands: * * M - means Map * Fm - means FlatMap * ent - means a Ast Query. Typically just a Ast Entity - * e.v - this dot-shorthand means Property(e, v) where e is an Ast Ident + * e.v - this dot-shorthand means Property(e, v) where e is an Ast Ident. This is essentially a scalar-projection + * from the entity e. + * leaf - Typically this is a query-ast clause that results in a scalar type. It could be M(ent,e,e.v) or + * an `infix"stuff".as[Query[Int/String/Boolean/etc...] ]` */ case class SheathLeafClauses(state: Option[String]) extends StatefulTransformerWithStack[Option[String]] { @@ -181,52 +187,67 @@ case class SheathLeafClauses(state: Option[String]) extends StatefulTransformerW // This happens in two phases: // 1) Elaborate Sheaths - This means that if in the map.query we produced some kind of state "property" then // we transform the alias `e` in the body to `e.property`. - // 2) - + // 2) SheathLeaves - This means that if there are more nested stages that use a leaf projection (e.g. `e.v`) + // when we have to wrap that projection into another CaseClass which will provide the column information for us (e.g. `CC(v->e.v)`). + // Note that in certain phases that automatically expect a leaf-project e.g. aggregations we do not want to do this phase. + // that means that we need to track what the parent-ast (that contains this one) is. case MapClause(NotGroupBy(ent), e, LeafQuat(body), remake) => - val (ent1, s) = apply(ent) - val e1 = Ident(e.name, ent1.quat) - val bodyA = elaborateSheath(body)(s.state, e, e1) - val (bodyB, s1) = { - if (parentShouldNeverHaveLeaves) - sheathLeaf(bodyA) - else - (bodyA, None) - } - val (bodyC, _) = apply(bodyB) trace"Sheath Map(qry) with $stateInfo in $qq becomes" andReturn { + val (ent1, s) = apply(ent) + val e1 = Ident(e.name, ent1.quat) + val bodyA = elaborateSheath(body)(s.state, e, e1) + val (bodyB, s1) = { + if (parentShouldNeverHaveLeaves) + sheathLeaf(bodyA) + else + (bodyA, None) + } + val (bodyC, _) = apply(bodyB) (remake(ent1, e1, bodyC), SheathLeafClauses(s1)) } case FlatMap(ent, e, body) => - val (ent1, s) = apply(ent) - val e1 = Ident(e.name, ent1.quat) - val bodyA = elaborateSheath(body)(s.state, e, e1) // TODO Should it be ent1.quat? - val (bodyB, s1) = s.apply(bodyA) trace"Sheath FlatMap(qry) with $stateInfo in $qq becomes" andReturn { + val (ent1, s) = apply(ent) + val e1 = Ident(e.name, ent1.quat) + // Produced state from the inner clause is 'consumed' by the elaborateSheath. No need to propagate it further. + val bodyA = elaborateSheath(body)(s.state, e, e1) + val (bodyB, s1) = apply(bodyA) (FlatMap(ent1, e1, bodyB), s1) } + // When Sheathing leaves inside of a the two join clauses you get resulting state for them. + // The issue is that it is very difficult to continue propagating the state onward so instead + // we immediately map back to the leaf node before the join happens. + // For example, say we have J(M(ent,e,e.v+123),M(ent,e,e.v+456)) that the sheathing changes to J(M(ent,e,CC(x->e.v+123)),M(ent,e,CC(x->e.v+456))). + // We would then like to add an extra layer of mapping J(M(M(ent,e,CC(v->e.v+123)),e,e.x)),M(M(ent,e,CC(v->e.v+456)),e,e.x). + // This might seem to do undo the original intent however, this if the parts e.g. v+123 are not actually reducible e.g. infix"AVG($v) OVER (PARTITION BY...)" + // this additional information will not be removed via the applyMap normalization and actually help SqlQuery understand that the variable `x` should + // be used to identify this column. + // Also note that the alternative could be to produce an outer join e.g. + // J(M(ent,e,CC(x->e.v+123)),M(ent,e,CC(x->e.v+456))) -> M(J(M(ent,e,CC(x->e.v+123)),M(ent,e,CC(x->e.v+456))),id,Tup(id._1.x,id._2.x)) + // Unfortunately however due to the ExpandJoin phase being non-well typed this would cause various kinds of queries to fail. Some + // examples are in SqlQuerySpec. If ExpandJoin can be rewritten to be well-typed this approach can be re-examined. case Join(t, a, b, iA, iB, on) => - val (a1, sa) = apply(a) - val (b1, sb) = apply(b) - val (iA1, iB1) = (Ident(iA.name, a1.quat), Ident(iB.name, b1.quat)) + trace"Sheath Join with $stateInfo in $qq becomes" andReturn { + val (a1, sa) = apply(a) + val (b1, sb) = apply(b) + val (iA1, iB1) = (Ident(iA.name, a1.quat), Ident(iB.name, b1.quat)) - val a1m = sa.state.map(a => Map(a1, iA1, Property(iA1, a))).getOrElse(a1) - val b1m = sb.state.map(a => Map(b1, iB1, Property(iB1, a))).getOrElse(b1) + val a1m = sa.state.map(a => Map(a1, iA1, Property(iA1, a))).getOrElse(a1) + val b1m = sb.state.map(a => Map(b1, iB1, Property(iB1, a))).getOrElse(b1) - trace"Sheath Join with $stateInfo in $qq becomes" andReturn { (Join(t, a1m, b1m, iA, iB, on), SheathLeafClauses(None)) } case Filter(ent, e, LeafQuat(body)) => - val (ent1, s) = apply(ent) - val e1 = Ident(e.name, ent1.quat) - // For filter clauses we want to go from: Filter(M(ent,e,e.v),e == 123) to Filter(M(ent,e,CC(v->e.v)),e,e.v == 123) - // the body should not be re-sheathed since a body of CC(v->e.v == 123) would make no sense since - // that's not even a boolean. Instead we just need to do e.v == 123. - val bodyC = elaborateSheath(body)(s.state, e, e1) trace"Sheath Filter(qry) with $stateInfo in $qq becomes" andReturn { + val (ent1, s) = apply(ent) + val e1 = Ident(e.name, ent1.quat) + // For filter clauses we want to go from: Filter(M(ent,e,e.v),e == 123) to Filter(M(ent,e,CC(v->e.v)),e,e.v == 123) + // the body should not be re-sheathed since a body of CC(v->e.v == 123) would make no sense since + // that's not even a boolean. Instead we just need to do e.v == 123. + val bodyC = elaborateSheath(body)(s.state, e, e1) (Filter(ent1, e1, bodyC), s) } diff --git a/quill-sql-portable/src/main/scala/io/getquill/sql/norm/ExpandJoin.scala b/quill-sql-portable/src/main/scala/io/getquill/sql/norm/ExpandJoin.scala index 22145e811f..6a9274cc57 100644 --- a/quill-sql-portable/src/main/scala/io/getquill/sql/norm/ExpandJoin.scala +++ b/quill-sql-portable/src/main/scala/io/getquill/sql/norm/ExpandJoin.scala @@ -1,58 +1,44 @@ -package io.getquill.context.sql.norm +package io.getquill.sql.norm import io.getquill.ast._ -import io.getquill.norm.BetaReduction -import io.getquill.norm.Normalize -/** - * This phase expands inner joins adding the correct aliases so they will function. Unfortunately, - * since it introduces aliases into the clauses that don't actually exist in the inner expressions, - * it is not technically type-safe but will not result in a Quat error since Quats cannot check - * for Ident scoping. For a better implementation, that uses a well-typed FlatMap/FlatJoin cascade, have - * a look here: - * [[https://gist.github.com/deusaquilus/dfb42880656df12779a0afd4f20ef1bb Better Typed ExpandJoin which uses FlatMap/FlatJoin]] - * - * The reason the above implementation is not currently used is because `ExpandNestedQueries` does not - * yet use Quat fields for expansion. Once this is changed, using that implementation here - * should be reconsidered. - */ -object ExpandJoin { - - def apply(q: Ast) = expand(q, None) - - def expand(q: Ast, id: Option[Ident]) = - Transform(q) { - case q @ Join(_, _, _, Ident(a, _), Ident(b, _), _) => // Ident a and Ident b should have the same Quat, could add an assertion for that - val (qr, tuple) = expandedTuple(q) - Map(qr, id.getOrElse(Ident(s"$a$b", q.quat)), tuple) - } - - private def expandedTuple(q: Join): (Join, Tuple) = - q match { - - case Join(t, a: Join, b: Join, tA, tB, o) => - val (ar, at) = expandedTuple(a) - val (br, bt) = expandedTuple(b) - val or = BetaReduction(o, tA -> at, tB -> bt) - (Join(t, ar, br, tA, tB, or), Tuple(List(at, bt))) - - case Join(t, a: Join, b, tA, tB, o) => - val (ar, at) = expandedTuple(a) - val or = BetaReduction(o, tA -> at) - (Join(t, ar, b, tA, tB, or), Tuple(List(at, tB))) - - case Join(t, a, b: Join, tA, tB, o) => - val (br, bt) = expandedTuple(b) - val or = BetaReduction(o, tB -> bt) - (Join(t, a, br, tA, tB, or), Tuple(List(tA, bt))) - - case q @ Join(t, a, b, tA, tB, on) => - (Join(t, nestedExpand(a, tA), nestedExpand(b, tB), tA, tB, on), Tuple(List(tA, tB))) - } - - private def nestedExpand(q: Ast, id: Ident) = - Normalize(expand(q, Some(id))) match { - case Map(q, _, _) => q - case q => q +object ExpandJoin extends StatelessTransformer { + + object ExcludedFromNesting { + def unapply(ast: Ast) = + ast match { + case _: Entity => true + case _: Infix => true + case _: FlatJoin => true + case _ => false + } + } + + override def apply(e: Query): Query = { + e match { + // Need to nest any map/flatMap clauses found in the `query` slot of a FlatMap or + // Verifier will cause Found an `ON` table reference of a table that is not available when they are + // found in this position. + case fm: FlatMap => + super.apply(fm) match { + case fm @ FlatMap(ExcludedFromNesting(), _, _) => fm + case FlatMap(ent, alias, body) => + FlatMap(Nested(ent), alias, body) + case other => + throw new IllegalArgumentException(s"Result of a flatMap reduction $fm needs to be a flatMap. It was $other.") + } + + // Note that quats in iA, iB shuold not need to change since this is a just a re-wrapping + case Join(typ, a, b, iA, iB, on) => + val a1 = + apply(a) match { + case v @ ExcludedFromNesting() => v + case other => Nested(other) + } + val b1 = apply(b) + FlatMap(a1, iA, Map(FlatJoin(typ, b1, iB, on), iB, Tuple(List(iA, iB)))) + + case _ => super.apply(e) } -} \ No newline at end of file + } +} diff --git a/quill-sql-portable/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala b/quill-sql-portable/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala index 51d651247e..d883d0216b 100644 --- a/quill-sql-portable/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala +++ b/quill-sql-portable/src/main/scala/io/getquill/sql/norm/SqlNormalize.scala @@ -5,6 +5,7 @@ import io.getquill.ast.Ast import io.getquill.norm.ConcatBehavior.AnsiConcat import io.getquill.norm.EqualityBehavior.AnsiEquality import io.getquill.norm.capture.{ AvoidAliasConflict, DemarcateExternalAliases } +import io.getquill.sql.norm.ExpandJoin import io.getquill.util.Messages.{ TraceType, title } object SqlNormalize { diff --git a/quill-sql/src/test/scala/io/getquill/context/sql/norm/ExpandJoinSpec.scala b/quill-sql/src/test/scala/io/getquill/context/sql/norm/ExpandJoinSpec.scala index e5686905ad..5a19a7e27a 100644 --- a/quill-sql/src/test/scala/io/getquill/context/sql/norm/ExpandJoinSpec.scala +++ b/quill-sql/src/test/scala/io/getquill/context/sql/norm/ExpandJoinSpec.scala @@ -1,4 +1,4 @@ -package io.getquill.context.sql.norm +package io.getquill.sql.norm import io.getquill.Spec import io.getquill.context.sql.testContext.qr1