[CORE-41] Update BQ spending report query to remove duplicates (#3161)

* remove project name from query
broadinstitute · Jan 8, 2025 · 02e9f4f · 02e9f4f
1 parent 23f4c8d
commit 02e9f4f
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 14 deletions.
diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingService.scala
@@ -4,8 +4,7 @@ import java.util.{Currency, UUID}
 import akka.http.scaladsl.model.StatusCodes
 import cats.effect.IO
 import cats.effect.unsafe.implicits.global
-import com.google.cloud.bigquery.{JobStatistics, Option => _, _}
-import com.google.cloud.bigquery.FieldValueList
+import com.google.cloud.bigquery.{Field, FieldValue, FieldValueList, JobStatistics, Option => _, _}
 import com.typesafe.scalalogging.LazyLogging
 import nl.grons.metrics4.scala.{Counter, Histogram}
 import org.broadinstitute.dsde.rawls.billing.{
@@ -366,7 +365,6 @@ class SpendReportingService(
     val baseQuery = s"""
                        |  SELECT
                        |    project.id AS project_id,
-                       |    project.name AS project_name,
                        |    currency,
                        |    SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits,
                        |    CASE
@@ -382,7 +380,6 @@ class SpendReportingService(
                        |    _PARTITIONTIME BETWEEN @startDate AND @endDate
                        |  GROUP BY
                        |    project_id,
-                       |    project_name,
                        |    spend_category,
                        |    currency""".stripMargin.trim
 
@@ -402,7 +399,6 @@ class SpendReportingService(
        |)
        |SELECT
        |  project_id,
-       |  project_name,
        |  SUM(category_cost) AS total_cost,
        |  SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost,
        |  SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost,
@@ -415,7 +411,6 @@ class SpendReportingService(
        |  spend_categories
        |GROUP BY
        |  project_id,
-       |  project_name,
        |  currency
        |ORDER BY
        |  total_cost DESC

diff --git a/...c/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala b/...c/test/scala/org/broadinstitute/dsde/rawls/spendreporting/SpendReportingServiceSpec.scala
@@ -1276,7 +1276,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki
       s"""|WITH spend_categories AS (
           |  SELECT
           |    project.id AS project_id,
-          |    project.name AS project_name,
           |    currency,
           |    SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits,
           |    CASE
@@ -1292,13 +1291,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki
           |    _PARTITIONTIME BETWEEN @startDate AND @endDate
           |  GROUP BY
           |    project_id,
-          |    project_name,
           |    spend_category,
           |    currency
           | UNION ALL
           |    SELECT
           |      project.id AS project_id,
-          |      project.name AS project_name,
           |      currency,
           |      SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits,
           |      CASE
@@ -1314,13 +1311,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki
           |      _PARTITIONTIME BETWEEN @startDate AND @endDate
           |    GROUP BY
           |      project_id,
-          |      project_name,
           |      spend_category,
           |      currency
           | UNION ALL
           |    SELECT
           |      project.id AS project_id,
-          |      project.name AS project_name,
           |      currency,
           |      SUM(IFNULL((SELECT SUM(c.amount) FROM UNNEST(credits) c), 0)) as credits,
           |      CASE
@@ -1336,13 +1331,11 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki
           |      fakeTimePartitionColumn BETWEEN @startDate AND @endDate
           |    GROUP BY
           |      project_id,
-          |      project_name,
           |      spend_category,
           |      currency
           |)
           |SELECT
           |  project_id,
-          |  project_name,
           |  SUM(category_cost) AS total_cost,
           |  SUM(CASE WHEN spend_category = 'Storage' THEN category_cost ELSE 0 END) AS storage_cost,
           |  SUM(CASE WHEN spend_category = 'Compute' THEN category_cost ELSE 0 END) AS compute_cost,
@@ -1355,7 +1348,6 @@ class SpendReportingServiceSpec extends AnyFlatSpecLike with Matchers with Mocki
           |  spend_categories
           |GROUP BY
           |  project_id,
-          |  project_name,
           |  currency
           |ORDER BY
           |  total_cost DESC