Merge remote-tracking branch 'origin/develop' into aen_an_297

core/src/main/resources/swagger/api-docs.yaml

@@ -450,6 +450,59 @@ paths:
                 $ref: '#/components/schemas/ErrorReport'
         500:
           $ref: '#/components/responses/RawlsInternalError'
+  /api/billing/v2/spendReport:
+    get:
+      tags:
+        - billing_v2
+      summary: get spend report for all workspaces user has owner access to
+      description: get spend report for all workspaces user has owner access to
+      operationId: getSpendReportAllWorkspaces
+      parameters:
+        - name: startDate
+          in: query
+          description: start date of report (YYYY-MM-DD). Data included in report will start at 12 AM UTC on this date.
+          required: true
+          schema:
+            type: string
+            format: date
+        - name: endDate
+          in: query
+          description: end date of report (YYYY-MM-DD). Data included in report will end at 11:59 PM UTC on this date.
+          required: true
+          schema:
+            type: string
+            format: date
+        - name: pageSize
+          in: query
+          description: how many workspaces to return at a time
+          required: false
+          default: 100
+        - name: offset
+          in: query
+          description: The number of items to skip before starting to collect the result
+          required: false
+          default: 0
+      responses:
+        200:
+          description: Success
+          content:
+            'application/json':
+              schema:
+                $ref: '#/components/schemas/SpendReport'
+        400:
+          description: invalid spend report parameters
+          content:
+            'application/json':
+              schema:
+                $ref: '#/components/schemas/ErrorReport'
+        403:
+          description: You must be a workspace owner to view the spend report of a workspace
+          content:
+            'application/json':
+              schema:
+                $ref: '#/components/schemas/ErrorReport'
+        500:
+          $ref: '#/components/responses/RawlsInternalError'
   /api/billing/v2/{projectId}/spendReportConfiguration:
     get:
       tags:

core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala

@@ -516,7 +516,8 @@ object Boot extends IOApp with LazyLogging {
           billingRepository,
           billingProfileManagerDAO,
           samDAO,
-          spendReportingServiceConfig
+          spendReportingServiceConfig,
+          workspaceServiceConstructor
         )
 
       val billingAdminServiceConstructor: RawlsRequestContext => BillingAdminService =

core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala

@@ -14,12 +14,21 @@ import org.broadinstitute.dsde.rawls.model._
 import org.broadinstitute.dsde.rawls.util.{FutureSupport, Retry}
 import org.broadinstitute.dsde.workbench.client.sam
 import org.broadinstitute.dsde.workbench.client.sam.api._
+import org.broadinstitute.dsde.workbench.client.sam.model.{
+  FilteredFlatResource,
+  FilteredFlatResourcePolicy,
+  FilteredHierarchicalResource,
+  FilteredHierarchicalResourcePolicy,
+  FilteredResourcesHierarchicalResponse,
+  ListResourcesV2200Response
+}
 import org.broadinstitute.dsde.workbench.client.sam.{ApiCallback, ApiClient, ApiException}
 import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName}
 
 import java.time.Instant
 import java.time.temporal.ChronoUnit
 import java.util
+import java.util.List
 import scala.concurrent.duration.FiniteDuration
 import scala.concurrent.{ExecutionContext, Future, Promise}
 import scala.jdk.CollectionConverters._
@@ -513,6 +522,31 @@ class HttpSamDAO(baseSamServiceURL: String, rawlsCredential: RawlsCredential, ti
       }
     }
 
+  override def listResourcesWithActions(resourceTypeName: SamResourceTypeName,
+                                        action: SamResourceAction,
+                                        ctx: RawlsRequestContext
+  ): Future[Seq[FilteredFlatResource]] =
+    retry(when401or5xx) { () =>
+      val callback = new SamApiCallback[ListResourcesV2200Response]("listResourcesV2")
+
+      resourcesApi(ctx).listResourcesV2Async(
+        /* format = */ "flat",
+        /* resourceTypes = */ util.List.of(resourceTypeName.value),
+        /* policies = */ util.List.of(),
+        /* roles = */ util.List.of(),
+        /* actions = */ util.List.of(action.value),
+        /* includePublic = */ true,
+        callback
+      )
+
+      callback.future.map { resourcesResponse =>
+        resourcesResponse.getFilteredResourcesFlatResponse
+          .getResources()
+          .asScala
+          .toSeq
+      }
+    }
+
   private def toSamRolesAndActions(rolesAndActions: sam.model.RolesAndActions) =
     SamRolesAndActions(
       rolesAndActions.getRoles.asScala.map(SamResourceRole).toSet,

core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala

@@ -26,6 +26,7 @@ import org.broadinstitute.dsde.rawls.model.{
   UserIdInfo,
   UserInfo
 }
+import org.broadinstitute.dsde.workbench.client.sam.model.{FilteredFlatResource, FilteredHierarchicalResource}
 import org.broadinstitute.dsde.workbench.model._
 
 import scala.concurrent.Future
@@ -100,6 +101,11 @@ trait SamDAO {
 
   def listUserResources(resourceTypeName: SamResourceTypeName, ctx: RawlsRequestContext): Future[Seq[SamUserResource]]
 
+  def listResourcesWithActions(resourceTypeName: SamResourceTypeName,
+                               action: SamResourceAction,
+                               ctx: RawlsRequestContext
+  ): Future[Seq[FilteredFlatResource]]
+
   def listPoliciesForResource(resourceTypeName: SamResourceTypeName,
                               resourceId: String,
                               ctx: RawlsRequestContext

core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala

@@ -1,6 +1,7 @@
 package org.broadinstitute.dsde.rawls.dataaccess.slick
 
 import cats.implicits.catsSyntaxOptionId
+import com.google.common.annotations.VisibleForTesting
 import org.broadinstitute.dsde.rawls.RawlsException
 import org.broadinstitute.dsde.rawls.dataaccess.GoogleApiTypes.GoogleApiType
 import org.broadinstitute.dsde.rawls.dataaccess.GoogleOperationNames.GoogleOperationName
@@ -18,6 +19,7 @@ import slick.jdbc.JdbcType
 import java.sql.Timestamp
 import java.time.Instant
 import java.util.UUID
+import scala.util.Try
 
 final case class RawlsBillingProjectRecord(projectName: String,
                                            creationStatus: String,
@@ -361,6 +363,7 @@ trait RawlsBillingProjectComponent {
     def clearBillingProjectSpendConfiguration(billingProjectName: RawlsBillingProjectName): WriteAction[Int] =
       setBillingProjectSpendConfiguration(billingProjectName, None, None, None)
 
+    // Throws an error if the Billing Project does not have a Billing Account
     def getBillingProjectSpendConfiguration(
       billingProjectName: RawlsBillingProjectName
     ): ReadAction[Option[BillingProjectSpendExport]] =
@@ -369,6 +372,17 @@ trait RawlsBillingProjectComponent {
         .result
         .map(_.headOption.map(RawlsBillingProjectRecord.toBillingProjectSpendExport))
 
+    // Ignores any Billing Projects that don't have Billing Accounts
+    def getBillingProjectsSpendConfiguration(
+      billingProjectNames: Seq[RawlsBillingProjectName]
+    ): ReadAction[Seq[Option[BillingProjectSpendExport]]] =
+      rawlsBillingProjectQuery
+        .withProjectNames(billingProjectNames)
+        .result
+        .map(projectRecords =>
+          projectRecords.map(record => Try(RawlsBillingProjectRecord.toBillingProjectSpendExport(record)).toOption)
+        )
+
     def insertOperations(operations: Seq[RawlsBillingProjectOperationRecord]): WriteAction[Unit] =
       (rawlsBillingProjectOperationQuery ++= operations).map(_ => ())
 
@@ -540,6 +554,7 @@ trait RawlsBillingProjectComponent {
       )
     }
 
+    @VisibleForTesting
     def getLastChange(billingProject: RawlsBillingProjectName): ReadAction[Option[BillingAccountChange]] =
       BillingAccountChanges
         .withProjectName(billingProject)
@@ -574,27 +589,6 @@ trait RawlsBillingProjectComponent {
     def withProjectName(billingProjectName: RawlsBillingProjectName): BillingAccountChangeQuery =
       query.filter(_.billingProjectName === billingProjectName.value)
 
-    /* SELECT *
-     * FROM BILLING_ACCOUNT_CHANGES BAC,
-     * (  SELECT BILLING_PROJECT_NAME, MAX(ID) AS MAXID
-     *    FROM BILLING_ACCOUNT_CHANGES
-     *    GROUP BY BILLING_PROJECT_NAME
-     * ) AS SUBTABLE
-     * WHERE SUBTABLE.MAXID = BAC.ID
-     */
-    /**
-      * Selects the latest changes for all billing projects in query.
-      */
-    def latestChanges: BillingAccountChangeQuery = {
-      val latestChangeIds = query
-        .groupBy(_.billingProjectName)
-        .map { case (_, group) => group.map(_.id).max }
-
-      query
-        .filter(_.id.in(latestChangeIds))
-        .sortBy(_.id.asc)
-    }
-
     def unsynced: BillingAccountChangeQuery =
       query.filter(_.googleSyncTime.isEmpty)
 

core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala

@@ -7,6 +7,7 @@ import cats.{Monoid, MonoidK}
 import org.broadinstitute.dsde.rawls.RawlsException
 import org.broadinstitute.dsde.rawls.model.Attributable.AttributeMap
 import org.broadinstitute.dsde.rawls.model.WorkspaceState.WorkspaceState
+import org.broadinstitute.dsde.rawls.model.WorkspaceType.WorkspaceType
 import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion
 import org.broadinstitute.dsde.rawls.model._
 import org.broadinstitute.dsde.rawls.util.CollectionUtils
@@ -260,6 +261,21 @@ trait WorkspaceComponent {
     def listWithBillingProject(billingProject: RawlsBillingProjectName): ReadAction[Seq[Workspace]] =
       workspaceQuery.withBillingProject(billingProject).read
 
+    def groupByBillingProjectOfType(workspaceIds: List[UUID],
+                                    workspaceType: WorkspaceType
+    ): ReadWriteAction[Map[RawlsBillingProjectName, Seq[Workspace]]] = {
+      val query = for {
+        workspace <- workspaceQuery if workspace.id inSetBind workspaceIds.toSet
+        if workspace.workspaceType === workspaceType.toString
+      } yield (workspace.namespace, workspace)
+
+      query.result.map { rows =>
+        rows.groupBy(_._1).map { case (billingProjectName, workspaces) =>
+          RawlsBillingProjectName(billingProjectName) -> workspaces.map(_._2).map(WorkspaceRecord.toWorkspace)
+        }
+      }
+    }
+
     def getTags(queryString: Option[String],
                 limit: Option[Int] = None,
                 ownerIds: Option[Seq[UUID]] = None
@@ -644,6 +660,9 @@ trait WorkspaceComponent {
     def withBillingProject(projectName: RawlsBillingProjectName): WorkspaceQueryType =
       query.filter(_.namespace === projectName.value)
 
+    def withBillingProjects(projectNames: List[RawlsBillingProjectName]): WorkspaceQueryType =
+      query.filter(_.namespace.inSetBind(projectNames.map(_.value)))
+
     def withGoogleProjectId(googleProjectId: GoogleProjectId): WorkspaceQueryType =
       query.filter(_.googleProjectId === googleProjectId.value)
 

core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala

@@ -70,6 +70,7 @@ object SamWorkspaceActions {
   val delete = SamResourceAction("delete")
   val migrate = SamResourceAction("migrate")
   val viewMigrationStatus = SamResourceAction("view_migration_status")
+  val readSpendReport = SamResourceAction("read_spend_report")
   def sharePolicy(policy: String) = SamResourceAction(s"share_policy::$policy")
 }
 

core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/BillingAccountChangeSynchronizer.scala

@@ -15,13 +15,12 @@ import cats.implicits.{
   toFunctorOps
 }
 import cats.mtl.Ask
-import cats.{Applicative, Functor, Monad, MonadThrow}
+import cats.{Functor, Monad, MonadThrow}
 import com.typesafe.scalalogging.LazyLogging
 import org.broadinstitute.dsde.rawls.dataaccess.slick.{
   BillingAccountChange,
   BillingAccountChangeStatus,
-  ReadWriteAction,
-  WriteAction
+  ReadWriteAction
 }
 import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO, SlickDataSource}
 import org.broadinstitute.dsde.rawls.model._
@@ -47,7 +46,7 @@ object BillingAccountChangeSynchronizer {
     Behaviors.setup { context =>
       val actor = BillingAccountChangeSynchronizer(dataSource, gcsDAO, samDAO)
       Behaviors.withTimers { scheduler =>
-        scheduler.startTimerAtFixedRate(UpdateBillingAccounts, initialDelay, pollInterval)
+        scheduler.startTimerWithFixedDelay(UpdateBillingAccounts, initialDelay, pollInterval)
         Behaviors.receiveMessage { case UpdateBillingAccounts =>
           try actor.updateBillingAccounts.unsafeRunSync()
           catch {
@@ -88,9 +87,7 @@ final case class BillingAccountChangeSynchronizer(dataSource: SlickDataSource,
 
   def readABillingProjectChange: IO[Option[BillingAccountChange]] =
     inTransaction {
-      BillingAccountChanges.latestChanges.unsynced
-        .take(1)
-        .result
+      BillingAccountChanges.nextOutstanding().result
     }.map(_.headOption)
 
   private def syncBillingAccountChange[F[_]](tracingContext: RawlsTracingContext)(implicit
@@ -101,12 +98,6 @@ final case class BillingAccountChangeSynchronizer(dataSource: SlickDataSource,
     for {
       billingProject <- loadBillingProject
 
-      // Try out the new query using the STATUS column and compare results. If validation fails, don't
-      // fail the entire process.
-      _ <- validateStatusColumn.recover { case e: Exception =>
-        logger.warn(s"BillingAccountChangeStatus logic failed with ${e.getMessage}", e)
-      }
-
       // v1 billing projects are backed by google projects and are used for v1 workspace billing
       updateBillingProjectOutcome <- M.ifM(isV1BillingProject(billingProject.projectName))(
         updateBillingProjectGoogleProject(billingProject, tracingContext),
@@ -117,44 +108,6 @@ final case class BillingAccountChangeSynchronizer(dataSource: SlickDataSource,
       _ <- writeBillingAccountChangeOutcome(updateBillingProjectOutcome |+| updateWorkspacesOutcome)
     } yield ()
 
-  private def validateStatusColumn[F[_]](implicit
-    R: Ask[F, BillingAccountChange],
-    M: Monad[F],
-    L: LiftIO[F]
-  ): F[Unit] =
-    for {
-      (changeStatus, changeId) <- R.reader(x => (x.status, x.id))
-      byStatus <- inTransaction(BillingAccountChanges.nextOutstanding().result)
-    } yield
-    /* validation:
-          - byStatus should find exactly 1
-          - byStatus should have same id
-          - changeStatus should have status == Outstanding
-     */
-    if (
-      byStatus.length == 1 &&
-      byStatus.head.id == changeId &&
-      changeStatus == BillingAccountChangeStatus.Outstanding
-    ) {
-      logger.info(s"BillingAccountChangeStatus logic correct for change id $changeId")
-    } else {
-      // collect errors
-      val sb = new StringBuilder(s"BillingAccountChangeStatus logic error for change id $changeId!")
-      if (byStatus.isEmpty) {
-        sb.append(" By-status lookup was empty.")
-      }
-      if (byStatus.length > 1) {
-        sb.append(s" By-status lookup had length ${byStatus.length}.")
-      }
-      if (changeStatus != BillingAccountChangeStatus.Outstanding) {
-        sb.append(s" Legacy lookup had status $changeStatus.")
-      }
-      if (changeId != byStatus.head.id) {
-        sb.append(s" Legacy found id $changeId, but by-status head had id ${byStatus.head.id}")
-      }
-      logger.warn(sb.toString())
-    }
-
   private def loadBillingProject[F[_]](implicit
     R: Ask[F, BillingAccountChange],
     M: Monad[F],