From 0d160d57f20f1c2fe43dcaf979fe0c8483ffa6a1 Mon Sep 17 00:00:00 2001 From: dvoet Date: Wed, 21 Oct 2020 14:27:36 -0400 Subject: [PATCH] Ca 967 use google_project, remove cromwell auth bucket, remove googleRole apis --- .gitallowed | 1 + core/src/main/resources/swagger/api-docs.yaml | 117 ++---------------- .../org/broadinstitute/dsde/rawls/Boot.scala | 3 - .../rawls/dataaccess/GoogleServicesDAO.scala | 39 +++--- .../dataaccess/HttpGoogleServicesDAO.scala | 117 +++++------------- .../dsde/rawls/dataaccess/HttpSamDAO.scala | 8 +- .../RequesterPaysSetupService.scala | 22 ++-- .../dsde/rawls/dataaccess/SamDAO.scala | 6 +- .../slick/RawlsBillingProjectComponent.scala | 9 +- .../dataaccess/slick/WorkspaceComponent.scala | 4 +- .../datarepo/DataRepoEntityProvider.scala | 4 +- .../jobexec/SubmissionMonitorActor.scala | 8 +- .../jobexec/WorkflowSubmissionActor.scala | 7 +- .../dsde/rawls/model/ExecutionModel.scala | 3 +- .../dsde/rawls/model/SamModel.scala | 1 - .../dsde/rawls/model/UserAuth.scala | 7 +- .../rawls/monitor/AvroUpsertMonitor.scala | 41 +++--- .../dsde/rawls/monitor/BootMonitors.scala | 7 +- .../CreatingBillingProjectMonitor.scala | 6 +- .../dsde/rawls/user/UserService.scala | 67 +++------- .../dsde/rawls/util/AuthUtil.scala | 6 +- .../dsde/rawls/util/WorkspaceSupport.scala | 6 +- .../rawls/webservice/BillingApiService.scala | 18 +-- .../rawls/workspace/WorkspaceService.scala | 30 +++-- .../dataaccess/MockGoogleServicesDAO.scala | 34 +++-- .../RequesterPaysSetupServiceSpec.scala | 16 +-- .../slick/TestDriverComponent.scala | 14 +-- .../slick/WorkspaceComponentSpec.scala | 2 +- .../DataRepoEntityProviderSpecSupport.scala | 4 +- .../dsde/rawls/jobexec/SubmissionSpec.scala | 1 - .../jobexec/WorkflowSubmissionSpec.scala | 9 +- .../dsde/rawls/mock/MockSamDAO.scala | 4 +- .../dsde/rawls/model/ExecutionModelSpec.scala | 3 - .../rawls/monitor/AvroUpsertMonitorSpec.scala | 3 +- .../CreatingBillingProjectMonitorSpec.scala | 27 ++-- .../dsde/rawls/user/UserServiceSpec.scala | 18 ++- .../rawls/webservice/ApiServiceSpec.scala | 1 - .../webservice/BillingApiServiceSpec.scala | 5 +- .../dsde/rawls/webservice/PetSASpec.scala | 2 +- .../webservice/SubmissionApiServiceSpec.scala | 2 +- .../rawls/webservice/UserApiServiceSpec.scala | 6 +- .../WorkspaceApiGetOptionsSpec.scala | 2 +- .../WorkspaceApiListOptionsSpec.scala | 2 +- .../webservice/WorkspaceApiServiceSpec.scala | 2 +- .../workspace/WorkspaceServiceSpec.scala | 1 - .../dsde/rawls/model/WorkspaceModel.scala | 16 ++- 46 files changed, 258 insertions(+), 453 deletions(-) create mode 100644 .gitallowed diff --git a/.gitallowed b/.gitallowed new file mode 100644 index 0000000000..56be524c8b --- /dev/null +++ b/.gitallowed @@ -0,0 +1 @@ +core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala \ No newline at end of file diff --git a/core/src/main/resources/swagger/api-docs.yaml b/core/src/main/resources/swagger/api-docs.yaml index c73741b2c1..46c4a0754d 100644 --- a/core/src/main/resources/swagger/api-docs.yaml +++ b/core/src/main/resources/swagger/api-docs.yaml @@ -276,115 +276,6 @@ paths: - openid - email - profile - /api/billing/{projectId}/googleRole/{role}/{email}: - put: - tags: - - billing - summary: grant a google role to a user and their pet in the billing project - the caller owns - description: grant a google role to a user and their pet in the billing project - the caller owns - operationId: grantGoogleRoleToUser - parameters: - - name: projectId - in: path - description: Project ID - required: true - schema: - type: string - - name: role - in: path - description: google role of user for project - required: true - schema: - type: string - - name: email - in: path - description: email of user - required: true - schema: - type: string - responses: - 200: - description: Successfully Granted Google Role to User - content: {} - 403: - description: You must be a project owner granting a role on the whitelist - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - 404: - description: User not found - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - 500: - description: Rawls Internal Error - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - security: - - authorization: - - openid - - email - - profile - delete: - tags: - - billing - summary: remove a google role from a user and their pet in the billing project - the caller owns - description: remove a google role from a user and their pet in the billing project - the caller owns - operationId: removeGoogleRoleFromUser - parameters: - - name: projectId - in: path - description: Project ID - required: true - schema: - type: string - - name: role - in: path - description: google role of user for project - required: true - schema: - type: string - - name: email - in: path - description: email of user - required: true - schema: - type: string - responses: - 200: - description: Successfully Removed Google Role from User - content: {} - 403: - description: You must be a project owner removing a role on the whitelist - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - 404: - description: User not found - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - 500: - description: Rawls Internal Error - content: - 'application/json': - schema: - $ref: '#/components/schemas/ErrorReport' - security: - - authorization: - - openid - - email - - profile /api/admin/project/registration: post: tags: @@ -5421,6 +5312,8 @@ components: - name - namespace - workspaceId + - googleProject + - workspaceVersion type: object properties: attributes: @@ -5464,6 +5357,12 @@ components: workspaceId: type: string description: A UUID associated with the workspace + googleProject: + type: string + description: the google project used by the workspace for compute and storage + workspaceVersion: + type: string + description: internal use description: "" WorkspaceSubmissionStats: required: diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala index 7aba2259b1..847fb35347 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/Boot.scala @@ -254,8 +254,6 @@ object Boot extends IOApp with LazyLogging { gcsConfig.getStringList("projectTemplate.owners").asScala val projectEditors = gcsConfig.getStringList("projectTemplate.editors").asScala - val projectOwnerGrantableRoles = - gcsConfig.getStringList("projectTemplate.ownerGrantableRoles") val requesterPaysRole = gcsConfig.getString("requesterPaysRole") val projectTemplate = ProjectTemplate(projectOwners, projectEditors) @@ -274,7 +272,6 @@ object Boot extends IOApp with LazyLogging { gcsDAO, notificationDAO, samDAO, - projectOwnerGrantableRoles.asScala, requesterPaysRole, dmConfig, projectTemplate diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala index 8f5ed270ec..aa817dfb2f 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/GoogleServicesDAO.scala @@ -28,9 +28,9 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { val billingGroupEmail: String // returns bucket and group information - def setupWorkspace(userInfo: UserInfo, projectName: RawlsBillingProjectName, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null): Future[GoogleWorkspaceInfo] + def setupWorkspace(userInfo: UserInfo, googleProject: GoogleProjectId, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null): Future[GoogleWorkspaceInfo] - def getGoogleProject(projectName: RawlsBillingProjectName): Future[Project] + def getGoogleProject(googleProject: GoogleProjectId): Future[Project] /** Mark all objects in the bucket for deletion, then attempts to delete the bucket from Google Cloud Storage. * @@ -46,9 +46,7 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { */ def deleteBucket(bucketName: String): Future[Boolean] - def getCromwellAuthBucketName(billingProject: RawlsBillingProjectName) = s"cromwell-auth-${billingProject.value}" - - def getStorageLogsBucketName(billingProject: RawlsBillingProjectName) = s"storage-logs-${billingProject.value}" + def getStorageLogsBucketName(googleProject: GoogleProjectId) = s"storage-logs-${googleProject.value}" def isAdmin(userEmail: String): Future[Boolean] @@ -72,12 +70,12 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { * with. For that reason, the maxResults parameter should be removed in favor of extracting the creation of Storage * objects from the service implementation to enable test doubles to be injected. * - * @param projectName the name of the project that owns the bucket + * @param googleProject the name of the project that owns the bucket * @param bucketName the name of the bucket to query * @param maxResults (optional) the page size to use when fetching objects * @return the size in bytes of the data stored in the bucket */ - def getBucketUsage(projectName: RawlsBillingProjectName, bucketName: String, maxResults: Option[Long] = None): Future[BigInt] + def getBucketUsage(googleProject: GoogleProjectId, bucketName: String, maxResults: Option[Long] = None): Future[BigInt] /** * Gets a Google bucket. @@ -138,14 +136,13 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { def checkGenomicsOperationsHealth(implicit executionContext: ExecutionContext): Future[Boolean] def toGoogleGroupName(groupName: RawlsGroupName): String - def toBillingProjectGroupName(billingProjectName: RawlsBillingProjectName, role: ProjectRoles.ProjectRole) = s"PROJECT_${billingProjectName.value}-${role.toString}" def getUserCredentials(rawlsUserRef: RawlsUserRef): Future[Option[Credential]] def getBucketServiceAccountCredential: Credential def getServiceAccountRawlsUser(): Future[RawlsUser] def getServiceAccountUserInfo(): Future[UserInfo] - def getBucketDetails(bucket: String, project: RawlsBillingProjectName): Future[WorkspaceBucketOptions] + def getBucketDetails(bucket: String, project: GoogleProjectId): Future[WorkspaceBucketOptions] /** * The project creation process is now mostly handled by Deployment Manager. @@ -156,18 +153,18 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { * - Polling is handled by CreatingBillingProjectMonitor. Once the deployment is completed, CBPM deletes the deployment, as * there is a per-project limit on number of deployments, and then marks the project as fully created. */ - def createProject(projectName: RawlsBillingProjectName, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] + def createProject(googleProject: GoogleProjectId, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] /** * */ - def cleanupDMProject(projectName: RawlsBillingProjectName): Future[Unit] + def cleanupDMProject(googleProject: GoogleProjectId): Future[Unit] /** * Removes the IAM policies from the project's existing policies * @return true if the policy was actually changed */ - def removePolicyBindings(projectName: RawlsBillingProjectName, policiesToRemove: Map[String, Set[String]]): Future[Boolean] = updatePolicyBindings(projectName) { existingPolicies => + def removePolicyBindings(googleProject: GoogleProjectId, policiesToRemove: Map[String, Set[String]]): Future[Boolean] = updatePolicyBindings(googleProject) { existingPolicies => val updatedKeysWithRemovedPolicies: Map[String, Set[String]] = policiesToRemove.keys.map { k => val existingForKey = existingPolicies.get(k).getOrElse(Set.empty) val updatedForKey = existingForKey diff policiesToRemove(k) @@ -182,7 +179,7 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { * Adds the IAM policies to the project's existing policies * @return true if the policy was actually changed */ - def addPolicyBindings(projectName: RawlsBillingProjectName, policiesToAdd: Map[String, Set[String]]): Future[Boolean] = updatePolicyBindings(projectName) { existingPolicies => + def addPolicyBindings(googleProject: GoogleProjectId, policiesToAdd: Map[String, Set[String]]): Future[Boolean] = updatePolicyBindings(googleProject) { existingPolicies => // |+| is a semigroup: it combines a map's keys by combining their values' members instead of replacing them import cats.implicits._ existingPolicies |+| policiesToAdd @@ -190,29 +187,23 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { /** * Internal function to update project IAM bindings. - * @param projectName google project name + * @param googleProject google project name * @param updatePolicies function (existingPolicies => updatedPolicies). May return policies with no members * which will be handled appropriately when sent to google. * @return true if google was called to update policies, false otherwise */ - protected def updatePolicyBindings(projectName: RawlsBillingProjectName)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] + protected def updatePolicyBindings(googleProject: GoogleProjectId)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] /** * - * @param billingProject * @param bucketName * @param readers emails of users to be granted read access * @return bucket name */ - def grantReadAccess(billingProject: RawlsBillingProjectName, - bucketName: String, - readers: Set[WorkbenchEmail]): Future[String] + def grantReadAccess(bucketName: String, readers: Set[WorkbenchEmail]): Future[String] def pollOperation(operationId: OperationId): Future[OperationStatus] - def deleteProject(projectName: RawlsBillingProjectName): Future[Unit] - - def addRoleToGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] - def removeRoleFromGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] + def deleteProject(googleProject: GoogleProjectId): Future[Unit] def getAccessTokenUsingJson(saKey: String) : Future[String] def getUserInfoUsingJson(saKey: String): Future[UserInfo] @@ -222,7 +213,7 @@ abstract class GoogleServicesDAO(groupsPrefix: String) extends ErrorReportable { prefix + s.toLowerCase.replaceAll("[^a-z0-9\\-_]", "-").take(63) } - def addProjectToFolder(projectName: RawlsBillingProjectName, folderId: String): Future[Unit] + def addProjectToFolder(googleProject: GoogleProjectId, folderId: String): Future[Unit] def getFolderId(folderName: String): Future[Option[String]] } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala index 0d3f8e40bb..f3e8bef46a 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpGoogleServicesDAO.scala @@ -146,7 +146,7 @@ class HttpGoogleServicesDAO( //we only have to do this once, because there's only one DM project lazy val getDeploymentManagerSAEmail: Future[String] = { - getGoogleProject(RawlsBillingProjectName(deploymentMgrProject)) + getGoogleProject(GoogleProjectId(deploymentMgrProject)) .map( p => s"${p.getProjectNumber}@cloudservices.gserviceaccount.com") } @@ -186,7 +186,7 @@ class HttpGoogleServicesDAO( executeGoogleRequest(storage.bucketAccessControls.insert(bucketName, bac)) } - override def setupWorkspace(userInfo: UserInfo, projectName: RawlsBillingProjectName, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null): Future[GoogleWorkspaceInfo] = { + override def setupWorkspace(userInfo: UserInfo, googleProject: GoogleProjectId, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null): Future[GoogleWorkspaceInfo] = { def updateBucketIam(policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail]): Stream[IO, Unit] = { //default object ACLs are no longer used. bucket only policy is enabled on buckets to ensure that objects //do not have separate permissions that deviate from the bucket-level permissions. @@ -226,7 +226,7 @@ class HttpGoogleServicesDAO( |""".stripMargin.getBytes)) // use an object name that will always be superseded by a real storage log val storageObject = new StorageObject().setName(s"${bucketName}_storage_00_initial_log") - val objectInserter = getStorage(getBucketServiceAccountCredential).objects().insert(getStorageLogsBucketName(projectName), storageObject, stream) + val objectInserter = getStorage(getBucketServiceAccountCredential).objects().insert(getStorageLogsBucketName(googleProject), storageObject, stream) executeGoogleRequest(objectInserter) } } @@ -236,7 +236,7 @@ class HttpGoogleServicesDAO( val traceId = TraceId(UUID.randomUUID()) for { - _ <- traceWithParent("insertBucket", parentSpan)(_ => googleStorageService.insertBucket(GoogleProject(projectName.value), GcsBucketName(bucketName), None, labels, Option(traceId), true, Option(GcsBucketName(getStorageLogsBucketName(projectName)))).compile.drain.unsafeToFuture()) //ACL = None because bucket IAM will be set separately in updateBucketIam + _ <- traceWithParent("insertBucket", parentSpan)(_ => googleStorageService.insertBucket(GoogleProject(googleProject.value), GcsBucketName(bucketName), None, labels, Option(traceId), true, Option(GcsBucketName(getStorageLogsBucketName(googleProject)))).compile.drain.unsafeToFuture()) //ACL = None because bucket IAM will be set separately in updateBucketIam updateBucketIamFuture = traceWithParent("updateBucketIam", parentSpan)(_ => updateBucketIam(policyGroupsByAccessLevel).compile.drain.unsafeToFuture()) insertInitialStorageLogFuture = traceWithParent("insertInitialStorageLog", parentSpan)(_ => insertInitialStorageLog(bucketName)) _ <- updateBucketIamFuture @@ -244,32 +244,7 @@ class HttpGoogleServicesDAO( } yield GoogleWorkspaceInfo(bucketName, policyGroupsByAccessLevel) } - def createCromwellAuthBucket(billingProject: RawlsBillingProjectName, projectNumber: Long, authBucketReaders: Set[WorkbenchEmail]): Future[String] = { - implicit val service = GoogleInstrumentedService.Storage - val bucketName = getCromwellAuthBucketName(billingProject) - retryWithRecoverWhen500orGoogleError( - () => { - val bucketAcls = List( - newBucketAccessControl("project-editors-" + projectNumber, "OWNER"), - newBucketAccessControl("project-owners-" + projectNumber, "OWNER")) ++ - authBucketReaders.map(email => newBucketAccessControl(makeGroupEntityString(email.value), "READER")).toList - - val defaultObjectAcls = List( - newObjectAccessControl("project-editors-" + projectNumber, "OWNER"), - newObjectAccessControl("project-owners-" + projectNumber, "OWNER")) ++ - authBucketReaders.map(email => newObjectAccessControl(makeGroupEntityString(email.value), "READER")).toList - - val bucket = new Bucket().setName(bucketName).setAcl(bucketAcls.asJava).setDefaultObjectAcl(defaultObjectAcls.asJava) - val inserter = getStorage(getBucketServiceAccountCredential).buckets.insert(billingProject.value, bucket) - executeGoogleRequest(inserter) - - bucketName - }) { case t: HttpResponseException if t.getStatusCode == 409 => bucketName } - } - - def grantReadAccess(billingProject: RawlsBillingProjectName, - bucketName: String, - authBucketReaders: Set[WorkbenchEmail]): Future[String] = { + def grantReadAccess(bucketName: String, authBucketReaders: Set[WorkbenchEmail]): Future[String] = { implicit val service = GoogleInstrumentedService.Storage def insertNewAcls() = for { @@ -292,27 +267,6 @@ class HttpGoogleServicesDAO( } } - def createStorageLogsBucket(billingProject: RawlsBillingProjectName): Future[String] = { - implicit val service = GoogleInstrumentedService.Storage - val bucketName = getStorageLogsBucketName(billingProject) - logger debug s"storage log bucket: $bucketName" - - retryWithRecoverWhen500orGoogleError(() => { - val bucket = new Bucket().setName(bucketName) - val storageLogExpiration = new Lifecycle.Rule() - .setAction(new Action().setType("Delete")) - .setCondition(new Condition().setAge(bucketLogsMaxAge)) - bucket.setLifecycle(new Lifecycle().setRule(List(storageLogExpiration).asJava)) - val inserter = getStorage(getBucketServiceAccountCredential).buckets().insert(billingProject.value, bucket) - executeGoogleRequest(inserter) - - bucketName - }) { - // bucket already exists - case t: HttpResponseException if t.getStatusCode == 409 => bucketName - } - } - private def newBucketAccessControl(entity: String, accessLevel: String) = new BucketAccessControl().setEntity(entity).setRole(accessLevel) @@ -387,7 +341,7 @@ class HttpGoogleServicesDAO( } } - override def getBucketUsage(projectName: RawlsBillingProjectName, bucketName: String, maxResults: Option[Long]): Future[BigInt] = { + override def getBucketUsage(googleProject: GoogleProjectId, bucketName: String, maxResults: Option[Long]): Future[BigInt] = { implicit val service = GoogleInstrumentedService.Storage def usageFromLogObject(o: StorageObject): Future[BigInt] = { @@ -403,7 +357,7 @@ class HttpGoogleServicesDAO( // Fetch objects with a prefix of "${bucketName}_storage_", (ignoring "_usage_" logs) val fetcher = getStorage(getBucketServiceAccountCredential). objects(). - list(getStorageLogsBucketName(projectName)). + list(getStorageLogsBucketName(googleProject)). setPrefix(s"${bucketName}_storage_") maxResults.foreach(fetcher.setMaxResults(_)) pageToken.foreach(fetcher.setPageToken) @@ -750,23 +704,23 @@ class HttpGoogleServicesDAO( }) } - override def getGoogleProject(projectName: RawlsBillingProjectName): Future[Project] = { + override def getGoogleProject(googleProject: GoogleProjectId): Future[Project] = { implicit val service = GoogleInstrumentedService.Billing val credential = getDeploymentManagerAccountCredential val cloudResManager = getCloudResourceManager(credential) retryWhen500orGoogleError(() => { - executeGoogleRequest(cloudResManager.projects().get(projectName.value)) + executeGoogleRequest(cloudResManager.projects().get(googleProject.value)) }) } - def getDMConfigYamlString(projectName: RawlsBillingProjectName, dmTemplatePath: String, properties: Map[String, JsValue]): String = { + def getDMConfigYamlString(googleProject: GoogleProjectId, dmTemplatePath: String, properties: Map[String, JsValue]): String = { import DeploymentManagerJsonSupport._ import cats.syntax.either._ import io.circe.yaml.syntax._ - val configContents = ConfigContents(Seq(Resources(projectName.value, dmTemplatePath, properties))) + val configContents = ConfigContents(Seq(Resources(googleProject.value, dmTemplatePath, properties))) val jsonVersion = io.circe.jawn.parse(configContents.toJson.toString).valueOr(throw _) jsonVersion.asYaml.spaces2 } @@ -775,20 +729,20 @@ class HttpGoogleServicesDAO( * Set the deployment policy to "abandon" -- i.e. allows the created project to persist even if the deployment is deleted -- * and then delete the deployment. There's a limit of 1000 deployments so this is important to do. */ - override def cleanupDMProject(projectName: RawlsBillingProjectName): Future[Unit] = { + override def cleanupDMProject(googleProject: GoogleProjectId): Future[Unit] = { implicit val service = GoogleInstrumentedService.DeploymentManager val credential = getDeploymentManagerAccountCredential val deploymentManager = getDeploymentManager(credential) if( cleanupDeploymentAfterCreating ) { executeGoogleRequestWithRetry( - deploymentManager.deployments().delete(deploymentMgrProject, projectToDM(projectName)).setDeletePolicy("ABANDON")).void + deploymentManager.deployments().delete(deploymentMgrProject, projectToDM(googleProject)).setDeletePolicy("ABANDON")).void } else { Future.successful(()) } } - def projectToDM(projectName: RawlsBillingProjectName) = s"dm-${projectName.value}" + def projectToDM(googleProject: GoogleProjectId) = s"dm-${googleProject.value}" def parseTemplateLocation(path: String): Option[TemplateLocation] = { @@ -802,7 +756,7 @@ class HttpGoogleServicesDAO( } } - override def createProject(projectName: RawlsBillingProjectName, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = { + override def createProject(googleProject: GoogleProjectId, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = { implicit val service = GoogleInstrumentedService.DeploymentManager val credential = getDeploymentManagerAccountCredential val deploymentManager = getDeploymentManager(credential) @@ -816,7 +770,7 @@ class HttpGoogleServicesDAO( val properties = Map ( "billingAccountId" -> billingAccount.accountName.value.toJson, "billingAccountFriendlyName" -> billingAccount.displayName.toJson, - "projectId" -> projectName.value.toJson, + "projectId" -> googleProject.value.toJson, "parentOrganization" -> orgID.toJson, "fcBillingGroup" -> billingGroupEmail.toJson, "projectOwnersGroup" -> ownerGroupEmail.value.toJson, @@ -831,16 +785,16 @@ class HttpGoogleServicesDAO( ) ++ parentFolderId.map("parentFolder" -> folderNumberOnly(_).toJson).toMap //a list of one resource: type=composite-type, name=whocares, properties=pokein - val yamlConfig = new ConfigFile().setContent(getDMConfigYamlString(projectName, dmTemplatePath, properties)) + val yamlConfig = new ConfigFile().setContent(getDMConfigYamlString(googleProject, dmTemplatePath, properties)) val deploymentConfig = new TargetConfiguration().setConfig(yamlConfig) retryWhen500orGoogleError(() => { executeGoogleRequest { - deploymentManager.deployments().insert(deploymentMgrProject, new Deployment().setName(projectToDM(projectName)).setTarget(deploymentConfig)) + deploymentManager.deployments().insert(deploymentMgrProject, new Deployment().setName(projectToDM(googleProject)).setTarget(deploymentConfig)) } }) map { googleOperation => val errorStr = Option(googleOperation.getError).map(errors => errors.getErrors.asScala.map(e => toErrorMessage(e.getMessage, e.getCode)).mkString("\n")) - RawlsBillingProjectOperationRecord(projectName.value, GoogleOperationNames.DeploymentManagerCreateProject, googleOperation.getName, false, errorStr, GoogleApiTypes.DeploymentManagerApi) + RawlsBillingProjectOperationRecord(googleProject.value, GoogleOperationNames.DeploymentManagerCreateProject, googleOperation.getName, false, errorStr, GoogleApiTypes.DeploymentManagerApi) } } @@ -898,12 +852,12 @@ class HttpGoogleServicesDAO( * 3) if updated policies are the same as existing policies return false, don't call google * 4) if updated policies are different than existing policies update google and return true * - * @param projectName google project name + * @param googleProject google project name * @param updatePolicies function (existingPolicies => updatedPolicies). May return policies with no members * which will be handled appropriately when sent to google. * @return true if google was called to update policies, false otherwise */ - override protected def updatePolicyBindings(projectName: RawlsBillingProjectName)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] = { + override protected def updatePolicyBindings(googleProject: GoogleProjectId)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] = { val cloudResManager = getCloudResourceManager(getBillingServiceAccountCredential) implicit val service = GoogleInstrumentedService.CloudResourceManager @@ -912,7 +866,7 @@ class HttpGoogleServicesDAO( // it is important that we call getIamPolicy within the same retry block as we call setIamPolicy // getIamPolicy gets the etag that is used in setIamPolicy, the etag is used to detect concurrent // modifications and if that happens we need to be sure to get a new etag before retrying setIamPolicy - val existingPolicy = executeGoogleRequest(cloudResManager.projects().getIamPolicy(projectName.value, null)) + val existingPolicy = executeGoogleRequest(cloudResManager.projects().getIamPolicy(googleProject.value, null)) val existingPolicies: Map[String, Set[String]] = existingPolicy.getBindings.asScala.map { policy => policy.getRole -> policy.getMembers.asScala.toSet }.toMap val updatedPolicies = updatePolicies(existingPolicies) @@ -926,7 +880,7 @@ class HttpGoogleServicesDAO( // when setting IAM policies, always reuse the existing policy so the etag is preserved. val policyRequest = new SetIamPolicyRequest().setPolicy(existingPolicy.setBindings(updatedBindings.asJava)) - executeGoogleRequest(cloudResManager.projects().setIamPolicy(projectName.value, policyRequest)) + executeGoogleRequest(cloudResManager.projects().setIamPolicy(googleProject.value, policyRequest)) true } }) @@ -934,26 +888,17 @@ class HttpGoogleServicesDAO( } yield updated } - override def addRoleToGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] = { - addPolicyBindings(projectName, Map(s"roles/$role" -> Set(s"group:${groupEmail.value}"))) - } - - override def removeRoleFromGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] = { - removePolicyBindings(projectName, Map(s"roles/$role" -> Set(s"group:${groupEmail.value}"))) - } - - override def deleteProject(projectName: RawlsBillingProjectName): Future[Unit]= { + override def deleteProject(googleProject: GoogleProjectId): Future[Unit]= { implicit val service = GoogleInstrumentedService.Billing val billingServiceAccountCredential = getBillingServiceAccountCredential val resMgr = getCloudResourceManager(billingServiceAccountCredential) val billingManager = getCloudBillingManager(billingServiceAccountCredential) - val projectNameString = projectName.value for { _ <- retryWhen500orGoogleError(() => { - executeGoogleRequest(billingManager.projects().updateBillingInfo(s"projects/${projectName.value}", new ProjectBillingInfo().setBillingEnabled(false))) + executeGoogleRequest(billingManager.projects().updateBillingInfo(s"projects/${googleProject.value}", new ProjectBillingInfo().setBillingEnabled(false))) }) _ <- retryWithRecoverWhen500orGoogleError(() => { - executeGoogleRequest(resMgr.projects().delete(projectNameString)) + executeGoogleRequest(resMgr.projects().delete(googleProject.value)) }) { case e: GoogleJsonResponseException if e.getDetails.getCode == 403 && "Cannot delete an inactive project.".equals(e.getDetails.getMessage) => new Empty() // stop trying to delete an already deleted project @@ -963,9 +908,9 @@ class HttpGoogleServicesDAO( } } - def projectUsageExportBucketName(projectName: RawlsBillingProjectName) = s"${projectName.value}-usage-export" + def projectUsageExportBucketName(googleProject: GoogleProjectId) = s"${googleProject.value}-usage-export" - override def getBucketDetails(bucketName: String, project: RawlsBillingProjectName): Future[WorkspaceBucketOptions] = { + override def getBucketDetails(bucketName: String, project: GoogleProjectId): Future[WorkspaceBucketOptions] = { implicit val service = GoogleInstrumentedService.Storage val cloudStorage = getStorage(getBucketServiceAccountCredential) for { @@ -1149,15 +1094,15 @@ class HttpGoogleServicesDAO( retryWhen500orGoogleError(() => { executeGoogleFetch(getter) { is => f(is) } }) } - override def addProjectToFolder(projectName: RawlsBillingProjectName, folderId: String): Future[Unit] = { + override def addProjectToFolder(googleProject: GoogleProjectId, folderId: String): Future[Unit] = { implicit val service = GoogleInstrumentedService.CloudResourceManager val cloudResourceManager = getCloudResourceManager(getBillingServiceAccountCredential) retryWhen500orGoogleError( () => { - val existingProject = executeGoogleRequest(cloudResourceManager.projects().get(projectName.value)) + val existingProject = executeGoogleRequest(cloudResourceManager.projects().get(googleProject.value)) val folderResourceId = new ResourceId().setType(GoogleResourceTypes.Folder.value).setId(folderNumberOnly(folderId)) - executeGoogleRequest(cloudResourceManager.projects().update(projectName.value, existingProject.setParent(folderResourceId))) + executeGoogleRequest(cloudResourceManager.projects().update(googleProject.value, existingProject.setParent(folderResourceId))) }) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala index 7606d95b9f..64398ff3f4 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/HttpSamDAO.scala @@ -228,13 +228,13 @@ class HttpSamDAO(baseSamServiceURL: String, serviceAccountCreds: Credential)(imp retry(when401or500) { () => pipeline[Set[SamResourceIdWithPolicyName]](userInfo) apply RequestBuilding.Get(url) } } - override def getPetServiceAccountKeyForUser(googleProject: String, userEmail: RawlsUserEmail): Future[String] = { - val url = samServiceURL + s"/api/google/v1/petServiceAccount/$googleProject/${URLEncoder.encode(userEmail.value, UTF_8.name)}" + override def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail): Future[String] = { + val url = samServiceURL + s"/api/google/v1/petServiceAccount/${googleProject.value}/${URLEncoder.encode(userEmail.value, UTF_8.name)}" retry(when401or500) { () => asRawlsSAPipeline[String] apply RequestBuilding.Get(url) } } - override def deleteUserPetServiceAccount(googleProject: String, userInfo: UserInfo): Future[Unit] = { - val url = samServiceURL + s"/api/google/v1/user/petServiceAccount/$googleProject" + override def deleteUserPetServiceAccount(googleProject: GoogleProjectId, userInfo: UserInfo): Future[Unit] = { + val url = samServiceURL + s"/api/google/v1/user/petServiceAccount/${googleProject.value}" doSuccessOrFailureRequest(RequestBuilding.Delete(url), userInfo) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupService.scala index 7427f05a58..82e671e76e 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupService.scala @@ -1,6 +1,6 @@ package org.broadinstitute.dsde.rawls.dataaccess -import org.broadinstitute.dsde.rawls.model.{RawlsBillingProjectName, RawlsUserEmail, UserInfo, WorkspaceName} +import org.broadinstitute.dsde.rawls.model.{RawlsUserEmail, UserInfo, Workspace} import scala.concurrent.{ExecutionContext, Future} @@ -19,29 +19,29 @@ class RequesterPaysSetupService(dataSource: SlickDataSource, val googleServicesD } } - def grantRequesterPaysToLinkedSAs(userInfo: UserInfo, workspaceName: WorkspaceName): Future[List[BondServiceAccountEmail]] = { + def grantRequesterPaysToLinkedSAs(userInfo: UserInfo, workspace: Workspace): Future[List[BondServiceAccountEmail]] = { for { emails <- getBondProviderServiceAccountEmails(userInfo) - _ <- googleServicesDAO.addPolicyBindings(RawlsBillingProjectName(workspaceName.namespace), Map(requesterPaysRole -> emails.toSet.map{mail:BondServiceAccountEmail => "serviceAccount:" + mail.client_email})) - _ <- dataSource.inTransaction { dataAccess => dataAccess.workspaceRequesterPaysQuery.insertAllForUser(workspaceName, userInfo.userEmail, emails.toSet) } + _ <- googleServicesDAO.addPolicyBindings(workspace.googleProject, Map(requesterPaysRole -> emails.toSet.map{mail:BondServiceAccountEmail => "serviceAccount:" + mail.client_email})) + _ <- dataSource.inTransaction { dataAccess => dataAccess.workspaceRequesterPaysQuery.insertAllForUser(workspace.toWorkspaceName, userInfo.userEmail, emails.toSet) } } yield { emails } } - def revokeUserFromWorkspace(userEmail: RawlsUserEmail, workspaceName: WorkspaceName): Future[Seq[BondServiceAccountEmail]] = { + def revokeUserFromWorkspace(userEmail: RawlsUserEmail, workspace: Workspace): Future[Seq[BondServiceAccountEmail]] = { for { - emails <- dataSource.inTransaction { dataAccess => dataAccess.workspaceRequesterPaysQuery.listAllForUser(workspaceName, userEmail) } - _ <- revokeEmails(emails.toSet, userEmail, workspaceName) + emails <- dataSource.inTransaction { dataAccess => dataAccess.workspaceRequesterPaysQuery.listAllForUser(workspace.toWorkspaceName, userEmail) } + _ <- revokeEmails(emails.toSet, userEmail, workspace) } yield emails.map(BondServiceAccountEmail) } - private def revokeEmails(emails: Set[String], userEmail: RawlsUserEmail, workspaceName: WorkspaceName): Future[Unit] = { + private def revokeEmails(emails: Set[String], userEmail: RawlsUserEmail, workspace: Workspace): Future[Unit] = { for { keepBindings <- dataSource.inTransaction { dataAccess => for { - _ <- dataAccess.workspaceRequesterPaysQuery.deleteAllForUser(workspaceName, userEmail) - keepBindings <- dataAccess.workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(workspaceName.namespace, userEmail) + _ <- dataAccess.workspaceRequesterPaysQuery.deleteAllForUser(workspace.toWorkspaceName, userEmail) + keepBindings <- dataAccess.workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(workspace.namespace, userEmail) } yield keepBindings } @@ -49,7 +49,7 @@ class RequesterPaysSetupService(dataSource: SlickDataSource, val googleServicesD _ <- if (keepBindings) { Future.successful(()) } else { - googleServicesDAO.removePolicyBindings(RawlsBillingProjectName(workspaceName.namespace), Map(requesterPaysRole -> emails.map("serviceAccount:" + _))) + googleServicesDAO.removePolicyBindings(workspace.googleProject, Map(requesterPaysRole -> emails.map("serviceAccount:" + _))) } } yield () } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala index 3f54283b27..2779d25766 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/SamDAO.scala @@ -1,6 +1,6 @@ package org.broadinstitute.dsde.rawls.dataaccess -import org.broadinstitute.dsde.rawls.model.{RawlsUser, RawlsUserEmail, SamCreateResourceResponse, SamPolicy, SamPolicySyncStatus, SamPolicyWithNameAndEmail, SamResourceAction, SamResourceIdWithPolicyName, SamResourcePolicyName, SamResourceRole, SamResourceTypeName, SubsystemStatus, SyncReportItem, UserIdInfo, UserInfo} +import org.broadinstitute.dsde.rawls.model.{GoogleProjectId, RawlsUser, RawlsUserEmail, SamCreateResourceResponse, SamPolicy, SamPolicySyncStatus, SamPolicyWithNameAndEmail, SamResourceAction, SamResourceIdWithPolicyName, SamResourcePolicyName, SamResourceRole, SamResourceTypeName, SubsystemStatus, SyncReportItem, UserIdInfo, UserInfo} import org.broadinstitute.dsde.workbench.model._ import scala.concurrent.Future @@ -36,9 +36,9 @@ trait SamDAO { /** * @return a json blob */ - def getPetServiceAccountKeyForUser(googleProject: String, userEmail: RawlsUserEmail): Future[String] + def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail): Future[String] def getDefaultPetServiceAccountKeyForUser(userInfo: UserInfo): Future[String] - def deleteUserPetServiceAccount(googleProject: String, userInfo: UserInfo): Future[Unit] + def deleteUserPetServiceAccount(googleProject: GoogleProjectId, userInfo: UserInfo): Future[Unit] def getStatus(): Future[SubsystemStatus] } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala index d2b6ac14be..a65a3f266b 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/RawlsBillingProjectComponent.scala @@ -7,7 +7,7 @@ import org.broadinstitute.dsde.rawls.dataaccess.GoogleOperationNames.GoogleOpera import org.broadinstitute.dsde.rawls.model.CreationStatuses.CreationStatus import org.broadinstitute.dsde.rawls.model._ -case class RawlsBillingProjectRecord(projectName: String, cromwellAuthBucketUrl: String, creationStatus: String, billingAccount: Option[String], message: Option[String], cromwellBackend: Option[String], servicePerimeter: Option[String], googleProjectNumber: Option[String], invalidBillingAccount: Boolean) +case class RawlsBillingProjectRecord(projectName: String, creationStatus: String, billingAccount: Option[String], message: Option[String], cromwellBackend: Option[String], servicePerimeter: Option[String], googleProjectNumber: Option[String], invalidBillingAccount: Boolean) case class RawlsBillingProjectOperationRecord(projectName: String, operationName: GoogleOperationName, operationId: String, done: Boolean, errorMessage: Option[String], api: GoogleApiType) trait RawlsBillingProjectComponent { @@ -17,7 +17,6 @@ trait RawlsBillingProjectComponent { class RawlsBillingProjectTable(tag: Tag) extends Table[RawlsBillingProjectRecord](tag, "BILLING_PROJECT") { def projectName = column[String]("NAME", O.PrimaryKey, O.Length(254)) - def cromwellAuthBucketUrl = column[String]("CROMWELL_BUCKET_URL", O.Length(128)) def creationStatus = column[String]("CREATION_STATUS", O.Length(20)) def billingAccount = column[Option[String]]("BILLING_ACCOUNT", O.Length(100)) def message = column[Option[String]]("MESSAGE") @@ -26,7 +25,7 @@ trait RawlsBillingProjectComponent { def googleProjectNumber = column[Option[String]]("GOOGLE_PROJECT_NUMBER") def invalidBillingAccount = column[Boolean]("INVALID_BILLING_ACCT") - def * = (projectName, cromwellAuthBucketUrl, creationStatus, billingAccount, message, cromwellBackend, servicePerimeter, googleProjectNumber, invalidBillingAccount) <> (RawlsBillingProjectRecord.tupled, RawlsBillingProjectRecord.unapply) + def * = (projectName, creationStatus, billingAccount, message, cromwellBackend, servicePerimeter, googleProjectNumber, invalidBillingAccount) <> (RawlsBillingProjectRecord.tupled, RawlsBillingProjectRecord.unapply) } // these 2 implicits are lazy because there is a timing problem initializing MappedColumnType, if they are not lazy @@ -135,11 +134,11 @@ trait RawlsBillingProjectComponent { } private def marshalBillingProject(billingProject: RawlsBillingProject): RawlsBillingProjectRecord = { - RawlsBillingProjectRecord(billingProject.projectName.value, billingProject.cromwellAuthBucketUrl, billingProject.status.toString, billingProject.billingAccount.map(_.value), billingProject.message, billingProject.cromwellBackend.map(_.value), billingProject.servicePerimeter.map(_.value), billingProject.googleProjectNumber.map(_.value), billingProject.invalidBillingAccount) + RawlsBillingProjectRecord(billingProject.projectName.value, billingProject.status.toString, billingProject.billingAccount.map(_.value), billingProject.message, billingProject.cromwellBackend.map(_.value), billingProject.servicePerimeter.map(_.value), billingProject.googleProjectNumber.map(_.value), billingProject.invalidBillingAccount) } private def unmarshalBillingProject(projectRecord: RawlsBillingProjectRecord): RawlsBillingProject = { - RawlsBillingProject(RawlsBillingProjectName(projectRecord.projectName), projectRecord.cromwellAuthBucketUrl, CreationStatuses.withName(projectRecord.creationStatus), projectRecord.billingAccount.map(RawlsBillingAccountName), projectRecord.message, projectRecord.cromwellBackend.map(CromwellBackend), projectRecord.servicePerimeter.map(ServicePerimeterName), projectRecord.googleProjectNumber.map(GoogleProjectNumber), projectRecord.invalidBillingAccount) + RawlsBillingProject(RawlsBillingProjectName(projectRecord.projectName), CreationStatuses.withName(projectRecord.creationStatus), projectRecord.billingAccount.map(RawlsBillingAccountName), projectRecord.message, projectRecord.cromwellBackend.map(CromwellBackend), projectRecord.servicePerimeter.map(ServicePerimeterName), projectRecord.googleProjectNumber.map(GoogleProjectNumber), projectRecord.invalidBillingAccount) } private def findBillingProjectByName(name: RawlsBillingProjectName): RawlsBillingProjectQuery = { diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala index f380c9a48d..c41248ae46 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponent.scala @@ -390,11 +390,11 @@ trait WorkspaceComponent { } private def marshalNewWorkspace(workspace: Workspace) = { - WorkspaceRecord(workspace.namespace, workspace.name, UUID.fromString(workspace.workspaceId), workspace.bucketName, workspace.workflowCollectionName, new Timestamp(workspace.createdDate.getMillis), new Timestamp(workspace.lastModified.getMillis), workspace.createdBy, workspace.isLocked, 0, workspace.workspaceVersion.value, workspace.googleProject) + WorkspaceRecord(workspace.namespace, workspace.name, UUID.fromString(workspace.workspaceId), workspace.bucketName, workspace.workflowCollectionName, new Timestamp(workspace.createdDate.getMillis), new Timestamp(workspace.lastModified.getMillis), workspace.createdBy, workspace.isLocked, 0, workspace.workspaceVersion.value, workspace.googleProject.value) } private def unmarshalWorkspace(workspaceRec: WorkspaceRecord, attributes: AttributeMap): Workspace = { - Workspace(workspaceRec.namespace, workspaceRec.name, workspaceRec.id.toString, workspaceRec.bucketName, workspaceRec.workflowCollection, new DateTime(workspaceRec.createdDate), new DateTime(workspaceRec.lastModified), workspaceRec.createdBy, attributes, workspaceRec.isLocked, WorkspaceVersions.fromString(workspaceRec.workspaceVersion).getOrElse(throw new RawlsException(s"unexpected version string ${workspaceRec.workspaceVersion}")), workspaceRec.googleProject) + Workspace(workspaceRec.namespace, workspaceRec.name, workspaceRec.id.toString, workspaceRec.bucketName, workspaceRec.workflowCollection, new DateTime(workspaceRec.createdDate), new DateTime(workspaceRec.lastModified), workspaceRec.createdBy, attributes, workspaceRec.isLocked, WorkspaceVersions.fromStringThrows(workspaceRec.workspaceVersion), GoogleProjectId(workspaceRec.googleProject)) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala index 5afcc571b4..c2fbe4aafe 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProvider.scala @@ -37,8 +37,8 @@ class DataRepoEntityProvider(snapshotModel: SnapshotModel, requestArguments: Ent private lazy val googleProject = { // determine project to be billed for the BQ job TODO: need business logic from PO! requestArguments.billingProject match { - case Some(billing) => billing.projectName.value - case None => requestArguments.workspace.namespace + case Some(billing) => billing.googleProjectId + case None => requestArguments.workspace.googleProject } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorActor.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorActor.scala index 77c3346269..4e41336dff 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorActor.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionMonitorActor.scala @@ -183,9 +183,9 @@ trait SubmissionMonitor extends FutureSupport with LazyLogging with RawlsInstrum } } - def getPetSAUserInfo(workspaceNamespace: String, submitterEmail: RawlsUserEmail): Future[UserInfo] = { + def getPetSAUserInfo(googleProjectId: GoogleProjectId, submitterEmail: RawlsUserEmail): Future[UserInfo] = { for { - petSAJson <- samDAO.getPetServiceAccountKeyForUser(workspaceNamespace, submitterEmail) + petSAJson <- samDAO.getPetServiceAccountKeyForUser(googleProjectId, submitterEmail) petUserInfo <- googleServicesDAO.getUserInfoUsingJson(petSAJson) } yield { petUserInfo @@ -203,7 +203,7 @@ trait SubmissionMonitor extends FutureSupport with LazyLogging with RawlsInstrum } } flatMap { case (workflowRecs, submitter, workspaceRec) => for { - petUserInfo <- getPetSAUserInfo(workspaceRec.namespace, submitter) + petUserInfo <- getPetSAUserInfo(GoogleProjectId(workspaceRec.googleProject), submitter) abortResults <- Future.traverse(workflowRecs) { workflowRec => Future.successful(workflowRec.externalId).zip(executionServiceCluster.abort(workflowRec, petUserInfo)) } @@ -233,7 +233,7 @@ trait SubmissionMonitor extends FutureSupport with LazyLogging with RawlsInstrum } } flatMap { case (externalWorkflowIds, submitter, workspaceRec) => for { - petUserInfo <- getPetSAUserInfo(workspaceRec.namespace, submitter) + petUserInfo <- getPetSAUserInfo(GoogleProjectId(workspaceRec.googleProject), submitter) workflowOutputs <- gatherWorkflowOutputs(externalWorkflowIds, petUserInfo) } yield { workflowOutputs diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionActor.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionActor.scala index 5e4f51ce9e..8677a44cf3 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionActor.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionActor.scala @@ -203,11 +203,10 @@ trait WorkflowSubmission extends FutureSupport with LazyLogging with MethodWiths ExecutionServiceWorkflowOptions( s"gs://${workspace.bucketName}/${submissionId}", - workspace.namespace, + workspace.googleProject, userEmail.value, petSAEmail, petSAJson, - billingProject.cromwellAuthBucketUrl, s"gs://${workspace.bucketName}/${submissionId}/workflow.logs", runtimeOptions, useCallCache, @@ -314,7 +313,7 @@ trait WorkflowSubmission extends FutureSupport with LazyLogging with MethodWiths //yank things from the db. note this future has already started running and we're just waiting on it here (wfRecs, workflowBatch, billingProject, methodConfig) <- dbThingsFuture - petSAJson <- samDAO.getPetServiceAccountKeyForUser(billingProject.projectName.value, RawlsUserEmail(submissionRec.submitterEmail)) + petSAJson <- samDAO.getPetServiceAccountKeyForUser(GoogleProjectId(workspaceRec.googleProject), RawlsUserEmail(submissionRec.submitterEmail)) petUserInfo <- googleServicesDAO.getUserInfoUsingJson(petSAJson) wdl <- getWdl(methodConfig, petUserInfo) @@ -360,7 +359,7 @@ trait WorkflowSubmission extends FutureSupport with LazyLogging with MethodWiths // We still call Martha for those because we can verify the user has permission on the DRS object as // early as possible, rather than letting the workflow(s) launch and fail // AEN 2020-09-08 [WA-325] - _ <- if (dosServiceAccounts.isEmpty) Future.successful(false) else googleServicesDAO.addPolicyBindings(RawlsBillingProjectName(wfOpts.google_project), Map(requesterPaysRole -> dosServiceAccounts.map("serviceAccount:"+_))) + _ <- if (dosServiceAccounts.isEmpty) Future.successful(false) else googleServicesDAO.addPolicyBindings(GoogleProjectId(wfOpts.google_project), Map(requesterPaysRole -> dosServiceAccounts.map("serviceAccount:"+_))) // Should labels be an Option? It's not optional for rawls (but then wfOpts are options too) workflowSubmitResult <- executionServiceCluster.submitWorkflows(workflowRecs, wdl, wfInputsBatch, Option(wfOpts.toJson.toString), Option(wfLabels), wfCollection, petUserInfo) } yield { diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/ExecutionModel.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/ExecutionModel.scala index e7f8226d77..51912c0ce0 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/ExecutionModel.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/ExecutionModel.scala @@ -72,7 +72,6 @@ case class ExecutionServiceWorkflowOptions( account_name: String, google_compute_service_account: String, user_service_account_json: String, - auth_bucket: String, final_workflow_log_dir: String, default_runtime_attributes: Option[JsValue], read_from_cache: Boolean, @@ -379,7 +378,7 @@ class ExecutionJsonSupport extends JsonSupport { implicit val ExecutionServiceLogsFormat = jsonFormat2(ExecutionServiceLogs) - implicit val ExecutionServiceWorkflowOptionsFormat = jsonFormat13(ExecutionServiceWorkflowOptions) + implicit val ExecutionServiceWorkflowOptionsFormat = jsonFormat12(ExecutionServiceWorkflowOptions) implicit val ExecutionServiceLabelResponseFormat = jsonFormat2(ExecutionServiceLabelResponse) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala index 8dd616a011..454d6deeb7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/SamModel.scala @@ -71,7 +71,6 @@ object SamBillingProjectActions { val launchBatchCompute = SamResourceAction("launch_batch_compute") val alterPolicies = SamResourceAction("alter_policies") val readPolicies = SamResourceAction("read_policies") - val alterGoogleRole = SamResourceAction("alter_google_role") val addToServicePerimeter = SamResourceAction("add_to_service_perimeter") val deleteBillingProject = SamResourceAction("delete") def sharePolicy(policy: String) = SamResourceAction(s"share_policy::$policy") diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/UserAuth.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/UserAuth.scala index 0faf74a0ef..fa3dc68eac 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/model/UserAuth.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/model/UserAuth.scala @@ -44,7 +44,10 @@ object ManagedGroup { case class ManagedGroup(membersGroup: RawlsGroup, adminsGroup: RawlsGroup) extends Managed case class RawlsBillingAccount(accountName: RawlsBillingAccountName, firecloudHasAccess: Boolean, displayName: String) -case class RawlsBillingProject(projectName: RawlsBillingProjectName, cromwellAuthBucketUrl: String, status: CreationStatuses.CreationStatus, billingAccount: Option[RawlsBillingAccountName], message: Option[String], cromwellBackend: Option[CromwellBackend] = None, servicePerimeter: Option[ServicePerimeterName] = None, googleProjectNumber: Option[GoogleProjectNumber] = None, invalidBillingAccount: Boolean = false) +case class RawlsBillingProject(projectName: RawlsBillingProjectName, status: CreationStatuses.CreationStatus, billingAccount: Option[RawlsBillingAccountName], message: Option[String], cromwellBackend: Option[CromwellBackend] = None, servicePerimeter: Option[ServicePerimeterName] = None, googleProjectNumber: Option[GoogleProjectNumber] = None, invalidBillingAccount: Boolean = false) { + // def instead of val because val confuses the json formatter + def googleProjectId: GoogleProjectId = GoogleProjectId(projectName.value) +} case class RawlsBillingProjectTransfer(project: String, bucket: String, newOwnerEmail: String, newOwnerToken: String) @@ -140,7 +143,7 @@ class UserAuthJsonSupport extends JsonSupport { implicit val RawlsGroupMemberListFormat = jsonFormat4(RawlsGroupMemberList) - implicit val RawlsBillingProjectFormat = jsonFormat9(RawlsBillingProject) + implicit val RawlsBillingProjectFormat = jsonFormat8(RawlsBillingProject) implicit val RawlsBillingAccountFormat = jsonFormat3(RawlsBillingAccount) diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitor.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitor.scala index d8884db11d..1873769ad7 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitor.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitor.scala @@ -11,13 +11,13 @@ import cats.effect.{ContextShift, IO} import com.typesafe.scalalogging.LazyLogging import fs2.concurrent.SignallingRef import io.circe.fs2._ -import org.broadinstitute.dsde.rawls.RawlsExceptionWithErrorReport +import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} import org.broadinstitute.dsde.rawls.dataaccess._ import org.broadinstitute.dsde.rawls.entities.EntityService import org.broadinstitute.dsde.rawls.google.GooglePubSubDAO import org.broadinstitute.dsde.rawls.google.GooglePubSubDAO.PubSubMessage import org.broadinstitute.dsde.rawls.model.AttributeUpdateOperations._ -import org.broadinstitute.dsde.rawls.model.{Entity, ImportStatuses, RawlsUserEmail, UserInfo, WorkspaceName, ErrorReport => RawlsErrorReport} +import org.broadinstitute.dsde.rawls.model.{Entity, ImportStatuses, RawlsUserEmail, UserInfo, Workspace, WorkspaceName, ErrorReport => RawlsErrorReport} import org.broadinstitute.dsde.rawls.model.ImportStatuses.ImportStatus import org.broadinstitute.dsde.rawls.monitor.AvroUpsertMonitorSupervisor.{AvroUpsertMonitorConfig, KeepAlive, updateImportStatusFormat} import org.broadinstitute.dsde.rawls.util.AuthUtil @@ -58,7 +58,8 @@ object AvroUpsertMonitorSupervisor { pubSubDAO: GooglePubSubDAO, importServicePubSubDAO: GooglePubSubDAO, importServiceDAO: ImportServiceDAO, - avroUpsertMonitorConfig: AvroUpsertMonitorConfig)(implicit executionContext: ExecutionContext, cs: ContextShift[IO]): Props = + avroUpsertMonitorConfig: AvroUpsertMonitorConfig, + dataSource: SlickDataSource)(implicit executionContext: ExecutionContext, cs: ContextShift[IO]): Props = Props( new AvroUpsertMonitorSupervisor( entityService, @@ -68,7 +69,8 @@ object AvroUpsertMonitorSupervisor { pubSubDAO, importServicePubSubDAO, importServiceDAO, - avroUpsertMonitorConfig)) + avroUpsertMonitorConfig, + dataSource)) import spray.json.DefaultJsonProtocol._ @@ -91,7 +93,8 @@ class AvroUpsertMonitorSupervisor(entityService: UserInfo => EntityService, pubSubDAO: GooglePubSubDAO, importServicePubSubDAO: GooglePubSubDAO, importServiceDAO: ImportServiceDAO, - avroUpsertMonitorConfig: AvroUpsertMonitorConfig)(implicit cs: ContextShift[IO]) + avroUpsertMonitorConfig: AvroUpsertMonitorConfig, + dataSource: SlickDataSource)(implicit cs: ContextShift[IO]) extends Actor with LazyLogging { import AvroUpsertMonitorSupervisor._ @@ -113,7 +116,7 @@ class AvroUpsertMonitorSupervisor(entityService: UserInfo => EntityService, def startOne(): Unit = { logger.info("starting AvroUpsertMonitorActor") - actorOf(AvroUpsertMonitor.props(avroUpsertMonitorConfig.pollInterval, avroUpsertMonitorConfig.pollIntervalJitter, entityService, googleServicesDAO, samDAO, googleStorage, pubSubDAO, importServicePubSubDAO, avroUpsertMonitorConfig.importRequestPubSubSubscription, avroUpsertMonitorConfig.updateImportStatusPubSubTopic, importServiceDAO, avroUpsertMonitorConfig.batchSize)) + actorOf(AvroUpsertMonitor.props(avroUpsertMonitorConfig.pollInterval, avroUpsertMonitorConfig.pollIntervalJitter, entityService, googleServicesDAO, samDAO, googleStorage, pubSubDAO, importServicePubSubDAO, avroUpsertMonitorConfig.importRequestPubSubSubscription, avroUpsertMonitorConfig.updateImportStatusPubSubTopic, importServiceDAO, avroUpsertMonitorConfig.batchSize, dataSource)) } override val supervisorStrategy = @@ -145,9 +148,10 @@ object AvroUpsertMonitor { pubSubSubscriptionName: String, importStatusPubSubTopic: String, importServiceDAO: ImportServiceDAO, - batchSize: Int)(implicit cs: ContextShift[IO]): Props = + batchSize: Int, + dataSource: SlickDataSource)(implicit cs: ContextShift[IO]): Props = Props(new AvroUpsertMonitorActor(pollInterval, pollIntervalJitter, entityService, googleServicesDAO, samDAO, googleStorage, pubSubDao, importServicePubSubDAO, - pubSubSubscriptionName, importStatusPubSubTopic, importServiceDAO, batchSize)) + pubSubSubscriptionName, importStatusPubSubTopic, importServiceDAO, batchSize, dataSource)) } class AvroUpsertMonitorActor( @@ -162,7 +166,8 @@ class AvroUpsertMonitorActor( pubSubSubscriptionName: String, importStatusPubSubTopic: String, importServiceDAO: ImportServiceDAO, - batchSize: Int)(implicit cs: ContextShift[IO]) + batchSize: Int, + dataSource: SlickDataSource)(implicit cs: ContextShift[IO]) extends Actor with LazyLogging with FutureSupport @@ -233,7 +238,15 @@ class AvroUpsertMonitorActor( val attributes = parseMessage(message) val importFuture = for { - petUserInfo <- getPetServiceAccountUserInfo(attributes.workspace.namespace, attributes.userEmail) + workspace <- dataSource.inTransaction { dataAccess => + dataAccess.workspaceQuery.findByName(attributes.workspace).map { + case Some(workspace) => workspace + case None => + publishMessageToUpdateImportStatus(attributes.importId, None, ImportStatuses.Error, Option(s"Workspace ${attributes.workspace} not found")) + throw new RawlsException(s"Workspace ${attributes.workspace} not found while importing entities") + } + } + petUserInfo <- getPetServiceAccountUserInfo(workspace.googleProject, attributes.userEmail) importStatus <- importServiceDAO.getImportStatus(attributes.importId, attributes.workspace, petUserInfo) _ <- importStatus match { // Currently, there is only one upsert monitor thread - but if we decide to make more threads, we might @@ -241,7 +254,7 @@ class AvroUpsertMonitorActor( // of the same import. case Some(status) if status == ImportStatuses.ReadyForUpsert => { publishMessageToUpdateImportStatus(attributes.importId, Option(status), ImportStatuses.Upserting, None) - toFutureTry(initUpsert(attributes.upsertFile, attributes.importId, message.ackId, attributes.workspace, attributes.userEmail)) map { + toFutureTry(initUpsert(attributes.upsertFile, attributes.importId, message.ackId, workspace, attributes.userEmail)) map { case Success(importUpsertResults) => val msg = s"Successfully updated ${importUpsertResults.successes} entities; ${importUpsertResults.failures.size} updates failed." publishMessageToUpdateImportStatus(attributes.importId, Option(status), ImportStatuses.Done, Option(msg)) @@ -272,7 +285,7 @@ class AvroUpsertMonitorActor( } - private def initUpsert(upsertFile: String, jobId: UUID, ackId: String, workspaceName: WorkspaceName, userEmail: RawlsUserEmail): Future[ImportUpsertResults] = { + private def initUpsert(upsertFile: String, jobId: UUID, ackId: String, workspace: Workspace, userEmail: RawlsUserEmail): Future[ImportUpsertResults] = { val startTime = System.currentTimeMillis() logger.info(s"beginning upsert process for $jobId ...") @@ -314,8 +327,8 @@ class AvroUpsertMonitorActor( def performUpsertBatch(idx: Long, upsertBatch: Seq[EntityUpdateDefinition]): Future[Traversable[Entity]] = { logger.info(s"upserting batch #$idx of ${upsertBatch.size} entities for jobId ${jobId.toString} ...") for { - petUserInfo <- getPetServiceAccountUserInfo(workspaceName.namespace, userEmail) - upsertResults <- entityService.apply(petUserInfo).batchUpdateEntitiesInternal(workspaceName, upsertBatch, upsert = true) + petUserInfo <- getPetServiceAccountUserInfo(workspace.googleProject, userEmail) + upsertResults <- entityService.apply(petUserInfo).batchUpdateEntitiesInternal(workspace.toWorkspaceName, upsertBatch, upsert = true) } yield { upsertResults } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/BootMonitors.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/BootMonitors.scala index a09f49bc8f..cd9c781454 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/BootMonitors.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/BootMonitors.scala @@ -91,7 +91,7 @@ object BootMonitors extends LazyLogging { //Boot the avro upsert monitor to read and process messages in the specified PubSub topic startAvroUpsertMonitor(system, entityService, gcsDAO, samDAO, googleStorage, pubSubDAO, importServicePubSubDAO, - importServiceDAO, avroUpsertMonitorConfig) + importServiceDAO, avroUpsertMonitorConfig, slickDataSource) } private def startCreatingBillingProjectMonitor(system: ActorSystem, slickDataSource: SlickDataSource, gcsDAO: GoogleServicesDAO, samDAO: SamDAO, projectTemplate: ProjectTemplate, requesterPaysRole: String): Unit = { @@ -189,7 +189,7 @@ object BootMonitors extends LazyLogging { system.actorOf(BucketDeletionMonitor.props(slickDataSource, gcsDAO, 10 seconds, 6 hours)) } - private def startAvroUpsertMonitor(system: ActorSystem, entityService: UserInfo => EntityService, googleServicesDAO: GoogleServicesDAO, samDAO: SamDAO, googleStorage: GoogleStorageService[IO], googlePubSubDAO: GooglePubSubDAO, importServicePubSubDAO: GooglePubSubDAO, importServiceDAO: HttpImportServiceDAO, avroUpsertMonitorConfig: AvroUpsertMonitorConfig)(implicit cs: ContextShift[IO]) = { + private def startAvroUpsertMonitor(system: ActorSystem, entityService: UserInfo => EntityService, googleServicesDAO: GoogleServicesDAO, samDAO: SamDAO, googleStorage: GoogleStorageService[IO], googlePubSubDAO: GooglePubSubDAO, importServicePubSubDAO: GooglePubSubDAO, importServiceDAO: HttpImportServiceDAO, avroUpsertMonitorConfig: AvroUpsertMonitorConfig, dataSource: SlickDataSource)(implicit cs: ContextShift[IO]) = { system.actorOf( AvroUpsertMonitorSupervisor.props( entityService, @@ -199,7 +199,8 @@ object BootMonitors extends LazyLogging { googlePubSubDAO, importServicePubSubDAO, importServiceDAO, - avroUpsertMonitorConfig + avroUpsertMonitorConfig, + dataSource )) } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitor.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitor.scala index 09811d7c55..16456c54f4 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitor.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitor.scala @@ -313,8 +313,8 @@ trait CreatingBillingProjectMonitor extends LazyLogging with FutureSupport { private def onSuccessfulProjectCreate(project: RawlsBillingProject): Future[RawlsBillingProject] = { for { - _ <- gcsDAO.cleanupDMProject(project.projectName) - googleProject <- gcsDAO.getGoogleProject(project.projectName) + _ <- gcsDAO.cleanupDMProject(project.googleProjectId) + googleProject <- gcsDAO.getGoogleProject(project.googleProjectId) } yield { val status = project.servicePerimeter match { case Some(_) => CreationStatuses.AddingToPerimeter @@ -326,7 +326,7 @@ trait CreatingBillingProjectMonitor extends LazyLogging with FutureSupport { private def onFailedProjectCreate(project: RawlsBillingProject, error: String): Future[RawlsBillingProject] = { logger.debug(s"project ${project.projectName.value} creation finished with errors: $error") - gcsDAO.cleanupDMProject(project.projectName) map { _ => + gcsDAO.cleanupDMProject(project.googleProjectId) map { _ => project.copy(status = CreationStatuses.Error, message = Option(s"project ${project.projectName.value} creation finished with errors: $error")) } } diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/user/UserService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/user/UserService.scala index 617dc49b8b..91de6de0db 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/user/UserService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/user/UserService.scala @@ -30,8 +30,8 @@ import scala.util.{Failure, Success} object UserService { val allUsersGroupRef = RawlsGroupRef(RawlsGroupName("All_Users")) - def constructor(dataSource: SlickDataSource, googleServicesDAO: GoogleServicesDAO, notificationDAO: NotificationDAO, samDAO: SamDAO, projectOwnerGrantableRoles: Seq[String], requesterPaysRole: String, dmConfig: DeploymentManagerConfig, projectTemplate: ProjectTemplate)(userInfo: UserInfo)(implicit executionContext: ExecutionContext) = - new UserService(userInfo, dataSource, googleServicesDAO, notificationDAO, samDAO, projectOwnerGrantableRoles, requesterPaysRole, dmConfig, projectTemplate) + def constructor(dataSource: SlickDataSource, googleServicesDAO: GoogleServicesDAO, notificationDAO: NotificationDAO, samDAO: SamDAO, requesterPaysRole: String, dmConfig: DeploymentManagerConfig, projectTemplate: ProjectTemplate)(userInfo: UserInfo)(implicit executionContext: ExecutionContext) = + new UserService(userInfo, dataSource, googleServicesDAO, notificationDAO, samDAO, requesterPaysRole, dmConfig, projectTemplate) case class OverwriteGroupMembers(groupRef: RawlsGroupRef, memberList: RawlsGroupMemberList) @@ -57,7 +57,7 @@ object UserService { } } -class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSource, protected val gcsDAO: GoogleServicesDAO, notificationDAO: NotificationDAO, samDAO: SamDAO, projectOwnerGrantableRoles: Seq[String], requesterPaysRole: String, protected val dmConfig: DeploymentManagerConfig, protected val projectTemplate: ProjectTemplate)(implicit protected val executionContext: ExecutionContext) extends RoleSupport with FutureSupport with UserWiths with LazyLogging { +class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSource, protected val gcsDAO: GoogleServicesDAO, notificationDAO: NotificationDAO, samDAO: SamDAO, requesterPaysRole: String, protected val dmConfig: DeploymentManagerConfig, protected val projectTemplate: ProjectTemplate)(implicit protected val executionContext: ExecutionContext) extends RoleSupport with FutureSupport with UserWiths with LazyLogging { import dataSource.dataAccess.driver.api._ @@ -73,8 +73,6 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou def AddUserToBillingProject(projectName: RawlsBillingProjectName, projectAccessUpdate: ProjectAccessUpdate) = requireProjectAction(projectName, SamBillingProjectActions.alterPolicies) { addUserToBillingProject(projectName, projectAccessUpdate) } def RemoveUserFromBillingProject(projectName: RawlsBillingProjectName, projectAccessUpdate: ProjectAccessUpdate) = requireProjectAction(projectName, SamBillingProjectActions.alterPolicies) { removeUserFromBillingProject(projectName, projectAccessUpdate) } - def GrantGoogleRoleToUser(projectName: RawlsBillingProjectName, targetUserEmail: WorkbenchEmail, role: String) = requireProjectAction(projectName, SamBillingProjectActions.alterGoogleRole) { grantGoogleRoleToUser(projectName, targetUserEmail, role) } - def RemoveGoogleRoleFromUser(projectName: RawlsBillingProjectName, targetUserEmail: WorkbenchEmail, role: String) = requireProjectAction(projectName, SamBillingProjectActions.alterGoogleRole) { removeGoogleRoleFromUser(projectName, targetUserEmail, role) } def ListBillingAccounts = listBillingAccounts() def CreateBillingProjectFull(createProjectRequest: CreateRawlsBillingProjectFullRequest) = startBillingProjectCreation(createProjectRequest) @@ -232,18 +230,18 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou } } - private def deletePetsInProject(projectName: RawlsBillingProjectName, userInfo: UserInfo): Future[Unit] = { + private def deletePetsInProject(projectName: GoogleProjectId, userInfo: UserInfo): Future[Unit] = { for { projectUsers <- samDAO.listAllResourceMemberIds(SamResourceTypeNames.billingProject, projectName.value, userInfo) _ <- projectUsers.toList.traverse(destroyPet(_, projectName)) } yield () } - private def destroyPet(userIdInfo: UserIdInfo, projectName: RawlsBillingProjectName): Future[Unit] = { + private def destroyPet(userIdInfo: UserIdInfo, projectName: GoogleProjectId): Future[Unit] = { for { - petSAJson <- samDAO.getPetServiceAccountKeyForUser(projectName.value, RawlsUserEmail(userIdInfo.userEmail)) + petSAJson <- samDAO.getPetServiceAccountKeyForUser(projectName, RawlsUserEmail(userIdInfo.userEmail)) petUserInfo <- gcsDAO.getUserInfoUsingJson(petSAJson) - _ <- samDAO.deleteUserPetServiceAccount(projectName.value, petUserInfo) + _ <- samDAO.deleteUserPetServiceAccount(projectName, petUserInfo) } yield () } @@ -251,9 +249,9 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou // unregister then delete actual project in google val ownerUserInfo = UserInfo(RawlsUserEmail(ownerInfo("newOwnerEmail")), OAuth2BearerToken(ownerInfo("newOwnerToken")), 3600, RawlsUserSubjectId("0")) for { - _ <- deletePetsInProject(projectName, ownerUserInfo) + _ <- deletePetsInProject(GoogleProjectId(projectName.value), ownerUserInfo) // TODO remove for project per workspace _ <- unregisterBillingProjectWithUserInfo(projectName, ownerUserInfo) - _ <- gcsDAO.deleteProject(projectName) + _ <- gcsDAO.deleteProject(GoogleProjectId(projectName.value)) // TODO remove for project per workspace } yield RequestComplete(StatusCodes.NoContent) } @@ -270,9 +268,9 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou Future.successful(()) } // unregister then delete actual project in google - _ <- deletePetsInProject(projectName, userInfo) + _ <- deletePetsInProject(GoogleProjectId(projectName.value), userInfo) // TODO remove for project per workspace _ <- unregisterBillingProjectWithUserInfo(projectName, userInfo) - _ <- gcsDAO.deleteProject(projectName) + _ <- gcsDAO.deleteProject(GoogleProjectId(projectName.value)) // TODO remove for project per workspace } yield RequestComplete(StatusCodes.NoContent) } @@ -281,7 +279,7 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou //but that seems extremely shady def registerBillingProject(xfer: RawlsBillingProjectTransfer): Future[PerRequestMessage] = { val billingProjectName = RawlsBillingProjectName(xfer.project) - val project = RawlsBillingProject(billingProjectName, s"gs://${xfer.bucket}", CreationStatuses.Ready, None, None) + val project = RawlsBillingProject(billingProjectName, CreationStatuses.Ready, None, None) val ownerUserInfo = UserInfo(RawlsUserEmail(xfer.newOwnerEmail), OAuth2BearerToken(xfer.newOwnerToken), 3600, RawlsUserSubjectId("0")) @@ -296,8 +294,8 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou policiesToAdd = getDefaultGoogleProjectPolicies(ownerGroupEmail, computeUserGroupEmail, requesterPaysRole) - _ <- gcsDAO.addPolicyBindings(billingProjectName, policiesToAdd) - _ <- gcsDAO.grantReadAccess(billingProjectName, xfer.bucket, Set(ownerGroupEmail, computeUserGroupEmail)) + _ <- gcsDAO.addPolicyBindings(project.googleProjectId, policiesToAdd) + _ <- gcsDAO.grantReadAccess(xfer.bucket, Set(ownerGroupEmail, computeUserGroupEmail)) } yield { RequestComplete(StatusCodes.Created) }).recoverWith { @@ -342,33 +340,6 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou } } - private def googleRoleWhitelistCheck(role: String): Future[Unit] = { - if (projectOwnerGrantableRoles.contains(role)) - Future.successful(()) - else - Future.failed(new RawlsExceptionWithErrorReport(errorReport = ErrorReport(StatusCodes.Forbidden, s"Cannot alter Google role $role: not in list [${projectOwnerGrantableRoles mkString ", "}]"))) - } - - def grantGoogleRoleToUser(projectName: RawlsBillingProjectName, targetUserEmail: WorkbenchEmail, role: String): Future[PerRequestMessage] = { - for { - _ <- googleRoleWhitelistCheck(role) - proxyGroupEmail <- samDAO.getProxyGroup(userInfo, targetUserEmail) - _ <- gcsDAO.addRoleToGroup(projectName, proxyGroupEmail, role) - } yield { - RequestComplete(StatusCodes.OK) - } - } - - def removeGoogleRoleFromUser(projectName: RawlsBillingProjectName, targetUserEmail: WorkbenchEmail, role: String): Future[PerRequestMessage] = { - for { - _ <- googleRoleWhitelistCheck(role) - proxyGroupEmail <- samDAO.getProxyGroup(userInfo, targetUserEmail) - _ <- gcsDAO.removeRoleFromGroup(projectName, proxyGroupEmail, role) - } yield { - RequestComplete(StatusCodes.OK) - } - } - def deleteRefreshToken(rawlsUserRef: RawlsUserRef): Future[PerRequestMessage] = { deleteRefreshTokenInternal(rawlsUserRef).map(_ => RequestComplete(StatusCodes.OK)) @@ -417,14 +388,14 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou private def internalStartBillingProjectCreation(createProjectRequest: CreateRawlsBillingProjectFullRequest, billingAccount: RawlsBillingAccount): Future[PerRequestMessage] = { for { - _ <- dataSource.inTransaction { dataAccess => + project <- dataSource.inTransaction { dataAccess => dataAccess.rawlsBillingProjectQuery.load(createProjectRequest.projectName) flatMap { case None => for { _ <- DBIO.from(samDAO.createResource(SamResourceTypeNames.billingProject, createProjectRequest.projectName.value, userInfo)) _ <- DBIO.from(samDAO.overwritePolicy(SamResourceTypeNames.billingProject, createProjectRequest.projectName.value, SamBillingProjectPolicyNames.workspaceCreator, SamPolicy(Set.empty, Set.empty, Set(SamProjectRoles.workspaceCreator)), userInfo)) _ <- DBIO.from(samDAO.overwritePolicy(SamResourceTypeNames.billingProject, createProjectRequest.projectName.value, SamBillingProjectPolicyNames.canComputeUser, SamPolicy(Set.empty, Set.empty, Set(SamProjectRoles.batchComputeUser, SamProjectRoles.notebookUser)), userInfo)) - project <- dataAccess.rawlsBillingProjectQuery.create(RawlsBillingProject(createProjectRequest.projectName, "gs://" + gcsDAO.getCromwellAuthBucketName(createProjectRequest.projectName), CreationStatuses.Creating, Option(createProjectRequest.billingAccount), None, None, createProjectRequest.servicePerimeter)) + project <- dataAccess.rawlsBillingProjectQuery.create(RawlsBillingProject(createProjectRequest.projectName, CreationStatuses.Creating, Option(createProjectRequest.billingAccount), None, None, createProjectRequest.servicePerimeter)) } yield project case Some(_) => throw new RawlsExceptionWithErrorReport(ErrorReport(StatusCodes.Conflict, "project by that name already exists")) @@ -438,7 +409,7 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou // each service perimeter should have a folder which is used to make an aggregate log sink for flow logs parentFolderId <- createProjectRequest.servicePerimeter.traverse(lookupFolderIdFromServicePerimeterName) - createProjectOperation <- gcsDAO.createProject(createProjectRequest.projectName, billingAccount, dmConfig.templatePath, createProjectRequest.highSecurityNetwork.getOrElse(false), createProjectRequest.enableFlowLogs.getOrElse(false), createProjectRequest.privateIpGoogleAccess.getOrElse(false), requesterPaysRole, ownerGroupEmail, computeUserGroupEmail, projectTemplate, parentFolderId).recoverWith { + createProjectOperation <- gcsDAO.createProject(project.googleProjectId, billingAccount, dmConfig.templatePath, createProjectRequest.highSecurityNetwork.getOrElse(false), createProjectRequest.enableFlowLogs.getOrElse(false), createProjectRequest.privateIpGoogleAccess.getOrElse(false), requesterPaysRole, ownerGroupEmail, computeUserGroupEmail, projectTemplate, parentFolderId).recoverWith { case t: Throwable => // failed to create project in google land, rollback inserts above dataSource.inTransaction { dataAccess => dataAccess.rawlsBillingProjectQuery.delete(createProjectRequest.projectName) } map(_ => throw t) @@ -498,11 +469,11 @@ class UserService(protected val userInfo: UserInfo, val dataSource: SlickDataSou // each service perimeter should have a folder which is used to make an aggregate log sink for flow logs folderId <- lookupFolderIdFromServicePerimeterName(servicePerimeterName) - _ <- gcsDAO.addProjectToFolder(projectName, folderId) + _ <- gcsDAO.addProjectToFolder(billingProject.googleProjectId, folderId) googleProjectNumber <- billingProject.googleProjectNumber match { case Some(existingGoogleProjectNumber) => Future.successful(existingGoogleProjectNumber) - case None => gcsDAO.getGoogleProject(billingProject.projectName).map(googleProject => GoogleProjectNumber(googleProject.getProjectNumber.toString)) + case None => gcsDAO.getGoogleProject(billingProject.googleProjectId).map(googleProject => GoogleProjectNumber(googleProject.getProjectNumber.toString)) } _ <- dataSource.inTransaction { dataAccess => diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/AuthUtil.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/AuthUtil.scala index 78159d3dc8..f15d0bb01a 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/AuthUtil.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/AuthUtil.scala @@ -1,7 +1,7 @@ package org.broadinstitute.dsde.rawls.util import org.broadinstitute.dsde.rawls.dataaccess.{GoogleServicesDAO, SamDAO} -import org.broadinstitute.dsde.rawls.model.RawlsUserEmail +import org.broadinstitute.dsde.rawls.model.{GoogleProjectId, RawlsUserEmail} import scala.concurrent.ExecutionContext @@ -9,9 +9,9 @@ trait AuthUtil { val samDAO: SamDAO val googleServicesDAO: GoogleServicesDAO - def getPetServiceAccountUserInfo(workspaceNamespace: String, userEmail: RawlsUserEmail)(implicit executionContext: ExecutionContext) = { + def getPetServiceAccountUserInfo(googleProjectId: GoogleProjectId, userEmail: RawlsUserEmail)(implicit executionContext: ExecutionContext) = { for { - petSAJson <- samDAO.getPetServiceAccountKeyForUser(workspaceNamespace, userEmail) + petSAJson <- samDAO.getPetServiceAccountKeyForUser(googleProjectId, userEmail) petUserInfo <- googleServicesDAO.getUserInfoUsingJson(petSAJson) } yield { petUserInfo diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala index b0c3a86d42..a06d8ec654 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/util/WorkspaceSupport.scala @@ -78,11 +78,11 @@ trait WorkspaceSupport { response <- userHasAction match { case true => traceDBIOWithParent("loadBillingProject", parentSpan)( _ => dataAccess.rawlsBillingProjectQuery.load(projectName)).flatMap { - case Some(RawlsBillingProject(_, _, CreationStatuses.Ready, _, _, _, _, _, _)) => op //Sam will check to make sure the Auth Domain selection is valid - case Some(RawlsBillingProject(RawlsBillingProjectName(name), _, CreationStatuses.Creating, _, _, _, _, _, _)) => + case Some(RawlsBillingProject(_, CreationStatuses.Ready, _, _, _, _, _, _)) => op //Sam will check to make sure the Auth Domain selection is valid + case Some(RawlsBillingProject(RawlsBillingProjectName(name), CreationStatuses.Creating, _, _, _, _, _, _)) => DBIO.failed(new RawlsExceptionWithErrorReport(errorReport = ErrorReport(StatusCodes.BadRequest, s"${name} is still being created"))) - case Some(RawlsBillingProject(RawlsBillingProjectName(name), _, CreationStatuses.Error, _, messageOp, _, _, _, _)) => + case Some(RawlsBillingProject(RawlsBillingProjectName(name), CreationStatuses.Error, _, messageOp, _, _, _, _)) => DBIO.failed(new RawlsExceptionWithErrorReport(errorReport = ErrorReport(StatusCodes.BadRequest, s"Error creating ${name}: ${messageOp.getOrElse("no message")}"))) case Some(_) | None => // this can't happen with the current code but a 404 would be the correct response diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiService.scala index 54124dc9a3..32c0ca837a 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiService.scala @@ -1,11 +1,10 @@ package org.broadinstitute.dsde.rawls.webservice +import akka.http.scaladsl.server +import akka.http.scaladsl.server.Directives._ import org.broadinstitute.dsde.rawls.model._ import org.broadinstitute.dsde.rawls.openam.UserInfoDirectives import org.broadinstitute.dsde.rawls.user.UserService -import org.broadinstitute.dsde.workbench.model.WorkbenchEmail -import akka.http.scaladsl.server -import akka.http.scaladsl.server.Directives._ import scala.concurrent.ExecutionContext @@ -16,9 +15,9 @@ import scala.concurrent.ExecutionContext trait BillingApiService extends UserInfoDirectives { implicit val executionContext: ExecutionContext - import org.broadinstitute.dsde.rawls.model.UserAuthJsonSupport._ - import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import PerRequest.requestCompleteMarshaller + import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ + import org.broadinstitute.dsde.rawls.model.UserAuthJsonSupport._ val userServiceConstructor: UserInfo => UserService @@ -29,15 +28,6 @@ trait BillingApiService extends UserInfoDirectives { complete { userServiceConstructor(userInfo).GetBillingProjectMembers(RawlsBillingProjectName(projectId)) } } } ~ - // these routes are for setting/unsetting Google cloud roles - path("googleRole" / Segment / Segment) { (googleRole, userEmail) => - put { - complete { userServiceConstructor(userInfo).GrantGoogleRoleToUser(RawlsBillingProjectName(projectId), WorkbenchEmail(userEmail), googleRole) } - } ~ - delete { - complete { userServiceConstructor(userInfo).RemoveGoogleRoleFromUser(RawlsBillingProjectName(projectId), WorkbenchEmail(userEmail), googleRole) } - } - } ~ // these routes are for adding/removing users from projects path(Segment / Segment) { (workbenchRole, userEmail) => put { diff --git a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala index 9ca71c509b..5fb3a72d98 100644 --- a/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala +++ b/core/src/main/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceService.scala @@ -245,7 +245,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa // determine which functions to use for the various part of the response def bucketOptionsFuture(): Future[Option[WorkspaceBucketOptions]] = if (options.contains("bucketOptions")) { - traceWithParent("getBucketDetails",s1)(_ => gcsDAO.getBucketDetails(workspaceContext.bucketName, RawlsBillingProjectName(workspaceContext.namespace)).map(Option(_))) + traceWithParent("getBucketDetails",s1)(_ => gcsDAO.getBucketDetails(workspaceContext.bucketName, workspaceContext.googleProject).map(Option(_))) } else { noFuture } @@ -311,7 +311,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa def getBucketOptions(workspaceName: WorkspaceName): Future[PerRequestMessage] = { getWorkspaceContextAndPermissions(workspaceName, SamWorkspaceActions.read) flatMap { workspaceContext => dataSource.inTransaction { dataAccess => - DBIO.from(gcsDAO.getBucketDetails(workspaceContext.bucketName, RawlsBillingProjectName(workspaceContext.namespace))) map { details => + DBIO.from(gcsDAO.getBucketDetails(workspaceContext.bucketName, workspaceContext.googleProject)) map { details => RequestComplete(StatusCodes.OK, details) } } @@ -869,7 +869,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa samDAO.removeUserFromPolicy(SamResourceTypeNames.workspace, workspace.workspaceId, policyName, email, userInfo) } - _ <- revokeRequesterPaysForLinkedSAs(workspaceName, policyRemovals, policyAdditions) + _ <- revokeRequesterPaysForLinkedSAs(workspace, policyRemovals, policyAdditions) _ <- maybeShareProjectComputePolicy(policyAdditions, workspaceName) @@ -893,12 +893,12 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa * removals from applicable policies then removing all the additions to applicable policies. Revoke linked SAs for * all resulting users. * - * @param workspaceName + * @param workspace * @param policyRemovals * @param policyAdditions * @return */ - private def revokeRequesterPaysForLinkedSAs(workspaceName: WorkspaceName, policyRemovals: Set[(SamResourcePolicyName, String)], policyAdditions: Set[(SamResourcePolicyName, String)]): Future[Unit] = { + private def revokeRequesterPaysForLinkedSAs(workspace: Workspace, policyRemovals: Set[(SamResourcePolicyName, String)], policyAdditions: Set[(SamResourcePolicyName, String)]): Future[Unit] = { val applicablePolicies = Set(SamWorkspacePolicyNames.owner, SamWorkspacePolicyNames.writer) val applicableRemovals = policyRemovals.collect { case (policy, email) if applicablePolicies.contains(policy) => RawlsUserEmail(email) @@ -906,7 +906,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa val applicableAdditions = policyAdditions.collect { case (policy, email) if applicablePolicies.contains(policy) => RawlsUserEmail(email) } - Future.traverse(applicableRemovals -- applicableAdditions) { emailToRevoke => requesterPaysSetupService.revokeUserFromWorkspace(emailToRevoke, workspaceName) }.void + Future.traverse(applicableRemovals -- applicableAdditions) { emailToRevoke => requesterPaysSetupService.revokeUserFromWorkspace(emailToRevoke, workspace) }.void } private def validateAclChanges(aclChanges: Set[WorkspaceACLUpdate], existingAcls: Set[WorkspaceACLUpdate]) = { @@ -1646,7 +1646,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa } petKey <- if (maxAccessLevel >= WorkspaceAccessLevels.Write) - samDAO.getPetServiceAccountKeyForUser(workspace.namespace, userInfo.userEmail) + samDAO.getPetServiceAccountKeyForUser(workspace.googleProject, userInfo.userEmail) else samDAO.getDefaultPetServiceAccountKeyForUser(userInfo) @@ -1729,7 +1729,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa requireAccessIgnoreLockF(workspaceContext, SamWorkspaceActions.write) { //if we get here, we passed all the hoops, otherwise an exception would have been thrown - gcsDAO.getBucketUsage(RawlsBillingProjectName(workspaceName.namespace), workspaceContext.bucketName).map { usage => + gcsDAO.getBucketUsage(workspaceContext.googleProject, workspaceContext.bucketName).map { usage => RequestComplete(BucketUsageResponse(usage)) } } @@ -1772,15 +1772,23 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa case Some(workspace) => Future.successful(workspace) } _ <- accessCheck(workspace, SamWorkspaceActions.compute, ignoreLock = false) - _ <- requesterPaysSetupService.grantRequesterPaysToLinkedSAs(userInfo, workspaceName) + _ <- requesterPaysSetupService.grantRequesterPaysToLinkedSAs(userInfo, workspace) } yield { RequestComplete(StatusCodes.NoContent) } } def disableRequesterPaysForLinkedSAs(workspaceName: WorkspaceName): Future[PerRequestMessage] = { + // note that this does not throw an error if the workspace does not exist + // the user may no longer have access to the workspace so we can't confirm it exists + // but the user does have the right to remove their linked SAs for { - _ <- requesterPaysSetupService.revokeUserFromWorkspace(userInfo.userEmail, workspaceName) + maybeWorkspace <- dataSource.inTransaction { dataaccess => + dataaccess.workspaceQuery.findByName(workspaceName) + } + _ <- Future.traverse(maybeWorkspace.toList) { workspace => + requesterPaysSetupService.revokeUserFromWorkspace(userInfo.userEmail, workspace) + } } yield { RequestComplete(StatusCodes.NoContent) } @@ -1915,7 +1923,7 @@ class WorkspaceService(protected val userInfo: UserInfo, val dataSource: SlickDa } }.flatten.toMap) - _ <- traceDBIOWithParent("gcsDAO.setupWorkspace", s2)(s3 => DBIO.from(gcsDAO.setupWorkspace(userInfo, RawlsBillingProjectName(workspaceRequest.namespace), policyEmails, bucketName, getLabels(workspaceRequest.authorizationDomain.getOrElse(Set.empty).toList), s3))) + _ <- traceDBIOWithParent("gcsDAO.setupWorkspace", s2)(s3 => DBIO.from(gcsDAO.setupWorkspace(userInfo, savedWorkspace.googleProject, policyEmails, bucketName, getLabels(workspaceRequest.authorizationDomain.getOrElse(Set.empty).toList), s3))) response <- traceDBIOWithParent("doOp", s2)(_ => op(savedWorkspace)) } yield response }) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala index f9fd9e8373..d71d8d5e5d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/MockGoogleServicesDAO.scala @@ -30,7 +30,7 @@ class MockGoogleServicesDAO(groupsPrefix: String, private var tokenDate: DateTime = null private val groups: TrieMap[RawlsGroupRef, Set[Either[RawlsUser, RawlsGroup]]] = TrieMap() - val policies: TrieMap[RawlsBillingProjectName, Map[String, Set[String]]] = TrieMap() + val policies: TrieMap[GoogleProjectId, Map[String, Set[String]]] = TrieMap() val accessibleBillingAccountName = RawlsBillingAccountName("billingAccounts/firecloudHasThisOne") val inaccessibleBillingAccountName = RawlsBillingAccountName("billingAccounts/firecloudDoesntHaveThisOne") @@ -98,7 +98,7 @@ class MockGoogleServicesDAO(groupsPrefix: String, var mockProxyGroups = mutable.Map[RawlsUser, Boolean]() - override def setupWorkspace(userInfo: UserInfo, projectName: RawlsBillingProjectName, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null + override def setupWorkspace(userInfo: UserInfo, googleProject: GoogleProjectId, policyGroupsByAccessLevel: Map[WorkspaceAccessLevel, WorkbenchEmail], bucketName: String, labels: Map[String, String], parentSpan: Span = null ): Future[GoogleWorkspaceInfo] = { val googleWorkspaceInfo: GoogleWorkspaceInfo = GoogleWorkspaceInfo(bucketName, policyGroupsByAccessLevel) @@ -108,7 +108,7 @@ class MockGoogleServicesDAO(groupsPrefix: String, override def getAccessTokenUsingJson(saKey: String): Future[String] = Future.successful("token") override def getUserInfoUsingJson(saKey: String): Future[UserInfo] = Future.successful(UserInfo(RawlsUserEmail("foo@bar.com"), OAuth2BearerToken("test_token"), 0, RawlsUserSubjectId("12345678000"))) - override def getGoogleProject(billingProjectName: RawlsBillingProjectName): Future[Project] = Future.successful(new Project().setProjectNumber(42L)) + override def getGoogleProject(billingProjectName: GoogleProjectId): Future[Project] = Future.successful(new Project().setProjectNumber(42L)) override def deleteBucket(bucketName: String) = Future.successful(true) @@ -161,7 +161,7 @@ class MockGoogleServicesDAO(groupsPrefix: String, override def getGoogleGroup(groupName: String)(implicit executionContext: ExecutionContext): Future[Option[Group]] = Future.successful(Some(new Group)) - def getBucketUsage(projectName: RawlsBillingProjectName, bucketName: String, maxResults: Option[Long]): Future[BigInt] = Future.successful(42) + def getBucketUsage(googleProject: GoogleProjectId, bucketName: String, maxResults: Option[Long]): Future[BigInt] = Future.successful(42) override def addEmailToGoogleGroup(groupEmail: String, emailToAdd: String): Future[Unit] = { googleGroups(groupEmail) += emailToAdd @@ -193,41 +193,35 @@ class MockGoogleServicesDAO(groupsPrefix: String, Future.successful(true) } - override def createProject(projectName: RawlsBillingProjectName, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = - Future.successful(RawlsBillingProjectOperationRecord(projectName.value, GoogleOperationNames.DeploymentManagerCreateProject, "opid", false, None, GoogleApiTypes.DeploymentManagerApi)) + override def createProject(googleProject: GoogleProjectId, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = + Future.successful(RawlsBillingProjectOperationRecord(googleProject.value, GoogleOperationNames.DeploymentManagerCreateProject, "opid", false, None, GoogleApiTypes.DeploymentManagerApi)) - override def cleanupDMProject(projectName: RawlsBillingProjectName): Future[Unit] = Future.successful(()) + override def cleanupDMProject(googleProject: GoogleProjectId): Future[Unit] = Future.successful(()) - override def getBucketDetails(bucket: String, project: RawlsBillingProjectName): Future[WorkspaceBucketOptions] = { + override def getBucketDetails(bucket: String, project: GoogleProjectId): Future[WorkspaceBucketOptions] = { Future.successful(WorkspaceBucketOptions(false)) } - protected def updatePolicyBindings(projectName: RawlsBillingProjectName)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] = Future.successful { - val existingPolicies = policies.getOrElse(projectName, Map.empty) + protected def updatePolicyBindings(googleProject: GoogleProjectId)(updatePolicies: Map[String, Set[String]] => Map[String, Set[String]]): Future[Boolean] = Future.successful { + val existingPolicies = policies.getOrElse(googleProject, Map.empty) val updatedPolicies = updatePolicies(existingPolicies) if (updatedPolicies.equals(existingPolicies)) { false } else { - policies.put(projectName, updatedPolicies) + policies.put(googleProject, updatedPolicies) true } } - override def grantReadAccess(billingProject: RawlsBillingProjectName, - bucketName: String, - readers: Set[WorkbenchEmail]): Future[String] = Future(bucketName) + override def grantReadAccess(bucketName: String, readers: Set[WorkbenchEmail]): Future[String] = Future(bucketName) override def pollOperation(operationId: OperationId): Future[OperationStatus] = { Future.successful(OperationStatus(true, None)) } - override def deleteProject(projectName: RawlsBillingProjectName): Future[Unit] = Future.successful(()) + override def deleteProject(googleProject: GoogleProjectId): Future[Unit] = Future.successful(()) - override def addRoleToGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] = Future.successful(false) - - override def removeRoleFromGroup(projectName: RawlsBillingProjectName, groupEmail: WorkbenchEmail, role: String): Future[Boolean] = Future.successful(false) - - override def addProjectToFolder(projectName: RawlsBillingProjectName, folderName: String): Future[Unit] = Future.successful(()) + override def addProjectToFolder(googleProject: GoogleProjectId, folderName: String): Future[Unit] = Future.successful(()) override def getFolderId(folderName: String): Future[Option[String]] = Future.successful(Option("folders/1234567")) } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupServiceSpec.scala index 0f0f8bc2a7..1f0e2f5b66 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/RequesterPaysSetupServiceSpec.scala @@ -57,12 +57,12 @@ class RequesterPaysSetupServiceSpec extends FlatSpec with Matchers with MockitoS when(service.bondApiDAO.getServiceAccountKey("p2", userInfo)).thenReturn(Future.successful(Some(BondResponseData(expectedEmail)))) runAndWait(workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(minimalTestData.billingProject.projectName.value, userInfo.userEmail)) shouldBe false - service.grantRequesterPaysToLinkedSAs(userInfo, minimalTestData.workspace.toWorkspaceName).futureValue shouldBe List(expectedEmail) + service.grantRequesterPaysToLinkedSAs(userInfo, minimalTestData.workspace).futureValue shouldBe List(expectedEmail) runAndWait(workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(minimalTestData.billingProject.projectName.value, userInfo.userEmail)) shouldBe true - service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.billingProject.projectName) shouldBe Some(Map(service.requesterPaysRole -> Set("serviceAccount:" + expectedEmail.client_email))) + service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.workspace.googleProject) shouldBe Some(Map(service.requesterPaysRole -> Set("serviceAccount:" + expectedEmail.client_email))) // second call should not fail - service.grantRequesterPaysToLinkedSAs(userInfo, minimalTestData.workspace.toWorkspaceName).futureValue shouldBe List(expectedEmail) + service.grantRequesterPaysToLinkedSAs(userInfo, minimalTestData.workspace).futureValue shouldBe List(expectedEmail) } "revokeUserFromWorkspace" should "unlink" in withMinimalTestDatabaseAndServices { service => @@ -75,16 +75,16 @@ class RequesterPaysSetupServiceSpec extends FlatSpec with Matchers with MockitoS // add user to mock google bindings val initialBindings = Map(service.requesterPaysRole -> Set("serviceAccount:" + expectedEmail.client_email)) - service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.put(minimalTestData.billingProject.projectName, initialBindings) + service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.put(minimalTestData.workspace.googleProject, initialBindings) // remove use from 1 workspace and check that it did not get removed from google bindings - service.revokeUserFromWorkspace(userInfo.userEmail, minimalTestData.workspace.toWorkspaceName).futureValue shouldBe List(expectedEmail) + service.revokeUserFromWorkspace(userInfo.userEmail, minimalTestData.workspace).futureValue shouldBe List(expectedEmail) runAndWait(workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(minimalTestData.billingProject.projectName.value, userInfo.userEmail)) shouldBe true - service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.billingProject.projectName) shouldBe Some(initialBindings) + service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.workspace.googleProject) shouldBe Some(initialBindings) // remove use from other workspace and check that it did get removed from google bindings - service.revokeUserFromWorkspace(userInfo.userEmail, minimalTestData.workspace2.toWorkspaceName).futureValue shouldBe List(expectedEmail) + service.revokeUserFromWorkspace(userInfo.userEmail, minimalTestData.workspace2).futureValue shouldBe List(expectedEmail) runAndWait(workspaceRequesterPaysQuery.userExistsInWorkspaceNamespace(minimalTestData.billingProject.projectName.value, userInfo.userEmail)) shouldBe false - service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.billingProject.projectName) shouldBe Some(Map(service.requesterPaysRole -> Set.empty)) + service.googleServicesDAO.asInstanceOf[MockGoogleServicesDAO].policies.get(minimalTestData.workspace.googleProject) shouldBe Some(Map(service.requesterPaysRole -> Set.empty)) } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/TestDriverComponent.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/TestDriverComponent.scala index 50cc7231d6..164ff8587c 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/TestDriverComponent.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/TestDriverComponent.scala @@ -142,7 +142,7 @@ trait TestDriverComponent extends DriverComponent with DataAccess with DefaultIn ) } - def billingProjectFromName(name: String) = (RawlsBillingProject(RawlsBillingProjectName(name), "mockBucketUrl", CreationStatuses.Ready, None, None)) + def billingProjectFromName(name: String) = (RawlsBillingProject(RawlsBillingProjectName(name), CreationStatuses.Ready, None, None)) def makeRawlsGroup(name: String, users: Set[RawlsUserRef], groups: Set[RawlsGroupRef] = Set.empty) = RawlsGroup(RawlsGroupName(name), RawlsGroupEmail(s"$name@example.com"), users, groups) @@ -223,16 +223,16 @@ trait TestDriverComponent extends DriverComponent with DataAccess with DefaultIn val nestedProjectGroup = makeRawlsGroup("nested_project_group", Set(userOwner)) val dbGapAuthorizedUsersGroup = ManagedGroupRef(RawlsGroupName("dbGapAuthorizedUsers")) - val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), CreationStatuses.Ready, None, None) val testProject1Name = RawlsBillingProjectName("arbitrary") - val testProject1 = RawlsBillingProject(testProject1Name, "http://cromwell-auth-url.example.com", CreationStatuses.Ready, None, None) + val testProject1 = RawlsBillingProject(testProject1Name, CreationStatuses.Ready, None, None) val testProject2Name = RawlsBillingProjectName("project2") - val testProject2 = RawlsBillingProject(testProject2Name, "http://cromwell-auth-url.example.com", CreationStatuses.Ready, None, None) + val testProject2 = RawlsBillingProject(testProject2Name, CreationStatuses.Ready, None, None) val testProject3Name = RawlsBillingProjectName("project3") - val testProject3 = RawlsBillingProject(testProject3Name, "http://cromwell-auth-url.example.com", CreationStatuses.Ready, None, None) + val testProject3 = RawlsBillingProject(testProject3Name, CreationStatuses.Ready, None, None) val wsAttrs = Map( AttributeName.withDefaultNS("string") -> AttributeString("yep, it's a string"), @@ -959,7 +959,7 @@ trait TestDriverComponent extends DriverComponent with DataAccess with DefaultIn } class MinimalTestData() extends TestData { - val billingProject = RawlsBillingProject(RawlsBillingProjectName("myNamespace"), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("myNamespace"), CreationStatuses.Ready, None, None) val wsName = WorkspaceName(billingProject.projectName.value, "myWorkspace") val wsName2 = WorkspaceName(billingProject.projectName.value, "myWorkspace2") val ownerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-OWNER", Set.empty) @@ -993,7 +993,7 @@ trait TestDriverComponent extends DriverComponent with DataAccess with DefaultIn val writerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-WRITER", Set(userWriter)) val readerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-READER", Set(userReader)) - val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), CreationStatuses.Ready, None, None) val wsAttrs = Map( AttributeName.withDefaultNS("string") -> AttributeString("yep, it's a string"), diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponentSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponentSpec.scala index 434c43dd80..3a267fba4e 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponentSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/WorkspaceComponentSpec.scala @@ -30,7 +30,7 @@ class WorkspaceComponentSpec extends TestDriverComponentWithFlatSpecAndMatchers AttributeName.withDefaultNS("attributeNum") -> AttributeNumber(3.14159)), false, WorkspaceVersions.V2, - "test_google_project" + GoogleProjectId("test_google_project") ) assertResult(None) { diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProviderSpecSupport.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProviderSpecSupport.scala index cce87e28d4..4f2d55ef4b 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProviderSpecSupport.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/entities/datarepo/DataRepoEntityProviderSpecSupport.scala @@ -10,7 +10,7 @@ import org.broadinstitute.dsde.rawls.dataaccess.workspacemanager.WorkspaceManage import org.broadinstitute.dsde.rawls.dataaccess.{GoogleBigQueryServiceFactory, MockBigQueryServiceFactory, SamDAO, SlickDataSource} import org.broadinstitute.dsde.rawls.entities.EntityRequestArguments import org.broadinstitute.dsde.rawls.mock.{MockDataRepoDAO, MockSamDAO, MockWorkspaceManagerDAO} -import org.broadinstitute.dsde.rawls.model.{DataReferenceName, RawlsUserEmail, UserInfo, Workspace} +import org.broadinstitute.dsde.rawls.model.{DataReferenceName, GoogleProjectId, RawlsUserEmail, UserInfo, Workspace} import org.joda.time.DateTime import scala.collection.JavaConverters._ @@ -141,7 +141,7 @@ trait DataRepoEntityProviderSpecSupport { */ class SpecSamDAO(dataSource: SlickDataSource = slickDataSource, petKeyForUserResponse: Either[Throwable, String]) extends MockSamDAO(dataSource) { - override def getPetServiceAccountKeyForUser(googleProject: String, userEmail: RawlsUserEmail): Future[String] = { + override def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail): Future[String] = { petKeyForUserResponse match { case Left(t) => Future.failed(t) case Right(key) => Future.successful(key) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionSpec.scala index 1e8e5ee141..c8795e799d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/SubmissionSpec.scala @@ -312,7 +312,6 @@ class SubmissionSpec(_system: ActorSystem) extends TestKit(_system) gcsDAO, notificationDAO, samDAO, - Seq("bigquery.jobUser"), "requesterPaysRole", DeploymentManagerConfig(testConf.getConfig("gcs.deploymentManager")), ProjectTemplate.from(testConf.getConfig("gcs.projectTemplate")) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionSpec.scala index 1a4fbcfe2b..383330c6d8 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/jobexec/WorkflowSubmissionSpec.scala @@ -237,7 +237,7 @@ class WorkflowSubmissionSpec(_system: ActorSystem) extends TestKit(_system) with mockExecCluster.getDefaultSubmitMember.asInstanceOf[MockExecutionServiceDAO].submitInput } - val petJson = Await.result(workflowSubmission.samDAO.getPetServiceAccountKeyForUser(testData.workspace.namespace, testData.userOwner.userEmail), Duration.Inf) + val petJson = Await.result(workflowSubmission.samDAO.getPetServiceAccountKeyForUser(testData.workspace.googleProject, testData.userOwner.userEmail), Duration.Inf) assertResult( Some( ExecutionServiceWorkflowOptions( @@ -247,7 +247,6 @@ class WorkflowSubmissionSpec(_system: ActorSystem) extends TestKit(_system) with google_compute_service_account = "pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com", user_service_account_json = """{"client_email": "pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com", "client_id": "104493171545941951815"}""", - auth_bucket = testData.billingProject.cromwellAuthBucketUrl, final_workflow_log_dir = s"gs://${testData.workspace.bucketName}/${testData.submission1.submissionId}/workflow.logs", default_runtime_attributes = Some(JsObject(Map("zones" -> JsString("us-central-someother")))), @@ -331,7 +330,7 @@ class WorkflowSubmissionSpec(_system: ActorSystem) extends TestKit(_system) with Await.result(workflowSubmission.submitWorkflowBatch(WorkflowBatch(workflowRecs.map(_.id), submissionRec, workspaceRec)), Duration.Inf) // Verify - mockGoogleServicesDAO.policies(RawlsBillingProjectName(ctx.namespace))(requesterPaysRole) should contain theSameElementsAs + mockGoogleServicesDAO.policies(ctx.googleProject)(requesterPaysRole) should contain theSameElementsAs List("serviceAccount:" + drsServiceAccount, "serviceAccount:" + differentDrsServiceAccount) } } @@ -364,7 +363,7 @@ class WorkflowSubmissionSpec(_system: ActorSystem) extends TestKit(_system) with Await.result(workflowSubmission.submitWorkflowBatch(WorkflowBatch(workflowRecs.map(_.id), submissionRec, workspaceRec)), Duration.Inf) // Verify - mockGoogleServicesDAO.policies(RawlsBillingProjectName(ctx.namespace))(requesterPaysRole) should contain theSameElementsAs + mockGoogleServicesDAO.policies(ctx.googleProject)(requesterPaysRole) should contain theSameElementsAs List("serviceAccount:" + drsServiceAccount, "serviceAccount:" + differentDrsServiceAccount) } } @@ -406,7 +405,7 @@ class WorkflowSubmissionSpec(_system: ActorSystem) extends TestKit(_system) with // Verify val expectedClientEmail = List("serviceAccount:" + drsServiceAccount, "serviceAccount:" + differentDrsServiceAccount) - mockGoogleServicesDAO.policies(RawlsBillingProjectName(ctx.namespace))(requesterPaysRole) should contain theSameElementsAs expectedClientEmail + mockGoogleServicesDAO.policies(ctx.googleProject)(requesterPaysRole) should contain theSameElementsAs expectedClientEmail } } diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala index 547a31013d..f232d3c249 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/mock/MockSamDAO.scala @@ -94,11 +94,11 @@ class MockSamDAO(dataSource: SlickDataSource)(implicit executionContext: Executi override def getResourceAuthDomain(resourceTypeName: SamResourceTypeName, resourceId: String, userInfo: UserInfo): Future[Seq[String]] = Future.successful(Seq.empty) - override def getPetServiceAccountKeyForUser(googleProject: String, userEmail: RawlsUserEmail): Future[String] = Future.successful("""{"client_email": "pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com", "client_id": "104493171545941951815"}""") + override def getPetServiceAccountKeyForUser(googleProject: GoogleProjectId, userEmail: RawlsUserEmail): Future[String] = Future.successful("""{"client_email": "pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com", "client_id": "104493171545941951815"}""") override def getDefaultPetServiceAccountKeyForUser(userInfo: UserInfo): Future[String] = Future.successful("""{"client_email": "pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com", "client_id": "104493171545941951815"}""") - override def deleteUserPetServiceAccount(googleProject: String, userInfo: UserInfo): Future[Unit] = Future.unit + override def deleteUserPetServiceAccount(googleProject: GoogleProjectId, userInfo: UserInfo): Future[Unit] = Future.unit override def getStatus(): Future[SubsystemStatus] = Future.successful(SubsystemStatus(true, None)) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala index e9d8c063b6..2ec5f99f93 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala @@ -137,7 +137,6 @@ class ExecutionModelSpec extends FlatSpec with Matchers { | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/pet-110347448408766049948%40broad-dsde-dev.iam.gserviceaccount.com" |}""".stripMargin, - auth_bucket = "auth_bucket", final_workflow_log_dir = "final_workflow_log_dir", default_runtime_attributes = None, read_from_cache = true, @@ -155,7 +154,6 @@ class ExecutionModelSpec extends FlatSpec with Matchers { | "google_compute_service_account": "account@foo.com", | "google_labels": {}, | "user_service_account_json": "{\n \"type\": \"service_account\",\n \"project_id\": \"broad-dsde-dev\",\n \"proovate_key_id\": \"120924d141277cef7a976320d3dc3e4e298ac447\",\n \"proovate_key\": \"-----BEGIN proovate KEY-----\\naloha\\n-----END proovate KEY-----\\n\",\n \"client_email\": \"pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com\",\n \"client_id\": \"110086970853956779852\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://accounts.google.com/o/oauth2/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/pet-110347448408766049948%40broad-dsde-dev.iam.gserviceaccount.com\"\n}", - | "auth_bucket": "auth_bucket", | "final_workflow_log_dir": "final_workflow_log_dir", | "read_from_cache": true, | "delete_intermediate_output_files": true, @@ -180,7 +178,6 @@ class ExecutionModelSpec extends FlatSpec with Matchers { | "google_compute_service_account": "account@foo.com", | "google_labels": {}, | "user_service_account_json": "{\n \"type\": \"service_account\",\n \"project_id\": \"broad-dsde-dev\",\n \"proovate_key_id\": \"120924d141277cef7a976320d3dc3e4e298ac447\",\n \"proovate_key\": \"-----BEGIN proovate KEY-----\\naloha\\n-----END proovate KEY-----\\n\",\n \"client_email\": \"pet-110347448408766049948@broad-dsde-dev.iam.gserviceaccount.com\",\n \"client_id\": \"110086970853956779852\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://accounts.google.com/o/oauth2/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/pet-110347448408766049948%40broad-dsde-dev.iam.gserviceaccount.com\"\n}", - | "auth_bucket": "auth_bucket", | "final_workflow_log_dir": "final_workflow_log_dir", | "read_from_cache": true, | "delete_intermediate_output_files": true, diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitorSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitorSpec.scala index 3b1dafa68a..9673d67677 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitorSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/AvroUpsertMonitorSpec.scala @@ -111,7 +111,8 @@ class AvroUpsertMonitorSpec(_system: ActorSystem) extends ApiServiceSpec with Mo services.gpsDAO, services.gpsDAO, mockImportServiceDAO, - config + config, + slickDataSource )) mockImportServiceDAO diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitorSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitorSpec.scala index ce01c8fefc..15cabe3d02 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitorSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/monitor/CreatingBillingProjectMonitorSpec.scala @@ -24,7 +24,6 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w val defaultServicePerimeterName: ServicePerimeterName = ServicePerimeterName("accessPolicies/policyName/servicePerimeters/servicePerimeterName") val defaultGoogleProjectNumber: GoogleProjectNumber = GoogleProjectNumber("42") - val defaultCromwellBucketUrl = "bucket-url" val defaultBillingProjectName = RawlsBillingProjectName("test-bp") def getCreatingBillingProjectMonitor(dataSource: SlickDataSource, mockGcsDAO: GoogleServicesDAO = new MockGoogleServicesDAO("test"))(implicit executionContext: ExecutionContext): CreatingBillingProjectMonitor = { @@ -40,7 +39,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w "CreatingBillingProjectMonitor" should "set project status to 'AddingToPerimeter' when it's been successfully created and it has a service perimeter" in { withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.Creating, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.Creating, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) val creatingOperation = RawlsBillingProjectOperationRecord(billingProject.projectName.value, GoogleOperationNames.DeploymentManagerCreateProject, "opid", true, None, GoogleApiTypes.DeploymentManagerApi) runAndWait(rawlsBillingProjectQuery.create(billingProject)) @@ -62,9 +61,9 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w it should "include projects that are already in the perimeter" in { withEmptyTestDatabase { dataSource: SlickDataSource => val newBillingProjectNumber = GoogleProjectNumber("1") - val newBillingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(newBillingProjectNumber)) + val newBillingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(newBillingProjectNumber)) val existingBillingProjectNumber = GoogleProjectNumber("2") - val existingBillingProject = RawlsBillingProject(RawlsBillingProjectName("existing-project"), defaultCromwellBucketUrl, CreationStatuses.Ready, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(existingBillingProjectNumber)) + val existingBillingProject = RawlsBillingProject(RawlsBillingProjectName("existing-project"), CreationStatuses.Ready, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(existingBillingProjectNumber)) runAndWait(rawlsBillingProjectQuery.create(newBillingProject)) runAndWait(rawlsBillingProjectQuery.create(existingBillingProject)) @@ -95,14 +94,14 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w withEmptyTestDatabase { dataSource: SlickDataSource => val projectNumber1 = GoogleProjectNumber("1") - val billingProject1 = RawlsBillingProject(RawlsBillingProjectName("test-bp1"), defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(projectNumber1)) + val billingProject1 = RawlsBillingProject(RawlsBillingProjectName("test-bp1"), CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(projectNumber1)) val projectNumber2 = GoogleProjectNumber("2") - val billingProject2 = RawlsBillingProject(RawlsBillingProjectName("test-bp2"), defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(projectNumber2)) + val billingProject2 = RawlsBillingProject(RawlsBillingProjectName("test-bp2"), CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(projectNumber2)) val otherServicePerimeter = ServicePerimeterName("other-perimeter") val projectNumber3 = GoogleProjectNumber("3") - val billingProject3 = RawlsBillingProject(RawlsBillingProjectName("test-bp3"), defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(otherServicePerimeter), googleProjectNumber = Option(projectNumber3)) + val billingProject3 = RawlsBillingProject(RawlsBillingProjectName("test-bp3"), CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(otherServicePerimeter), googleProjectNumber = Option(projectNumber3)) runAndWait(rawlsBillingProjectQuery.create(billingProject1)) runAndWait(rawlsBillingProjectQuery.create(billingProject2)) @@ -144,7 +143,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w // billing project in adding to perimeter, with still running operation -- should call google dao to poll but no state change withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) val addingProjectToPerimeterOperation = RawlsBillingProjectOperationRecord(billingProject.projectName.value, GoogleOperationNames.AddProjectToPerimeter, "opid", false, None, GoogleApiTypes.AccessContextManagerApi) runAndWait(rawlsBillingProjectQuery.create(billingProject)) @@ -170,7 +169,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w // billing project in adding to perimeter, with complete operation -- should call google dao to poll and change state (op done, project ready) withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) val addingProjectToPerimeterOperation = RawlsBillingProjectOperationRecord(billingProject.projectName.value, GoogleOperationNames.AddProjectToPerimeter, "opid", false, None, GoogleApiTypes.AccessContextManagerApi) runAndWait(rawlsBillingProjectQuery.create(billingProject)) @@ -198,7 +197,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w it should "raise an exception for duplicate primary keys when multiple operations with the same operation name exist for a single project" in { // billing project in adding to perimeter, with more than 1 operation -- project should error withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) val addingProjectToPerimeterOperation = RawlsBillingProjectOperationRecord(billingProject.projectName.value, GoogleOperationNames.AddProjectToPerimeter, "opid", false, None, GoogleApiTypes.AccessContextManagerApi) val extraAddProjectToPerimeterOperation = RawlsBillingProjectOperationRecord(billingProject.projectName.value, GoogleOperationNames.AddProjectToPerimeter, "opid", false, None, GoogleApiTypes.AccessContextManagerApi) @@ -212,7 +211,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w it should "set project state to 'Error' when its state is 'AddingToPerimeter' but no perimeter is saved in the DB" in { // billing project in adding to perimeter, no operations, no perimeter specified -- project should error withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = None, googleProjectNumber = Option(defaultGoogleProjectNumber)) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = None, googleProjectNumber = Option(defaultGoogleProjectNumber)) runAndWait(rawlsBillingProjectQuery.create(billingProject)) @@ -232,8 +231,8 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w it should "raise an exception when an existing project in a perimeter has no project number" in { // billing project in adding to perimeter, no operations but there is an existing project in perimeter with no project number -- exception withEmptyTestDatabase { dataSource: SlickDataSource => - val billingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) - val existingProjectWithoutNumber = RawlsBillingProject(RawlsBillingProjectName("no-google-project-number"), defaultCromwellBucketUrl, CreationStatuses.Ready, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = None) + val billingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(defaultGoogleProjectNumber)) + val existingProjectWithoutNumber = RawlsBillingProject(RawlsBillingProjectName("no-google-project-number"), CreationStatuses.Ready, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = None) runAndWait(rawlsBillingProjectQuery.create(billingProject)) runAndWait(rawlsBillingProjectQuery.create(existingProjectWithoutNumber)) @@ -254,7 +253,7 @@ class CreatingBillingProjectMonitorSpec extends MockitoSugar with FlatSpecLike w it should "set project state to 'Error' when call to Google fails" in { withEmptyTestDatabase { dataSource: SlickDataSource => val number = GoogleProjectNumber("1") - val billingProject = RawlsBillingProject(RawlsBillingProjectName("test-bp"), defaultCromwellBucketUrl, CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(number)) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("test-bp"), CreationStatuses.AddingToPerimeter, None, None, servicePerimeter = Option(defaultServicePerimeterName), googleProjectNumber = Option(number)) runAndWait(rawlsBillingProjectQuery.create(billingProject)) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/user/UserServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/user/UserServiceSpec.scala index d27b65c4c4..e2a196c116 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/user/UserServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/user/UserServiceSpec.scala @@ -25,9 +25,8 @@ class UserServiceSpec extends FlatSpecLike with TestDriverComponent with Mockito val defaultServicePerimeterName: ServicePerimeterName = ServicePerimeterName("accessPolicies/policyName/servicePerimeters/servicePerimeterName") val urlEncodedDefaultServicePerimeterName: String = URLEncoder.encode(defaultServicePerimeterName.value, UTF_8.name) val defaultGoogleProjectNumber: GoogleProjectNumber = GoogleProjectNumber("42") - val defaultCromwellBucketUrl: String = "bucket-url" val defaultBillingProjectName: RawlsBillingProjectName = RawlsBillingProjectName("test-bp") - val defaultBillingProject: RawlsBillingProject = RawlsBillingProject(defaultBillingProjectName, defaultCromwellBucketUrl, CreationStatuses.Ready, None, None, googleProjectNumber = Option(defaultGoogleProjectNumber)) + val defaultBillingProject: RawlsBillingProject = RawlsBillingProject(defaultBillingProjectName, CreationStatuses.Ready, None, None, googleProjectNumber = Option(defaultGoogleProjectNumber)) val defaultMockSamDAO: SamDAO = mock[SamDAO](RETURNS_SMART_NULLS) val defaultMockGcsDAO: GoogleServicesDAO = new MockGoogleServicesDAO("test") val testConf: Config = ConfigFactory.load() @@ -46,7 +45,6 @@ class UserServiceSpec extends FlatSpecLike with TestDriverComponent with Mockito gcsDAO, null, samDAO, - Seq.empty, "", DeploymentManagerConfig(testConf.getConfig("gcs.deploymentManager")), null @@ -75,10 +73,10 @@ class UserServiceSpec extends FlatSpecLike with TestDriverComponent with Mockito runAndWait(rawlsBillingProjectQuery.create(project)) val mockGcsDAO = mock[GoogleServicesDAO](RETURNS_SMART_NULLS) - when(mockGcsDAO.getGoogleProject(project.projectName)).thenReturn(Future.successful(new Project().setProjectNumber(42L))) + when(mockGcsDAO.getGoogleProject(project.googleProjectId)).thenReturn(Future.successful(new Project().setProjectNumber(42L))) val folderId = "folders/1234567" when(mockGcsDAO.getFolderId(defaultServicePerimeterName.value.split("/").last)).thenReturn(Future.successful(Option(folderId))) - when(mockGcsDAO.addProjectToFolder(project.projectName, folderId)).thenReturn(Future.successful(())) + when(mockGcsDAO.addProjectToFolder(project.googleProjectId, folderId)).thenReturn(Future.successful(())) val userService = getUserService(dataSource, gcsDAO = mockGcsDAO) @@ -163,20 +161,20 @@ class UserServiceSpec extends FlatSpecLike with TestDriverComponent with Mockito val mockSamDAO = mock[SamDAO](RETURNS_SMART_NULLS) when(mockSamDAO.userHasAction(SamResourceTypeNames.billingProject, project.projectName.value, SamBillingProjectActions.deleteBillingProject, userInfo)).thenReturn(Future.successful(true)) when(mockSamDAO.listAllResourceMemberIds(SamResourceTypeNames.billingProject, project.projectName.value, userInfo)).thenReturn(Future.successful(Set(userIdInfo))) - when(mockSamDAO.getPetServiceAccountKeyForUser(project.projectName.value, userInfo.userEmail)).thenReturn(Future.successful(petSAJson)) + when(mockSamDAO.getPetServiceAccountKeyForUser(project.googleProjectId, userInfo.userEmail)).thenReturn(Future.successful(petSAJson)) val mockGcsDAO = mock[GoogleServicesDAO](RETURNS_SMART_NULLS) when(mockGcsDAO.getUserInfoUsingJson(petSAJson)).thenReturn(Future.successful(userInfo)) - when(mockGcsDAO.deleteProject(project.projectName)).thenReturn(Future.successful()) - when(mockSamDAO.deleteUserPetServiceAccount(project.projectName.value, userInfo)).thenReturn(Future.successful()) + when(mockGcsDAO.deleteProject(project.googleProjectId)).thenReturn(Future.successful()) + when(mockSamDAO.deleteUserPetServiceAccount(project.googleProjectId, userInfo)).thenReturn(Future.successful()) when(mockSamDAO.deleteResource(SamResourceTypeNames.billingProject, project.projectName.value, userInfo)).thenReturn(Future.successful()) val userService = getUserService(dataSource, mockSamDAO, gcsDAO = mockGcsDAO) val actual = userService.DeleteBillingProject(defaultBillingProjectName).futureValue - verify(mockSamDAO).deleteUserPetServiceAccount(project.projectName.value, userInfo) + verify(mockSamDAO).deleteUserPetServiceAccount(project.googleProjectId, userInfo) verify(mockSamDAO).deleteResource(SamResourceTypeNames.billingProject, project.projectName.value, userInfo) - verify(mockGcsDAO).deleteProject(project.projectName) + verify(mockGcsDAO).deleteProject(project.googleProjectId) runAndWait(rawlsBillingProjectQuery.load(defaultBillingProjectName)) shouldBe empty actual shouldEqual RequestComplete(StatusCodes.NoContent) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala index 6ba5d23c22..e037717af1 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/ApiServiceSpec.scala @@ -152,7 +152,6 @@ trait ApiServiceSpec extends TestDriverComponentWithFlatSpecAndMatchers with Raw gcsDAO, notificationDAO, samDAO, - Seq("bigquery.jobUser"), "requesterPaysRole", DeploymentManagerConfig(testConf.getConfig("gcs.deploymentManager")), ProjectTemplate.from(testConf.getConfig("gcs.projectTemplate")) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceSpec.scala index 7b096ddc3b..603f2a0974 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/BillingApiServiceSpec.scala @@ -191,9 +191,6 @@ class BillingApiServiceSpec extends ApiServiceSpec with MockitoSugar { projects match { case Seq() => fail("project does not exist in db") case Seq(project) => - assertResult("gs://" + services.gcsDAO.getCromwellAuthBucketName(projectName)) { - project.cromwellAuthBucketUrl - } case _ => fail("too many projects") } } @@ -201,7 +198,7 @@ class BillingApiServiceSpec extends ApiServiceSpec with MockitoSugar { it should "rollback billing project inserts when there is a google error" in withDefaultTestDatabase { dataSource: SlickDataSource => withApiServices(dataSource, new MockGoogleServicesDAO("test") { - override def createProject(projectName: RawlsBillingProjectName, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = { + override def createProject(projectName: GoogleProjectId, billingAccount: RawlsBillingAccount, dmTemplatePath: String, highSecurityNetwork: Boolean, enableFlowLogs: Boolean, privateIpGoogleAccess: Boolean, requesterPaysRole: String, ownerGroupEmail: WorkbenchEmail, computeUserGroupEmail: WorkbenchEmail, projectTemplate: ProjectTemplate, parentFolderId: Option[String]): Future[RawlsBillingProjectOperationRecord] = { Future.failed(new Exception("test exception")) } }) { services => diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/PetSASpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/PetSASpec.scala index df164b5daf..bed91b1cf1 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/PetSASpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/PetSASpec.scala @@ -116,7 +116,7 @@ class PetSASpec extends ApiServiceSpec { val userSAProjectOwnerUserInfo = UserInfo(RawlsUserEmail("project-owner-access-sa@abc.iam.gserviceaccount.com"), OAuth2BearerToken("SA-but-not-pet-token"), 123, RawlsUserSubjectId("123456789876543210202")) val userSAProjectOwner = RawlsUser(userSAProjectOwnerUserInfo) - val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), CreationStatuses.Ready, None, None) val workspaceName = WorkspaceName(billingProject.projectName.value, "testworkspace") diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/SubmissionApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/SubmissionApiServiceSpec.scala index 8ea9e53fe6..ca8d551974 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/SubmissionApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/SubmissionApiServiceSpec.scala @@ -703,7 +703,7 @@ class SubmissionApiServiceSpec extends ApiServiceSpec with TableDrivenPropertyCh val ownerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-OWNER", Set(userOwner)) val writerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-WRITER", Set()) val readerGroup = makeRawlsGroup(s"${wsName.namespace}-${wsName.name}-READER", Set()) - val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName(wsName.namespace), CreationStatuses.Ready, None, None) val workspace = Workspace(wsName.namespace, wsName.name, UUID.randomUUID().toString, "aBucket", Some("workflow-collection"), currentTime(), currentTime(), "testUser", Map.empty) diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/UserApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/UserApiServiceSpec.scala index 2ed3157204..d2083ded5e 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/UserApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/UserApiServiceSpec.scala @@ -153,7 +153,7 @@ class UserApiServiceSpec extends ApiServiceSpec { } private def setupBillingProject(services: TestApiService): RawlsBillingProject = { - val project = RawlsBillingProject(RawlsBillingProjectName("project"), "mockBucketUrl", CreationStatuses.Ready, None, None) + val project = RawlsBillingProject(RawlsBillingProjectName("project"), CreationStatuses.Ready, None, None) val createRequest = CreateRawlsBillingProjectFullRequest(project.projectName, services.gcsDAO.accessibleBillingAccountName, None, None, None, None) @@ -291,7 +291,7 @@ class UserApiServiceSpec extends ApiServiceSpec { } it should "return 200 when adding a user to a billing project that the caller owns" in withTestDataApiServices { services => - val project1 = RawlsBillingProject(RawlsBillingProjectName("project1"), "mockBucketUrl", CreationStatuses.Ready, None, None) + val project1 = RawlsBillingProject(RawlsBillingProjectName("project1"), CreationStatuses.Ready, None, None) val createRequest = CreateRawlsBillingProjectFullRequest(project1.projectName, services.gcsDAO.accessibleBillingAccountName, None, None, None, None) import UserAuthJsonSupport.CreateRawlsBillingProjectFullRequestFormat @@ -319,7 +319,7 @@ class UserApiServiceSpec extends ApiServiceSpec { } it should "return 200 when removing a user from a billing project that the caller owns" in withTestDataApiServices { services => - val project1 = RawlsBillingProject(RawlsBillingProjectName("project1"), "mockBucketUrl", CreationStatuses.Ready, None, None) + val project1 = RawlsBillingProject(RawlsBillingProjectName("project1"), CreationStatuses.Ready, None, None) val createRequest = CreateRawlsBillingProjectFullRequest(project1.projectName, services.gcsDAO.accessibleBillingAccountName, None, None, None, None) import UserAuthJsonSupport.CreateRawlsBillingProjectFullRequestFormat diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiGetOptionsSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiGetOptionsSpec.scala index b1b252ee39..f4fdc3d6fe 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiGetOptionsSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiGetOptionsSpec.scala @@ -61,7 +61,7 @@ class WorkspaceApiGetOptionsSpec extends ApiServiceSpec { val userWriter = RawlsUser(UserInfo(testData.userWriter.userEmail, OAuth2BearerToken("token"), 123, RawlsUserSubjectId("123456789876543212346"))) val userReader = RawlsUser(UserInfo(testData.userReader.userEmail, OAuth2BearerToken("token"), 123, RawlsUserSubjectId("123456789876543212347"))) - val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), CreationStatuses.Ready, None, None) val workspaceName = WorkspaceName(billingProject.projectName.value, "testworkspace") val workspace2Name = WorkspaceName(billingProject.projectName.value, "emptyattrs") diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiListOptionsSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiListOptionsSpec.scala index 400bce0cf5..34fefc9eb7 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiListOptionsSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiListOptionsSpec.scala @@ -52,7 +52,7 @@ class WorkspaceApiListOptionsSpec extends ApiServiceSpec { class TestWorkspaces() extends TestData { val userOwner = RawlsUser(UserInfo(testData.userOwner.userEmail, OAuth2BearerToken("token"), 123, RawlsUserSubjectId("123456789876543212345"))) - val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), CreationStatuses.Ready, None, None) val workspaceName = WorkspaceName(billingProject.projectName.value, "testworkspace") diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceSpec.scala index 509f384e3a..556a504da8 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/webservice/WorkspaceApiServiceSpec.scala @@ -156,7 +156,7 @@ class WorkspaceApiServiceSpec extends ApiServiceSpec { val userWriter = RawlsUser(UserInfo(testData.userWriter.userEmail, OAuth2BearerToken("token"), 123, RawlsUserSubjectId("123456789876543212346"))) val userReader = RawlsUser(UserInfo(testData.userReader.userEmail, OAuth2BearerToken("token"), 123, RawlsUserSubjectId("123456789876543212347"))) - val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), "testBucketUrl", CreationStatuses.Ready, None, None) + val billingProject = RawlsBillingProject(RawlsBillingProjectName("ns"), CreationStatuses.Ready, None, None) val workspaceName = WorkspaceName(billingProject.projectName.value, "testworkspace") diff --git a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceSpec.scala b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceSpec.scala index d91fcb3de7..7c9683b78d 100644 --- a/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceSpec.scala +++ b/core/src/test/scala/org/broadinstitute/dsde/rawls/workspace/WorkspaceServiceSpec.scala @@ -107,7 +107,6 @@ class WorkspaceServiceSpec extends FlatSpec with ScalatestRouteTest with Matcher gcsDAO, notificationDAO, samDAO, - Seq("bigquery.jobUser"), "requesterPaysRole", DeploymentManagerConfig(testConf.getConfig("gcs.deploymentManager")), ProjectTemplate.from(testConf.getConfig("gcs.projectTemplate")) diff --git a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala index f3012c89a6..66a2644d52 100644 --- a/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala +++ b/model/src/main/scala/org/broadinstitute/dsde/rawls/model/WorkspaceModel.scala @@ -13,7 +13,7 @@ import org.broadinstitute.dsde.rawls.model.UserModelJsonSupport.ManagedGroupRefF import org.broadinstitute.dsde.rawls.model.WorkspaceAccessLevels.WorkspaceAccessLevel import org.broadinstitute.dsde.rawls.model.WorkspaceVersions.WorkspaceVersion import org.broadinstitute.dsde.rawls.{RawlsException, RawlsExceptionWithErrorReport} -import org.broadinstitute.dsde.workbench.model.ValueObject +import org.broadinstitute.dsde.workbench.model.{ValueObject, ValueObjectFormat} import org.joda.time.DateTime import spray.json._ @@ -101,6 +101,10 @@ object WorkspaceVersions { case _ => None } } + + def fromStringThrows(versionString: String): WorkspaceVersion = { + fromString(versionString).getOrElse(throw new RawlsException(s"unexpected version string ${versionString}, acceptable values are ${V1.value} or ${V2.value}")) + } } case class WorkspaceRequest ( @@ -116,6 +120,8 @@ case class WorkspaceRequest ( def path: String = toWorkspaceName.path } +case class GoogleProjectId(value: String) extends ValueObject + case class Workspace( namespace: String, name: String, @@ -128,7 +134,7 @@ case class Workspace( attributes: AttributeMap, isLocked: Boolean, workspaceVersion: WorkspaceVersion, - googleProject: String + googleProject: GoogleProjectId ) extends Attributable { def toWorkspaceName = WorkspaceName(namespace,name) def briefName: String = toWorkspaceName.toString @@ -148,7 +154,7 @@ object Workspace { createdBy: String, attributes: AttributeMap, isLocked: Boolean = false): Workspace = { - Workspace(namespace, name, workspaceId, bucketName, workflowCollectionName, createdDate, lastModified, createdBy, attributes, isLocked, WorkspaceVersions.V1, namespace) + Workspace(namespace, name, workspaceId, bucketName, workflowCollectionName, createdDate, lastModified, createdBy, attributes, isLocked, WorkspaceVersions.V1, GoogleProjectId(namespace)) } } @@ -497,7 +503,7 @@ case class WorkspaceDetails(namespace: String, isLocked: Boolean = false, authorizationDomain: Option[Set[ManagedGroupRef]], workspaceVersion: WorkspaceVersion, - googleProject: String) { + googleProject: GoogleProjectId) { def toWorkspace: Workspace = Workspace(namespace, name, workspaceId, bucketName, workflowCollectionName, createdDate, lastModified, createdBy, attributes.getOrElse(Map()), isLocked, workspaceVersion, googleProject) } @@ -769,6 +775,8 @@ class WorkspaceJsonSupport extends JsonSupport { } + implicit val GoogleProjectIdFormat = ValueObjectFormat(GoogleProjectId) + implicit val MethodConfigurationFormat = jsonFormat11(MethodConfiguration) implicit val AgoraMethodConfigurationFormat = jsonFormat7(AgoraMethodConfiguration)