TSPS-401 add output expiration date to getPipelineRunResult response (#…

…187)

Co-authored-by: Jose Soto <[email protected]>

common/openapi.yml

@@ -738,6 +738,12 @@ components:
       type: string
       format: string
 
+    PipelineOutputExpirationDate:
+      description: |
+        The Date when the pipeline outputs will expire.
+      type: string
+      format: string
+
     PipelineRun:
       description: |
         Object containing the job id, status, user-provided description, time submitted, and (if run is complete) time completed of a Pipeline Run.
@@ -777,6 +783,8 @@ components:
           $ref: "#/components/schemas/PipelineWdlMethodVersion"
         outputs:
           $ref: "#/components/schemas/PipelineRunOutputs"
+        outputExpirationDate:
+          $ref: "#/components/schemas/PipelineOutputExpirationDate"
 
     PipelineRunStatus:
       description: |

service/src/main/java/bio/terra/pipelines/app/configuration/internal/WdlPipelineConfiguration.java → service/src/main/java/bio/terra/pipelines/app/configuration/internal/PipelinesCommonConfiguration.java

@@ -4,10 +4,11 @@
 import lombok.Setter;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
-@ConfigurationProperties(prefix = "pipelines.wdl")
 @Getter
 @Setter
-public class WdlPipelineConfiguration {
+@ConfigurationProperties(prefix = "pipelines.common")
+public class PipelinesCommonConfiguration {
   private Long quotaConsumedPollingIntervalSeconds;
   private boolean quotaConsumedUseCallCaching;
+  private Long userDataTtlDays;
 }

service/src/main/java/bio/terra/pipelines/app/controller/PipelineRunsApiController.java

@@ -6,6 +6,7 @@
 import bio.terra.common.iam.SamUserFactory;
 import bio.terra.pipelines.app.configuration.external.IngressConfiguration;
 import bio.terra.pipelines.app.configuration.external.SamConfiguration;
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
 import bio.terra.pipelines.common.utils.CommonPipelineRunStatusEnum;
 import bio.terra.pipelines.common.utils.PipelinesEnum;
 import bio.terra.pipelines.common.utils.pagination.PageResponse;
@@ -19,6 +20,8 @@
 import bio.terra.pipelines.service.PipelinesService;
 import io.swagger.annotations.Api;
 import jakarta.servlet.http.HttpServletRequest;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -44,6 +47,7 @@ public class PipelineRunsApiController implements PipelineRunsApi {
   private final PipelineRunsService pipelineRunsService;
   private final PipelineInputsOutputsService pipelineInputsOutputsService;
   private final IngressConfiguration ingressConfiguration;
+  private final PipelinesCommonConfiguration pipelinesCommonConfiguration;
 
   @Autowired
   public PipelineRunsApiController(
@@ -54,7 +58,8 @@ public PipelineRunsApiController(
       PipelinesService pipelinesService,
       PipelineRunsService pipelineRunsService,
       PipelineInputsOutputsService pipelineInputsOutputsService,
-      IngressConfiguration ingressConfiguration) {
+      IngressConfiguration ingressConfiguration,
+      PipelinesCommonConfiguration pipelinesCommonConfiguration) {
     this.samConfiguration = samConfiguration;
     this.samUserFactory = samUserFactory;
     this.request = request;
@@ -63,6 +68,7 @@ public PipelineRunsApiController(
     this.pipelineRunsService = pipelineRunsService;
     this.pipelineInputsOutputsService = pipelineInputsOutputsService;
     this.ingressConfiguration = ingressConfiguration;
+    this.pipelinesCommonConfiguration = pipelinesCommonConfiguration;
   }
 
   private static final Logger logger = LoggerFactory.getLogger(PipelineRunsApiController.class);
@@ -249,6 +255,11 @@ private ApiAsyncPipelineRunResponse pipelineRunToApi(PipelineRun pipelineRun, Pi
 
     // if the pipeline run is successful, return the job report and add outputs to the response
     if (pipelineRun.getStatus().isSuccess()) {
+      // calculate the expiration date for the output files
+      Instant outputExpirationDate =
+          pipelineRun
+              .getUpdated()
+              .plus(pipelinesCommonConfiguration.getUserDataTtlDays(), ChronoUnit.DAYS);
       return response
           .jobReport(
               new ApiJobReport()
@@ -264,7 +275,8 @@ private ApiAsyncPipelineRunResponse pipelineRunToApi(PipelineRun pipelineRun, Pi
           .pipelineRunReport(
               response
                   .getPipelineRunReport()
-                  .outputs(pipelineInputsOutputsService.formatPipelineRunOutputs(pipelineRun)));
+                  .outputs(pipelineInputsOutputsService.formatPipelineRunOutputs(pipelineRun))
+                  .outputExpirationDate(outputExpirationDate.toString()));
     } else {
       JobApiUtils.AsyncJobResult<String> jobResult =
           jobService.retrieveAsyncJobResult(

service/src/main/java/bio/terra/pipelines/common/utils/FlightBeanBag.java

@@ -2,7 +2,7 @@
 
 import bio.terra.pipelines.app.configuration.external.CbasConfiguration;
 import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration;
-import bio.terra.pipelines.app.configuration.internal.WdlPipelineConfiguration;
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
 import bio.terra.pipelines.dependencies.cbas.CbasService;
 import bio.terra.pipelines.dependencies.leonardo.LeonardoService;
 import bio.terra.pipelines.dependencies.rawls.RawlsService;
@@ -42,7 +42,7 @@ public class FlightBeanBag {
   private final NotificationService notificationService;
   private final ImputationConfiguration imputationConfiguration;
   private final CbasConfiguration cbasConfiguration;
-  private final WdlPipelineConfiguration wdlPipelineConfiguration;
+  private final PipelinesCommonConfiguration pipelinesCommonConfiguration;
 
   @Lazy
   @Autowired
@@ -60,7 +60,7 @@ public FlightBeanBag(
       WorkspaceManagerService workspaceManagerService,
       ImputationConfiguration imputationConfiguration,
       CbasConfiguration cbasConfiguration,
-      WdlPipelineConfiguration wdlPipelineConfiguration) {
+      PipelinesCommonConfiguration pipelinesCommonConfiguration) {
     this.pipelinesService = pipelinesService;
     this.pipelineRunsService = pipelineRunsService;
     this.pipelineInputsOutputsService = pipelineInputsOutputsService;
@@ -74,7 +74,7 @@ public FlightBeanBag(
     this.notificationService = notificationService;
     this.imputationConfiguration = imputationConfiguration;
     this.cbasConfiguration = cbasConfiguration;
-    this.wdlPipelineConfiguration = wdlPipelineConfiguration;
+    this.pipelinesCommonConfiguration = pipelinesCommonConfiguration;
   }
 
   public static FlightBeanBag getFromObject(Object object) {

service/src/main/java/bio/terra/pipelines/stairway/flights/imputation/RunImputationGcpJobFlight.java

@@ -85,14 +85,14 @@ public RunImputationGcpJobFlight(FlightMap inputParameters, Object beanBag) {
         new SubmitQuotaConsumedSubmissionStep(
             flightBeanBag.getRawlsService(),
             flightBeanBag.getSamService(),
-            flightBeanBag.getWdlPipelineConfiguration()),
+            flightBeanBag.getPipelinesCommonConfiguration()),
         externalServiceRetryRule);
 
     addStep(
         new PollQuotaConsumedSubmissionStatusStep(
             flightBeanBag.getRawlsService(),
             flightBeanBag.getSamService(),
-            flightBeanBag.getWdlPipelineConfiguration()),
+            flightBeanBag.getPipelinesCommonConfiguration()),
         externalServiceRetryRule);
 
     addStep(

service/src/main/java/bio/terra/pipelines/stairway/steps/common/PollQuotaConsumedSubmissionStatusStep.java

@@ -1,6 +1,6 @@
 package bio.terra.pipelines.stairway.steps.common;
 
-import bio.terra.pipelines.app.configuration.internal.WdlPipelineConfiguration;
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
 import bio.terra.pipelines.common.utils.FlightUtils;
 import bio.terra.pipelines.dependencies.rawls.RawlsService;
 import bio.terra.pipelines.dependencies.sam.SamService;
@@ -22,17 +22,17 @@
 public class PollQuotaConsumedSubmissionStatusStep implements Step {
   private final RawlsService rawlsService;
   private final SamService samService;
-  private final WdlPipelineConfiguration wdlPipelineConfiguration;
+  private final PipelinesCommonConfiguration pipelinesCommonConfiguration;
   private final Logger logger =
       LoggerFactory.getLogger(PollQuotaConsumedSubmissionStatusStep.class);
 
   public PollQuotaConsumedSubmissionStatusStep(
       RawlsService rawlsService,
       SamService samService,
-      WdlPipelineConfiguration wdlPipelineConfiguration) {
+      PipelinesCommonConfiguration pipelinesCommonConfiguration) {
     this.samService = samService;
     this.rawlsService = rawlsService;
-    this.wdlPipelineConfiguration = wdlPipelineConfiguration;
+    this.pipelinesCommonConfiguration = pipelinesCommonConfiguration;
   }
 
   @Override
@@ -57,7 +57,7 @@ public StepResult doStep(FlightContext flightContext) throws InterruptedExceptio
         new RawlsSubmissionStepHelper(
             rawlsService, samService, controlWorkspaceProject, controlWorkspaceName, logger);
     return rawlsSubmissionStepHelper.pollRawlsSubmissionHelper(
-        quotaSubmissionId, wdlPipelineConfiguration.getQuotaConsumedPollingIntervalSeconds());
+        quotaSubmissionId, pipelinesCommonConfiguration.getQuotaConsumedPollingIntervalSeconds());
   }
 
   @Override

service/src/main/java/bio/terra/pipelines/stairway/steps/common/SubmitQuotaConsumedSubmissionStep.java

@@ -1,6 +1,6 @@
 package bio.terra.pipelines.stairway.steps.common;
 
-import bio.terra.pipelines.app.configuration.internal.WdlPipelineConfiguration;
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
 import bio.terra.pipelines.common.utils.FlightUtils;
 import bio.terra.pipelines.common.utils.PipelineVariableTypesEnum;
 import bio.terra.pipelines.common.utils.PipelinesEnum;
@@ -32,7 +32,7 @@
 public class SubmitQuotaConsumedSubmissionStep implements Step {
   private final SamService samService;
   private final RawlsService rawlsService;
-  private final WdlPipelineConfiguration wdlPipelineConfiguration;
+  private final PipelinesCommonConfiguration pipelinesCommonConfiguration;
 
   public static final String QUOTA_CONSUMED_METHOD_NAME = "QuotaConsumed";
   public static final List<PipelineOutputDefinition> QUOTA_CONSUMED_OUTPUT_DEFINITION_LIST =
@@ -45,10 +45,10 @@ public class SubmitQuotaConsumedSubmissionStep implements Step {
   public SubmitQuotaConsumedSubmissionStep(
       RawlsService rawlsService,
       SamService samService,
-      WdlPipelineConfiguration wdlPipelineConfiguration) {
+      PipelinesCommonConfiguration pipelinesCommonConfiguration) {
     this.samService = samService;
     this.rawlsService = rawlsService;
-    this.wdlPipelineConfiguration = wdlPipelineConfiguration;
+    this.pipelinesCommonConfiguration = pipelinesCommonConfiguration;
   }
 
   @Override
@@ -102,7 +102,7 @@ public StepResult doStep(FlightContext flightContext) {
         new SubmissionRequest()
             .entityName(flightContext.getFlightId())
             .entityType(pipelineName.getValue())
-            .useCallCache(wdlPipelineConfiguration.isQuotaConsumedUseCallCaching())
+            .useCallCache(pipelinesCommonConfiguration.isQuotaConsumedUseCallCaching())
             .deleteIntermediateOutputFiles(true)
             .useReferenceDisks(false)
             .userComment(

service/src/main/resources/application.yml

@@ -135,6 +135,11 @@ imputation:
   useReferenceDisk: false
 
 pipelines:
+  common:
+    userDataTtlDays: 14
+    quotaConsumedPollingIntervalSeconds: 60
+    quotaConsumedUseCallCaching: false
+
   ingress:
     domainName: ${env.ingress.domainName}
 
@@ -154,10 +159,6 @@ pipelines:
     dsn: ${SENTRY_DSN:}
     environment: ${DEPLOY_ENV:}
 
-  wdl:
-    quotaConsumedPollingIntervalSeconds: 60
-    quotaConsumedUseCallCaching: false
-
   notifications:
     projectId: ${env.pipelines.notifications.projectId}
     topicId: ${env.pipelines.notifications.topicId}

service/src/test/java/bio/terra/pipelines/common/utils/FlightBeanBagTest.java

@@ -4,7 +4,7 @@
 
 import bio.terra.pipelines.app.configuration.external.CbasConfiguration;
 import bio.terra.pipelines.app.configuration.internal.ImputationConfiguration;
-import bio.terra.pipelines.app.configuration.internal.WdlPipelineConfiguration;
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
 import bio.terra.pipelines.dependencies.cbas.CbasService;
 import bio.terra.pipelines.dependencies.leonardo.LeonardoService;
 import bio.terra.pipelines.dependencies.rawls.RawlsService;
@@ -40,7 +40,7 @@ class FlightBeanBagTest extends BaseEmbeddedDbTest {
 
   @Autowired private ImputationConfiguration imputationConfiguration;
   @Autowired private CbasConfiguration cbasConfiguration;
-  @Autowired private WdlPipelineConfiguration wdlPipelineConfiguration;
+  @Autowired private PipelinesCommonConfiguration pipelinesCommonConfiguration;
 
   @Test
   void testFlightBeanBag() {
@@ -59,7 +59,7 @@ void testFlightBeanBag() {
             workspaceManagerService,
             imputationConfiguration,
             cbasConfiguration,
-            wdlPipelineConfiguration);
+            pipelinesCommonConfiguration);
     assertEquals(pipelinesService, flightBeanBag.getPipelinesService());
     assertEquals(pipelineRunsService, flightBeanBag.getPipelineRunsService());
     assertEquals(pipelineInputsOutputsService, flightBeanBag.getPipelineInputsOutputsService());
@@ -73,6 +73,6 @@ void testFlightBeanBag() {
     assertEquals(notificationService, flightBeanBag.getNotificationService());
     assertEquals(imputationConfiguration, flightBeanBag.getImputationConfiguration());
     assertEquals(cbasConfiguration, flightBeanBag.getCbasConfiguration());
-    assertEquals(wdlPipelineConfiguration, flightBeanBag.getWdlPipelineConfiguration());
+    assertEquals(pipelinesCommonConfiguration, flightBeanBag.getPipelinesCommonConfiguration());
   }
 }

service/src/test/java/bio/terra/pipelines/configuration/internal/PipelinesCommonConfigurationTest.java

@@ -0,0 +1,20 @@
+package bio.terra.pipelines.configuration.internal;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import bio.terra.pipelines.app.configuration.internal.PipelinesCommonConfiguration;
+import bio.terra.pipelines.testutils.BaseEmbeddedDbTest;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+class PipelinesCommonConfigurationTest extends BaseEmbeddedDbTest {
+
+  @Autowired private PipelinesCommonConfiguration pipelinesCommonConfiguration;
+
+  @Test
+  void testPipelinesCommonConfiguration() {
+    assertEquals(1, pipelinesCommonConfiguration.getQuotaConsumedPollingIntervalSeconds());
+    assertTrue(pipelinesCommonConfiguration.isQuotaConsumedUseCallCaching());
+    assertEquals(2, pipelinesCommonConfiguration.getUserDataTtlDays());
+  }
+}