From d70e87ccdef314258511ca6788cd4ed3c85a5c40 Mon Sep 17 00:00:00 2001 From: michael-conway Date: Wed, 25 May 2016 13:31:02 -0700 Subject: [PATCH] #2 fix weirdness --- publisher-api | 1 - publisher-apix/pom.xml | 21 ++ .../publish/mechanism/api/JobSubmission.java | 323 ++++++++++++++++++ .../api/PublishActionDescriptor.java | 102 ++++++ .../publish/mechanism/api/PublishContext.java | 41 +++ .../mechanism/api/PublishMechanism.java | 71 ++++ .../mechanism/api/PublishPhaseEnum.java | 15 + .../publish/mechanism/api/PublishResult.java | 205 +++++++++++ .../mechanism/api/PublishStatusEnum.java | 15 + 9 files changed, 793 insertions(+), 1 deletion(-) delete mode 160000 publisher-api create mode 100644 publisher-apix/pom.xml create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/JobSubmission.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishActionDescriptor.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishContext.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishMechanism.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishPhaseEnum.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishResult.java create mode 100644 publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishStatusEnum.java diff --git a/publisher-api b/publisher-api deleted file mode 160000 index cbfc186..0000000 --- a/publisher-api +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cbfc186fd0e61772ab39dbc9f4b887b8f3dcdc4e diff --git a/publisher-apix/pom.xml b/publisher-apix/pom.xml new file mode 100644 index 0000000..446d844 --- /dev/null +++ b/publisher-apix/pom.xml @@ -0,0 +1,21 @@ + + + + org.iplantc.de + publisher + 1.0.0-SNAPSHOT + + 4.0.0 + org.iplantc.de + publisher-api + 1.0.0-SNAPSHOT + jar + publisher framework for DE - API + + + org.irods.jargon + jargon-core + + + diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/JobSubmission.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/JobSubmission.java new file mode 100644 index 0000000..1379acf --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/JobSubmission.java @@ -0,0 +1,323 @@ +package org.iplantc.de.publish.mechanism.api; + +import java.util.HashMap; +import java.util.Map; + +/** + * Represents a job submission to be sent to the Discovery Environment for + * asynchronous data publication. + */ +public class JobSubmission { + + public static final int MAX_JOB_NAME_LENGTH = 255; + public static final int MAX_APP_ID_LENGTH = 255; + + /** + * The job name. This field should contain a brief ({@code MAX_JOB_NAME_LENGTH} + * characters maximum) description of the job. Duplicate job names are permitted, + * but it is helpful for the job name to be unique. + */ + private final String name; + + /** + * @return the job name. + */ + public String getName() { + return name; + } + + /** + * The job description. This field contains a longer description of the job. + */ + private final String description; + + /** + * @return the job description. + */ + public String getDescription() { + return description; + } + + /** + * The ID of the Discovery Environment app that is used to determine how the + * job will be executed. + */ + private final String appId; + + /** + * @return the app ID. + */ + public String getAppId() { + return appId; + } + + /** + * The path to the directory in iRODS where log files will be placed after + * the publishing job is complete. + */ + private final String outputDir; + + /** + * @return the path to the output directory. + */ + public String getOutputDir() { + return outputDir; + } + + /** + * If set to {@code true}, the Discovery Environment will automatically + * create a subdirectory of the selected output directory, with a name + * that is generated from the job name and the job submission time. + */ + private final boolean createOutputSubdir; + + /** + * @return true if a subdirectory should be created for output files. + */ + public boolean shouldCreateOutputSubdir() { + return createOutputSubdir; + } + + /** + * If set to {@code true}, the Discovery Environment will archive log files + * from the job execution as job output files. + */ + private final boolean archiveLogs; + + /** + * @return true if log files should be archived. + */ + public boolean shouldArchiveLogs() { + return archiveLogs; + } + + /** + * If set to {@code true}, the Discovery Environment will notify the user + * when the job status changes and will send an email when the job is + * complete. + */ + private final boolean notify; + + /** + * @return true if the user should be notified of job status changes. + */ + public boolean shouldNotify() { + return notify; + } + + /** + * A map of parameter IDs to parameter values. This map determines which + * values are passed to the command line when the job is executed. + */ + private final Map parameterValues; + + /** + * @param builder the job submission builder. + */ + private JobSubmission(JobSubmissionBuilder builder) { + this.name = builder.name; + this.description = builder.description; + this.appId = builder.appId; + this.outputDir = builder.outputDir; + this.createOutputSubdir = builder.createOutputSubdir; + this.archiveLogs = builder.archiveLogs; + this.notify = builder.notify; + this.parameterValues = builder.parameterValues; + } + + /** + * A builder class that can be used to build a new job submission. + */ + public static class JobSubmissionBuilder { + private String name; + private String description; + private String appId; + private String outputDir; + private boolean createOutputSubdir = true; + private boolean archiveLogs = true; + private boolean notify = true; + private Map parameterValues = new HashMap(); + + /** + * Sets the name of the job, which has a maximum length of + * {@code MAX_JOB_NAME_LENGTH} characters. It is helpful, but not + * absolutely necessary, for the job name to be unique. This field + * is required. + * + * @param name the job name. + * @return the job submission builder. + */ + public JobSubmissionBuilder setName(String name) { + this.name = name; + return this; + } + + /** + * Sets the job description. This field is optional. + * + * @param description the job description. + * @return the job submission builder. + */ + public JobSubmissionBuilder setDescription(String description) { + this.description = description; + return this; + } + + /** + * Sets the ID of the app to use when a job is submitted. This field + * is required and has a maximum length of {@code MAX_APP_ID_LENGTH} + * characters. + * + * @param appId the app ID. + * @return the job submission builder. + */ + public JobSubmissionBuilder setAppId(String appId) { + this.appId = appId; + return this; + } + + /** + * Sets the path to the output directory in iRODS, which is where the + * log files and any job output files will be stored when they are + * archived. This field is required. + * + * @param outputDir the path to the output directory. + * @return the job submission builder. + */ + public JobSubmissionBuilder setOutputDir(String outputDir) { + this.outputDir = outputDir; + return this; + } + + /** + * Sets a flag indicating whether or not a subdirectory should be + * created underneath the selected output directory. If set to + * {@code true}, a new subdirectory with a named based on the job + * name and the job submission time will be created. This field + * defaults to {@code true} if not specified. + * + * @param create {@code true} if a subdirectory should be created. + * @return the job submission builder. + */ + public JobSubmissionBuilder setCreateOutputSubdir(boolean create) { + this.createOutputSubdir = create; + return this; + } + + /** + * Sets a flag indicating whether or not the log files that are + * generated during job execution should be archived in the job + * output directory (recommended). This field defaults to {@code true} + * if not specified. + * + * @param archive {@code true} if the log files should be archived. + * @return the job submission builder. + */ + public JobSubmissionBuilder setArchiveLogs(boolean archive) { + this.archiveLogs = archive; + return this; + } + + /** + * Sets a flag indicating whether or not the user should be notified + * when the job status changes (recommended). This field defaults to + * {@code true} if not specified. + * + * @param notify {@code true} if the user should be notified. + * @return the job submission builder. + */ + public JobSubmissionBuilder setNotify(boolean notify) { + this.notify = notify; + return this; + } + + /** + * Sets a property value for the job submission. The property ID is + * obtained from the app description. The property value is the value + * that should be used for that property when the job is executed. + * + * @param id the property ID. + * @param value the property value. + * @return the job submission builder. + */ + public JobSubmissionBuilder setParameterValue(String id, String value) { + this.parameterValues.put(id, value); + return this; + } + + /** + * Buildes the job submission. + * + * @return the job submission. + */ + public JobSubmission build() { + validateJobName(); + validateAppId(); + validateOutputDir(); + return new JobSubmission(this); + } + + /** + * Validates the job name. + * + * @throws NullPointerException if the job name is null. + * @throws IllegalArgumentException if the job name is empty or too long. + */ + private void validateJobName() { + if (name == null) { + throw new NullPointerException("a job name is required"); + } + if (name.length() == 0) { + throw new IllegalArgumentException("the job name may not be empty"); + } + if (name.length() > MAX_JOB_NAME_LENGTH) { + String msg = "the job name may not be more than " + MAX_JOB_NAME_LENGTH + + " characters long"; + throw new IllegalArgumentException(msg); + } + } + + /** + * Validates the app ID. + * + * @throws NullPointerException if the app ID is null. + * @throws IllegalArgumentException if the app ID is empty or too long. + */ + private void validateAppId() { + if (appId == null) { + throw new NullPointerException("an app ID is required"); + } + if (appId.length() == 0) { + throw new IllegalArgumentException("the app ID may not be empty"); + } + if (appId.length() > MAX_APP_ID_LENGTH) { + String msg = "the app ID may not be more than " + MAX_APP_ID_LENGTH + + " characters long"; + throw new IllegalArgumentException(msg); + } + } + + /** + * Validates the output directory. + * + * @throws NullPointerException if the output directory is null. + * @throws IllegalArgumentException if the output directory is empty. + */ + private void validateOutputDir() { + if (outputDir == null) { + throw new NullPointerException("the output directory may not be null"); + } + if (outputDir.length() == 0) { + throw new IllegalArgumentException("the output directory may not be empty"); + } + } + } + + /** + * @return a {@link JobSubmissionBuilder} that can be used to build a new + * job submission. + */ + public static JobSubmissionBuilder builder() { + return new JobSubmissionBuilder(); + } +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishActionDescriptor.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishActionDescriptor.java new file mode 100644 index 0000000..6308068 --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishActionDescriptor.java @@ -0,0 +1,102 @@ +package org.iplantc.de.publish.mechanism.api; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * Describes metadata associated with a publish action + * + * @author Mike Conway - DICE + * + */ +public class PublishActionDescriptor { + + /** + * absolute path to the iRODS collection that is the 'source' of the + * publish. The target will be determined by the publish mechanism, but may + * be guided by add'l parameters unique to that publish mechanism. These + * parameters may be provided to a publish mechanism through multiple means, + * and this descriptor provides an arbitrary map of parameter settings that + * may be interpreted by the publish mechanism + */ + private String publishSourceAbsolutePath = ""; + + /** + * Map of free-form string parameters that may pass information to the + * publish mechanism to tune the publish behavior + */ + private Map publishProperties = new HashMap(); + + public PublishActionDescriptor() { + } + + /** + * @return the publishSourceAbsolutePath + */ + public String getPublishSourceAbsolutePath() { + return publishSourceAbsolutePath; + } + + /** + * @param publishSourceAbsolutePath + * the publishSourceAbsolutePath to set + */ + public void setPublishSourceAbsolutePath(String publishSourceAbsolutePath) { + this.publishSourceAbsolutePath = publishSourceAbsolutePath; + } + + /** + * @return the publishProperties + */ + public Map getPublishProperties() { + return publishProperties; + } + + /** + * @param publishProperties + * the publishProperties to set + */ + public void setPublishProperties(Map publishProperties) { + this.publishProperties = publishProperties; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + final int maxLen = 10; + StringBuilder builder = new StringBuilder(); + builder.append("PublishActionDescriptor ["); + if (publishSourceAbsolutePath != null) { + builder.append("publishSourceAbsolutePath=") + .append(publishSourceAbsolutePath).append(", "); + } + if (publishProperties != null) { + builder.append("publishProperties=").append( + toString(publishProperties.entrySet(), maxLen)); + } + builder.append("]"); + return builder.toString(); + } + + private String toString(Collection collection, int maxLen) { + StringBuilder builder = new StringBuilder(); + builder.append("["); + int i = 0; + for (Iterator iterator = collection.iterator(); iterator.hasNext() + && i < maxLen; i++) { + if (i > 0) { + builder.append(", "); + } + builder.append(iterator.next()); + } + builder.append("]"); + return builder.toString(); + } + +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishContext.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishContext.java new file mode 100644 index 0000000..f20910f --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishContext.java @@ -0,0 +1,41 @@ +/** + * + */ +package org.iplantc.de.publish.mechanism.api; + +import org.irods.jargon.core.connection.IRODSAccount; +import org.irods.jargon.core.pub.IRODSAccessObjectFactory; + +import java.util.Map; + +/** + * Provides callbacks and context from DE publish service to the underlying + * foreground mechanism. This allows the preValidate and synchronous parts of + * the publish methods the ability to manipulate iRODS or call DE based services + * that are selectively exposed. + * + * @author Mike Conway - DICE + * + */ +public interface PublishContext { + + /** + * @return the account to use when connecting to iRODS. + */ + IRODSAccount getIrodsAccount(); + + /** + * @return the iRODS access object factory. + */ + IRODSAccessObjectFactory getAccessObjectFactory(); + + /** + * @param irodsPath an absolute path to a data object or collection in iRODS. + * @return the metadata associated with {@code irodsPath}. + */ + Map getPathMetadata(String irodsPath); + + /* + * TODO: add hooks for submitting jobs and for obtaining information about apps. + */ +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishMechanism.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishMechanism.java new file mode 100644 index 0000000..2b7a3a7 --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishMechanism.java @@ -0,0 +1,71 @@ +/** + * + */ +package org.iplantc.de.publish.mechanism.api; + +/** + * Describes a publish mechanism, this is a plugin that can format and publish + * collection data sorted as an AIP in the DE/iRODS environment as a DIP in an + * arbitrary target repository. The mechanism is a simple api that is developed + * underneath this interface, and can surface a preValidate foreground method + * that can do sanity checks and pre-processing, and a sych|asynch publish + * action that actually creates the DIP in the target repository. + * + * @author Mike Conway - DICE, Dennis Roberts, CyVerse + * + */ +public interface PublishMechanism { + + /** + * Do a pre-validate that may check for all necessary requirements, and may + * also pre-process the data before publishing. This is necessary as the + * actual publish phase may be run asynchronously as an app. Having an + * initial foreground step thus gives a hook for a publish mechanism that + * needs to itneract with iRODS or DE. + *

+ * Note that, depending on the mechanism, the publish action can also run in + * the foreground, and this may be suitable for light-weight publishing + * workflows. + *

+ * Note that the publish action should trap any checked or unchecked + * exception and internally convert it into a PublishResult so + * that the publish mechanism can handle exception and validation logic. + * + * @param publishActionDescriptor + * {@link PublishActionDescriptor} that provides the necessary + * context for the mechanism to find and do the publishing. + * @param publishContext + * {@link PublishContext} with callbacks and hooks to call DE and + * iRODS services + * @return {@link PublishResult} That represents the status and any error or + * validation information + */ + public PublishResult preValidate( + final PublishActionDescriptor publishActionDescriptor, + final PublishContext publishContext); + + /** + * This method will be called after success in the preValidate + * and represents the actual publish action. This may be done synchronously, + * or the result can reflect an asynchronous submission of a publication + * action. Asynchronous actions are handled through the DE App abstraction, + * and results are processed using the notification and error handling logic + * as it exists. Publish mechansims can also handle exceptions and any + * required clean-up witin this publish mechanism. + * + * @param publishActionDescriptor + * {@link PublishDescriptor} with the necessary context for an + * individual publicastion action + * @param publishContext + * {@link PublishContext} with callbacks and hooks to call DE and + * iRODS services + * @return {@link PublishResult} with the result of this publish action. For + * synchronous actions, this may be the failure or completion + * success, for an asynchyronous action, it would represent the + * success or failure in enquing the the publish action. + */ + public PublishResult publish( + final PublishActionDescriptor publishActionDescriptor, + final PublishContext publishContext); + +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishPhaseEnum.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishPhaseEnum.java new file mode 100644 index 0000000..1e4eccd --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishPhaseEnum.java @@ -0,0 +1,15 @@ +/** + * + */ +package org.iplantc.de.publish.mechanism.api; + +/** + * Describes a phase of publishing + * + * @author Mike Conway - DICE + * + */ +public enum PublishPhaseEnum { + REQUEST, PRE_VALIDATE, PUBLISH + +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishResult.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishResult.java new file mode 100644 index 0000000..b359388 --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishResult.java @@ -0,0 +1,205 @@ +/** + * + */ +package org.iplantc.de.publish.mechanism.api; + +import java.util.ArrayList; +import java.util.List; + +/** + * Describes the result of a pre-publish or publish action. This can include + * status information, and any validation messages, including an overall result + * as well as the ability to pass validation information per parameter + * + * @author Mike Conway - DICE + * + */ +public class PublishResult { + + /** + * Represents the phase of the publishing cycle that this response pertains + * to + */ + private PublishPhaseEnum publishPhase; + /** + * Enum value that represents the status at this particular phase + */ + + /** + * Represents an intermediate location for a DIP that may have been created + * during the pre-process phase. This allows a publish mechanism to publish + * from a location that is different than the original source. For example, + * the preprocess step may have created a new arrangement, or it may have + * created some sort of bag or archive file. + *

+ * The publish system will pass this intermediate DIP location between the + * result of the validate and the publication step. + */ + private String intermediateDipAbsolutePath; + + private PublishStatusEnum publishStatus; + + /** + * Numeric response code that may be particular to the publish mechanism + */ + private int responseCode = 0; + + /** + * Overall free text response for the publish result + */ + private String responseMessage = ""; + + /** + * Sequential list of log or validation messages + */ + private List messageLog = new ArrayList(); + + /** + * Optional stack trace information + */ + private String stackTrace = ""; + + /** + * @return the publishPhase + */ + public PublishPhaseEnum getPublishPhase() { + return publishPhase; + } + + /** + * @param publishPhase + * the publishPhase to set + */ + public void setPublishPhase(PublishPhaseEnum publishPhase) { + this.publishPhase = publishPhase; + } + + /** + * @return the publishStatus + */ + public PublishStatusEnum getPublishStatus() { + return publishStatus; + } + + /** + * @param publishStatus + * the publishStatus to set + */ + public void setPublishStatus(PublishStatusEnum publishStatus) { + this.publishStatus = publishStatus; + } + + /** + * @return the responseCode + */ + public int getResponseCode() { + return responseCode; + } + + /** + * @param responseCode + * the responseCode to set + */ + public void setResponseCode(int responseCode) { + this.responseCode = responseCode; + } + + /** + * @return the responseMessage + */ + public String getResponseMessage() { + return responseMessage; + } + + /** + * @param responseMessage + * the responseMessage to set + */ + public void setResponseMessage(String responseMessage) { + this.responseMessage = responseMessage; + } + + /** + * @return the messageLog + */ + public List getMessageLog() { + return messageLog; + } + + /** + * @param messageLog + * the messageLog to set + */ + public void setMessageLog(List messageLog) { + this.messageLog = messageLog; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + final int maxLen = 10; + StringBuilder builder = new StringBuilder(); + builder.append("PublishResult ["); + if (publishPhase != null) { + builder.append("publishPhase=").append(publishPhase).append(", "); + } + if (intermediateDipAbsolutePath != null) { + builder.append("intermediateDipAbsolutePath=") + .append(intermediateDipAbsolutePath).append(", "); + } + if (publishStatus != null) { + builder.append("publishStatus=").append(publishStatus).append(", "); + } + builder.append("responseCode=").append(responseCode).append(", "); + if (responseMessage != null) { + builder.append("responseMessage=").append(responseMessage) + .append(", "); + } + if (messageLog != null) { + builder.append("messageLog=") + .append(messageLog.subList(0, + Math.min(messageLog.size(), maxLen))).append(", "); + } + if (stackTrace != null) { + builder.append("stackTrace=").append(stackTrace); + } + builder.append("]"); + return builder.toString(); + } + + /** + * @return the stackTrace + */ + public String getStackTrace() { + return stackTrace; + } + + /** + * @param stackTrace + * the stackTrace to set + */ + public void setStackTrace(String stackTrace) { + this.stackTrace = stackTrace; + } + + /** + * @return the intermediateDipAbsolutePath + */ + public String getIntermediateDipAbsolutePath() { + return intermediateDipAbsolutePath; + } + + /** + * @param intermediateDipAbsolutePath + * the intermediateDipAbsolutePath to set + */ + public void setIntermediateDipAbsolutePath( + String intermediateDipAbsolutePath) { + this.intermediateDipAbsolutePath = intermediateDipAbsolutePath; + } + +} diff --git a/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishStatusEnum.java b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishStatusEnum.java new file mode 100644 index 0000000..c6059c7 --- /dev/null +++ b/publisher-apix/src/main/java/org/iplantc/de/publish/mechanism/api/PublishStatusEnum.java @@ -0,0 +1,15 @@ +/** + * + */ +package org.iplantc.de.publish.mechanism.api; + +/** + * Describes publish outcomes that can be represented by a publish status. + * + * @author Mike Conway - DICE + * + */ +public enum PublishStatusEnum { + VALIDATION_ERROR, SUBMITTED, COMPLETE, FAILURE + +}