diff --git a/cadc-test-vos/build.gradle b/cadc-test-vos/build.gradle index e4d65602..1fc36144 100644 --- a/cadc-test-vos/build.gradle +++ b/cadc-test-vos/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.1.2' +version = '2.1.3' description = 'OpenCADC VOSpace test library' def git_url = 'https://github.com/opencadc/vos' diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java index 6e6795cf..0981ec68 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java @@ -73,6 +73,7 @@ import ca.nrc.cadc.net.FileContent; import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.uws.ExecutionPhase; import ca.nrc.cadc.uws.Job; @@ -85,9 +86,12 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.charset.Charset; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.junit.Assert; @@ -108,6 +112,8 @@ public class TransferTest extends VOSTest { private static final Logger log = Logger.getLogger(TransferTest.class); + private static final List PUT_OK = Arrays.asList(new Integer[] { 200, 201}); + protected TransferTest(URI resourceID, File testCert) { super(resourceID, testCert); } @@ -127,49 +133,86 @@ public void syncPushPullTest() { // Create a push-to-vospace Transfer for the node Transfer pushTransfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace); pushTransfer.version = VOS.VOSPACE_21; - Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); - pushTransfer.getProtocols().add(protocol); + pushTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth + Protocol putWithCert = new Protocol(VOS.PROTOCOL_HTTPS_PUT); + putWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + pushTransfer.getProtocols().add(putWithCert); - // Do the transfer + // negotiate the transfer Transfer details = doTransfer(pushTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace, Direction.pushToVoSpace, details.getDirection()); - Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); - String endpoint = null; + Assert.assertNotNull(details.getProtocols()); + log.info(pushTransfer.getDirection() + " results: " + details.getProtocols().size()); + URL putURL = null; for (Protocol p : details.getProtocols()) { + String endpoint = p.getEndpoint(); + log.info("PUT endpoint: " + endpoint); try { - endpoint = p.getEndpoint(); - log.debug("endpoint: " + endpoint); - new URL(endpoint); + + URL u = new URL(endpoint); + if (putURL == null) { + putURL = u; // first + } } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); } } + Assert.assertNotNull(putURL); + + // put the bytes + Random rnd = new Random(); + byte[] data = new byte[1024]; + rnd.nextBytes(data); + FileContent content = new FileContent(data, "application/octet-stream"); + HttpUpload put = new HttpUpload(content, putURL); + put.run(); + log.info("put: " + put.getResponseCode() + " " + put.getThrowable()); + Assert.assertTrue(PUT_OK.contains(put.getResponseCode())); + Assert.assertNull(put.getThrowable()); // Create a pull-from-vospace Transfer for the node Transfer pullTransfer = new Transfer(nodeURI.getURI(), Direction.pullFromVoSpace); pullTransfer.version = VOS.VOSPACE_21; - protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); - pullTransfer.getProtocols().add(protocol); + pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth + Protocol getWithCert = new Protocol(VOS.PROTOCOL_HTTPS_GET); + getWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + pullTransfer.getProtocols().add(getWithCert); + // Do the transfer details = doTransfer(pullTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pullFromVoSpace, Direction.pullFromVoSpace, details.getDirection()); - Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); - endpoint = null; + Assert.assertNotNull(details.getProtocols()); + log.info(pullTransfer.getDirection() + " results: " + details.getProtocols().size()); + URL getURL = null; for (Protocol p : details.getProtocols()) { + String endpoint = p.getEndpoint(); + log.info("GET endpoint: " + endpoint); try { - endpoint = p.getEndpoint(); - log.debug("endpoint: " + endpoint); - new URL(endpoint); + URL u = new URL(endpoint); + if (getURL == null) { + getURL = u; // first + } } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); } } - + Assert.assertNotNull(getURL); + + // get the bytes + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(getURL, bos); + get.run(); + log.info("get: " + get.getResponseCode() + " " + get.getContentType() + " " + get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertNull(get.getThrowable()); + Assert.assertEquals(content.getBytes().length, get.getContentLength()); + Assert.assertEquals(content.getContentType(), get.getContentType()); + byte[] actual = bos.toByteArray(); + Assert.assertArrayEquals(content.getBytes(), actual); + // Delete the node delete(nodeURL, false); diff --git a/cadc-vos-server/build.gradle b/cadc-vos-server/build.gradle index 9bc743af..d2cd2f8d 100644 --- a/cadc-vos-server/build.gradle +++ b/cadc-vos-server/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.0.4' +version = '2.0.5' description = 'OpenCADC VOSpace server' def git_url = 'https://github.com/opencadc/vos' diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java index f8d92803..a42a5bca 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java @@ -105,6 +105,15 @@ import org.opencadc.vospace.transfer.TransferReader; import org.opencadc.vospace.transfer.TransferWriter; +/** + * Servlet to support output of a transfer result from + * /transfers/{jobID}/results/transferDetails. Implementation + * detail/requirement: This servlet MUST be deployed as a + * sibling of the /nodes endpoint and MUST be named /xfer + * in the servlet mapping. + * + * @author pdowler + */ public class TransferDetailsServlet extends HttpServlet { private static final long serialVersionUID = 2022026164700L; diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java index 433179e2..0a7a4c60 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java @@ -165,9 +165,10 @@ protected void updateTransferJob(Node target, URI resolvedPath, ExecutionPhase e Subject s = AuthenticationUtil.getCurrentSubject(); AuthMethod authMethod = AuthenticationUtil.getAuthMethod(s); // HACK: self-lookup - URL serviceURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_XFER_20, authMethod); - String path = "/" + job.getID(); - URL url = new URL(serviceURL.toExternalForm() + path); + URL nodesURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_NODES_20, authMethod); + String xferURL = nodesURL.toExternalForm().replace("/nodes", "/xfer"); // hard coded ugh + String surl = xferURL + "/" + job.getID(); + URL url = new URL(surl); log.debug("transfer URL: " + url); uri = url.toURI(); diff --git a/cavern/VERSION b/cavern/VERSION index c889b56c..3e94367f 100644 --- a/cavern/VERSION +++ b/cavern/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=0.6.1 +VER=0.6.2 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" unset VER diff --git a/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java b/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java index 4121eb35..e3f093aa 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java @@ -67,9 +67,9 @@ package org.opencadc.cavern.files; +import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.net.ResourceNotFoundException; import java.io.FileNotFoundException; -import java.io.OutputStream; import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -94,10 +94,11 @@ public GetAction() { @Override public void doAction() throws Exception { + ByteCountOutputStream out = null; try { Path source = resolveAndSetMetadata(); - OutputStream out = syncOutput.getOutputStream(); + out = new ByteCountOutputStream(syncOutput.getOutputStream()); log.debug("Starting copy of file " + source); Files.copy(source, out); log.debug("Completed copy of file " + source); @@ -112,6 +113,10 @@ public void doAction() throws Exception { } catch (AccessControlException | AccessDeniedException e) { log.debug(e); throw new AccessControlException(e.getMessage()); + } finally { + if (out != null && out.getByteCount() > 0L) { + logInfo.setBytes(out.getByteCount()); + } } } } diff --git a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java index 2d9042d6..cf0bd171 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; +import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.io.ByteLimitExceededException; import ca.nrc.cadc.io.MultiBufferIO; import ca.nrc.cadc.net.ResourceNotFoundException; @@ -97,7 +98,7 @@ import org.opencadc.vospace.VOSURI; /** - * + * @author pdowler * @author majorb * @author jeevesh */ @@ -132,6 +133,7 @@ public void doAction() throws Exception { boolean putStarted = false; boolean successful = false; + long bytesWritten = 0L; try { log.debug("put: start " + nodeURI.getURI().toASCIIString()); @@ -172,6 +174,13 @@ public void doAction() throws Exception { throw new IllegalArgumentException("not a data node"); } node = (DataNode) n; + if (node == null) { + log.warn("target node: " + node + ": creating"); + node = new DataNode(nodeURI.getName()); + node.owner = caller; + node.parent = cn; + nodePersistence.put(node); + } // check write permission if (!preauthGranted) { @@ -195,12 +204,15 @@ public void doAction() throws Exception { //Files.copy(vis, target, StandardCopyOption.REPLACE_EXISTING); // truncate: do not recreate file with wrong owner + StandardOpenOption openOption = StandardOpenOption.TRUNCATE_EXISTING; DigestOutputStream out = new DigestOutputStream( - Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), md); + Files.newOutputStream(target, StandardOpenOption.WRITE, openOption), md); + ByteCountOutputStream bcos = new ByteCountOutputStream(out); MultiBufferIO io = new MultiBufferIO(); - io.copy(in, out); - out.flush(); + io.copy(in, bcos); + bcos.flush(); log.debug("copy: done " + target); + bytesWritten = bcos.getByteCount(); URI expectedMD5 = syncInput.getDigest(); byte[] md5 = md.digest(); @@ -212,8 +224,6 @@ public void doAction() throws Exception { OutputStream trunc = Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); trunc.close(); actualMD5 = null; - //PosixFileAttributes attrs = Files.readAttributes(target, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - //log.warn("after truncate, size: " + attrs.size()); } // re-read node from filesystem @@ -221,7 +231,7 @@ public void doAction() throws Exception { // update Node node.owner = caller; - node.ownerID = identityManager.toPosixPrincipal(caller); + node.ownerID = null; // just in case log.debug(nodeURI + " MD5: " + propValue); NodeProperty csp = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5); @@ -240,6 +250,17 @@ public void doAction() throws Exception { csp.setValue(actualMD5.toASCIIString()); } } + // set/update content-type attr + String contentType = syncInput.getHeader("content-type"); + if (contentType != null) { + NodeProperty ctp = node.getProperty(VOS.PROPERTY_URI_TYPE); + if (ctp == null) { + ctp = new NodeProperty(VOS.PROPERTY_URI_TYPE, contentType); + node.getProperties().add(ctp); + } else { + ctp.setValue(contentType); + } + } nodePersistence.put(node); successful = true; @@ -279,6 +300,9 @@ public void doAction() throws Exception { throw ex; } finally { + if (bytesWritten > 0L) { + logInfo.setBytes(bytesWritten); + } if (successful) { log.debug("put: done " + nodeURI.getURI().toASCIIString()); } else if (putStarted) {