From c33b8fef5425d8c5cac917c4537cc0939aeeb9c3 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 12 Dec 2023 11:52:16 -0800 Subject: [PATCH 1/9] cadc-vos-server: temporary hack bypass /transfers/jobid/results/transferDetails and go directly to /xfer/jobid --- cadc-vos-server/build.gradle | 2 +- .../opencadc/vospace/server/transfers/TransferRunner.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cadc-vos-server/build.gradle b/cadc-vos-server/build.gradle index 9bc743af..d2cd2f8d 100644 --- a/cadc-vos-server/build.gradle +++ b/cadc-vos-server/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.0.4' +version = '2.0.5' description = 'OpenCADC VOSpace server' def git_url = 'https://github.com/opencadc/vos' diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java index a6cddd10..d31d24f4 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java @@ -386,12 +386,15 @@ private void doTransferRedirect(Transfer transfer, List additionalPar // standard redirect StringBuilder sb = new StringBuilder(); - sb.append("/").append(job.getID()).append("/results/transferDetails"); + //sb.append("/").append(job.getID()).append("/results/transferDetails"); + sb.append("/").append(job.getID()); try { AuthMethod authMethod = AuthenticationUtil.getAuthMethod(AuthenticationUtil.getCurrentSubject()); // HACK: self lookup - URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_TRANSFERS_20, authMethod); + //URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_TRANSFERS_20, authMethod); + // but directly to details + URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_XFER_20, authMethod); URL location = new URL(serviceURL.toExternalForm() + sb.toString()); String loc = location.toExternalForm(); log.debug("Location: " + loc); From e75c5ce1d0c387acf83abd4ef5223219d7cab988 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 13:46:35 -0800 Subject: [PATCH 2/9] change lookup of xfer endpoint so we don't need a standardID revert change in transfer details redirect --- .../opencadc/vospace/server/transfers/TransferRunner.java | 8 +++----- .../vospace/server/transfers/VOSpaceTransfer.java | 7 ++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java index d31d24f4..de11a87d 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java @@ -386,15 +386,13 @@ private void doTransferRedirect(Transfer transfer, List additionalPar // standard redirect StringBuilder sb = new StringBuilder(); - //sb.append("/").append(job.getID()).append("/results/transferDetails"); - sb.append("/").append(job.getID()); + sb.append("/").append(job.getID()).append("/results/transferDetails"); try { AuthMethod authMethod = AuthenticationUtil.getAuthMethod(AuthenticationUtil.getCurrentSubject()); // HACK: self lookup - //URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_TRANSFERS_20, authMethod); - // but directly to details - URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_XFER_20, authMethod); + URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_TRANSFERS_20, authMethod); + log.warn(Standards.VOSPACE_TRANSFERS_20 + " -> " + serviceURL); URL location = new URL(serviceURL.toExternalForm() + sb.toString()); String loc = location.toExternalForm(); log.debug("Location: " + loc); diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java index 433179e2..0a7a4c60 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java @@ -165,9 +165,10 @@ protected void updateTransferJob(Node target, URI resolvedPath, ExecutionPhase e Subject s = AuthenticationUtil.getCurrentSubject(); AuthMethod authMethod = AuthenticationUtil.getAuthMethod(s); // HACK: self-lookup - URL serviceURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_XFER_20, authMethod); - String path = "/" + job.getID(); - URL url = new URL(serviceURL.toExternalForm() + path); + URL nodesURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_NODES_20, authMethod); + String xferURL = nodesURL.toExternalForm().replace("/nodes", "/xfer"); // hard coded ugh + String surl = xferURL + "/" + job.getID(); + URL url = new URL(surl); log.debug("transfer URL: " + url); uri = url.toURI(); From 43bbbfe787f2090175c356112b420f9b4a19693e Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 13 Dec 2023 15:32:22 -0800 Subject: [PATCH 3/9] cavern: log number of bytes transferred by files endpoint --- cavern/VERSION | 2 +- .../java/org/opencadc/cavern/files/GetAction.java | 9 +++++++-- .../java/org/opencadc/cavern/files/PutAction.java | 13 +++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cavern/VERSION b/cavern/VERSION index c889b56c..3e94367f 100644 --- a/cavern/VERSION +++ b/cavern/VERSION @@ -1,6 +1,6 @@ ## deployable containers have a semantic and build tag # semantic version tag: major.minor # build version tag: timestamp -VER=0.6.1 +VER=0.6.2 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" unset VER diff --git a/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java b/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java index 4121eb35..e3f093aa 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/GetAction.java @@ -67,9 +67,9 @@ package org.opencadc.cavern.files; +import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.net.ResourceNotFoundException; import java.io.FileNotFoundException; -import java.io.OutputStream; import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -94,10 +94,11 @@ public GetAction() { @Override public void doAction() throws Exception { + ByteCountOutputStream out = null; try { Path source = resolveAndSetMetadata(); - OutputStream out = syncOutput.getOutputStream(); + out = new ByteCountOutputStream(syncOutput.getOutputStream()); log.debug("Starting copy of file " + source); Files.copy(source, out); log.debug("Completed copy of file " + source); @@ -112,6 +113,10 @@ public void doAction() throws Exception { } catch (AccessControlException | AccessDeniedException e) { log.debug(e); throw new AccessControlException(e.getMessage()); + } finally { + if (out != null && out.getByteCount() > 0L) { + logInfo.setBytes(out.getByteCount()); + } } } } diff --git a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java index 2d9042d6..73d7125e 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java @@ -69,6 +69,7 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; +import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.io.ByteLimitExceededException; import ca.nrc.cadc.io.MultiBufferIO; import ca.nrc.cadc.net.ResourceNotFoundException; @@ -132,6 +133,7 @@ public void doAction() throws Exception { boolean putStarted = false; boolean successful = false; + long bytesWritten = 0L; try { log.debug("put: start " + nodeURI.getURI().toASCIIString()); @@ -197,10 +199,12 @@ public void doAction() throws Exception { // truncate: do not recreate file with wrong owner DigestOutputStream out = new DigestOutputStream( Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), md); + ByteCountOutputStream bcos = new ByteCountOutputStream(out); MultiBufferIO io = new MultiBufferIO(); - io.copy(in, out); - out.flush(); + io.copy(in, bcos); + bcos.flush(); log.debug("copy: done " + target); + bytesWritten = bcos.getByteCount(); URI expectedMD5 = syncInput.getDigest(); byte[] md5 = md.digest(); @@ -212,8 +216,6 @@ public void doAction() throws Exception { OutputStream trunc = Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); trunc.close(); actualMD5 = null; - //PosixFileAttributes attrs = Files.readAttributes(target, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - //log.warn("after truncate, size: " + attrs.size()); } // re-read node from filesystem @@ -279,6 +281,9 @@ public void doAction() throws Exception { throw ex; } finally { + if (bytesWritten > 0L) { + logInfo.setBytes(bytesWritten); + } if (successful) { log.debug("put: done " + nodeURI.getURI().toASCIIString()); } else if (putStarted) { From 26f4ca82632751567bcd2bee2b4b90d2a6398b5b Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Thu, 14 Dec 2023 13:00:12 -0800 Subject: [PATCH 4/9] tweak TransferTest, still needs more work --- cadc-test-vos/build.gradle | 2 +- .../conformance/vos/TransferTest.java | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/cadc-test-vos/build.gradle b/cadc-test-vos/build.gradle index e4d65602..1fc36144 100644 --- a/cadc-test-vos/build.gradle +++ b/cadc-test-vos/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.1.2' +version = '2.1.3' description = 'OpenCADC VOSpace test library' def git_url = 'https://github.com/opencadc/vos' diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java index 6e6795cf..8697da98 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java @@ -127,20 +127,20 @@ public void syncPushPullTest() { // Create a push-to-vospace Transfer for the node Transfer pushTransfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace); pushTransfer.version = VOS.VOSPACE_21; - Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); - pushTransfer.getProtocols().add(protocol); + pushTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth + Protocol putWithCert = new Protocol(VOS.PROTOCOL_HTTPS_PUT); + putWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + pushTransfer.getProtocols().add(putWithCert); // Do the transfer Transfer details = doTransfer(pushTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace, Direction.pushToVoSpace, details.getDirection()); Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); - String endpoint = null; for (Protocol p : details.getProtocols()) { + String endpoint = p.getEndpoint(); + log.info("put endpoint: " + endpoint); try { - endpoint = p.getEndpoint(); - log.debug("endpoint: " + endpoint); new URL(endpoint); } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); @@ -150,20 +150,21 @@ public void syncPushPullTest() { // Create a pull-from-vospace Transfer for the node Transfer pullTransfer = new Transfer(nodeURI.getURI(), Direction.pullFromVoSpace); pullTransfer.version = VOS.VOSPACE_21; - protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); - pullTransfer.getProtocols().add(protocol); + pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth + Protocol getWithCert = new Protocol(VOS.PROTOCOL_HTTPS_GET); + getWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + pullTransfer.getProtocols().add(getWithCert); + // Do the transfer details = doTransfer(pullTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pullFromVoSpace, Direction.pullFromVoSpace, details.getDirection()); Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); - endpoint = null; for (Protocol p : details.getProtocols()) { + String endpoint = p.getEndpoint(); + log.info("get endpoint: " + endpoint); try { - endpoint = p.getEndpoint(); - log.debug("endpoint: " + endpoint); new URL(endpoint); } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); From ddff5c4a7da7963e25584561b05cf5cdf736c691 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 10:05:39 -0800 Subject: [PATCH 5/9] add actual byte transfer to TransferTest.syncPushPullTest --- .../conformance/vos/TransferTest.java | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java index 8697da98..f1231185 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java @@ -73,6 +73,7 @@ import ca.nrc.cadc.net.FileContent; import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; import ca.nrc.cadc.reg.Standards; import ca.nrc.cadc.uws.ExecutionPhase; import ca.nrc.cadc.uws.Job; @@ -85,9 +86,11 @@ import java.net.MalformedURLException; import java.net.URI; import java.net.URL; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import javax.security.auth.Subject; import org.apache.log4j.Logger; import org.junit.Assert; @@ -132,20 +135,38 @@ public void syncPushPullTest() { putWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT); pushTransfer.getProtocols().add(putWithCert); - // Do the transfer + // negotiate the transfer Transfer details = doTransfer(pushTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace, Direction.pushToVoSpace, details.getDirection()); - Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); + Assert.assertNotNull(details.getProtocols()); + log.info(pushTransfer.getDirection() + " results: " + details.getProtocols().size()); + URL putURL = null; for (Protocol p : details.getProtocols()) { String endpoint = p.getEndpoint(); - log.info("put endpoint: " + endpoint); + log.info("PUT endpoint: " + endpoint); try { - new URL(endpoint); + + URL u = new URL(endpoint); + if (putURL == null) { + putURL = u; // first + } } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); } } + Assert.assertNotNull(putURL); + + // put the bytes + Random rnd = new Random(); + byte[] data = new byte[1024]; + rnd.nextBytes(data); + FileContent content = new FileContent(data, "application/octet-stream"); + HttpUpload put = new HttpUpload(content, putURL); + put.run(); + log.info("put: " + put.getResponseCode() + " " + put.getThrowable()); + Assert.assertEquals(201, put.getResponseCode()); + Assert.assertNull(put.getThrowable()); // Create a pull-from-vospace Transfer for the node Transfer pullTransfer = new Transfer(nodeURI.getURI(), Direction.pullFromVoSpace); @@ -160,17 +181,35 @@ public void syncPushPullTest() { details = doTransfer(pullTransfer); Assert.assertEquals("expected transfer direction = " + Direction.pullFromVoSpace, Direction.pullFromVoSpace, details.getDirection()); - Assert.assertNotNull("expected > 0 protocols", details.getProtocols()); + Assert.assertNotNull(details.getProtocols()); + log.info(pullTransfer.getDirection() + " results: " + details.getProtocols().size()); + URL getURL = null; for (Protocol p : details.getProtocols()) { String endpoint = p.getEndpoint(); - log.info("get endpoint: " + endpoint); + log.info("GET endpoint: " + endpoint); try { - new URL(endpoint); + URL u = new URL(endpoint); + if (getURL == null) { + getURL = u; // first + } } catch (MalformedURLException e) { Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage())); } } - + Assert.assertNotNull(getURL); + + // get the bytes + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(getURL, bos); + get.run(); + log.info("get: " + get.getResponseCode() + " " + get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertNull(get.getThrowable()); + Assert.assertEquals(content.getBytes().length, get.getContentLength()); + Assert.assertEquals(content.getContentType(), get.getContentType()); + byte[] actual = bos.toByteArray(); + Assert.assertArrayEquals(content.getBytes(), actual); + // Delete the node delete(nodeURL, false); From 97c93117d241eeb0d48b34c9ee71b8b7d3506a8d Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 10:41:57 -0800 Subject: [PATCH 6/9] remove log.warn msg --- .../org/opencadc/vospace/server/transfers/TransferRunner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java index de11a87d..a6cddd10 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferRunner.java @@ -392,7 +392,6 @@ private void doTransferRedirect(Transfer transfer, List additionalPar AuthMethod authMethod = AuthenticationUtil.getAuthMethod(AuthenticationUtil.getCurrentSubject()); // HACK: self lookup URL serviceURL = regClient.getServiceURL(localServiceURI.getURI(), Standards.VOSPACE_TRANSFERS_20, authMethod); - log.warn(Standards.VOSPACE_TRANSFERS_20 + " -> " + serviceURL); URL location = new URL(serviceURL.toExternalForm() + sb.toString()); String loc = location.toExternalForm(); log.debug("Location: " + loc); From dcc9d6e275c1ffc09e8224bfaf1e9efc9fca37d5 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 10:45:45 -0800 Subject: [PATCH 7/9] add javadoc to servlet to specify deployment requirements --- .../vospace/server/transfers/TransferDetailsServlet.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java index f8d92803..a42a5bca 100644 --- a/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java +++ b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java @@ -105,6 +105,15 @@ import org.opencadc.vospace.transfer.TransferReader; import org.opencadc.vospace.transfer.TransferWriter; +/** + * Servlet to support output of a transfer result from + * /transfers/{jobID}/results/transferDetails. Implementation + * detail/requirement: This servlet MUST be deployed as a + * sibling of the /nodes endpoint and MUST be named /xfer + * in the servlet mapping. + * + * @author pdowler + */ public class TransferDetailsServlet extends HttpServlet { private static final long serialVersionUID = 2022026164700L; From d596c7be61734e511c8e83fa975b64c2e434370a Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 11:10:03 -0800 Subject: [PATCH 8/9] cadc-test-vos: allow 200 or 201 after put to negotiated transfer URL cavern: added missing set content-type prop in PutAction --- .../org/opencadc/conformance/vos/TransferTest.java | 7 +++++-- .../java/org/opencadc/cavern/files/PutAction.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java index f1231185..0981ec68 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java @@ -87,6 +87,7 @@ import java.net.URI; import java.net.URL; import java.nio.charset.Charset; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -111,6 +112,8 @@ public class TransferTest extends VOSTest { private static final Logger log = Logger.getLogger(TransferTest.class); + private static final List PUT_OK = Arrays.asList(new Integer[] { 200, 201}); + protected TransferTest(URI resourceID, File testCert) { super(resourceID, testCert); } @@ -165,7 +168,7 @@ public void syncPushPullTest() { HttpUpload put = new HttpUpload(content, putURL); put.run(); log.info("put: " + put.getResponseCode() + " " + put.getThrowable()); - Assert.assertEquals(201, put.getResponseCode()); + Assert.assertTrue(PUT_OK.contains(put.getResponseCode())); Assert.assertNull(put.getThrowable()); // Create a pull-from-vospace Transfer for the node @@ -202,7 +205,7 @@ public void syncPushPullTest() { ByteArrayOutputStream bos = new ByteArrayOutputStream(); HttpGet get = new HttpGet(getURL, bos); get.run(); - log.info("get: " + get.getResponseCode() + " " + get.getThrowable()); + log.info("get: " + get.getResponseCode() + " " + get.getContentType() + " " + get.getThrowable()); Assert.assertEquals(200, get.getResponseCode()); Assert.assertNull(get.getThrowable()); Assert.assertEquals(content.getBytes().length, get.getContentLength()); diff --git a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java index 73d7125e..095c2456 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java @@ -242,6 +242,17 @@ public void doAction() throws Exception { csp.setValue(actualMD5.toASCIIString()); } } + // set/update content-type attr + String contentType = syncInput.getHeader("content-type"); + if (contentType != null) { + NodeProperty ctp = node.getProperty(VOS.PROPERTY_URI_TYPE); + if (ctp == null) { + ctp = new NodeProperty(VOS.PROPERTY_URI_TYPE, contentType); + node.getProperties().add(ctp); + } else { + ctp.setValue(contentType); + } + } nodePersistence.put(node); successful = true; From 3d14726b7af8b486a6973559ba096005b1c027c2 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Mon, 18 Dec 2023 12:11:24 -0800 Subject: [PATCH 9/9] cavern: PutAction (files endpoint) can create a file if it doesn't exist --- .../java/org/opencadc/cavern/files/PutAction.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java index 095c2456..cf0bd171 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/PutAction.java @@ -98,7 +98,7 @@ import org.opencadc.vospace.VOSURI; /** - * + * @author pdowler * @author majorb * @author jeevesh */ @@ -174,6 +174,13 @@ public void doAction() throws Exception { throw new IllegalArgumentException("not a data node"); } node = (DataNode) n; + if (node == null) { + log.warn("target node: " + node + ": creating"); + node = new DataNode(nodeURI.getName()); + node.owner = caller; + node.parent = cn; + nodePersistence.put(node); + } // check write permission if (!preauthGranted) { @@ -197,8 +204,9 @@ public void doAction() throws Exception { //Files.copy(vis, target, StandardCopyOption.REPLACE_EXISTING); // truncate: do not recreate file with wrong owner + StandardOpenOption openOption = StandardOpenOption.TRUNCATE_EXISTING; DigestOutputStream out = new DigestOutputStream( - Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), md); + Files.newOutputStream(target, StandardOpenOption.WRITE, openOption), md); ByteCountOutputStream bcos = new ByteCountOutputStream(out); MultiBufferIO io = new MultiBufferIO(); io.copy(in, bcos); @@ -223,7 +231,7 @@ public void doAction() throws Exception { // update Node node.owner = caller; - node.ownerID = identityManager.toPosixPrincipal(caller); + node.ownerID = null; // just in case log.debug(nodeURI + " MD5: " + propValue); NodeProperty csp = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5);