Skip to content

Commit

Permalink
Merge pull request #209 from pdowler/master
Browse files Browse the repository at this point in the history 
cadc-vos-server minoc tweaking
  • Loading branch information
@pdowler
pdowler authored Dec 19, 2023
2 parents 2df0f93 + 3d14726 commit 5ecca60
Show file tree
Hide file tree
Showing 8 changed files with 115 additions and 33 deletions.
2 changes: 1 addition & 1 deletion cadc-test-vos/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '2.1.2'
version = '2.1.3'

description = 'OpenCADC VOSpace test library'
def git_url = 'https://github.com/opencadc/vos'
Expand Down
79 changes: 61 additions & 18 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
import ca.nrc.cadc.net.FileContent;
import ca.nrc.cadc.net.HttpGet;
import ca.nrc.cadc.net.HttpPost;
import ca.nrc.cadc.net.HttpUpload;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.uws.ExecutionPhase;
import ca.nrc.cadc.uws.Job;
Expand All @@ -85,9 +86,12 @@
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.junit.Assert;
Expand All @@ -108,6 +112,8 @@
public class TransferTest extends VOSTest {
private static final Logger log = Logger.getLogger(TransferTest.class);

private static final List<Integer> PUT_OK = Arrays.asList(new Integer[] { 200, 201});

protected TransferTest(URI resourceID, File testCert) {
super(resourceID, testCert);
}
Expand All @@ -127,49 +133,86 @@ public void syncPushPullTest() {
// Create a push-to-vospace Transfer for the node
Transfer pushTransfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace);
pushTransfer.version = VOS.VOSPACE_21;
Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
pushTransfer.getProtocols().add(protocol);
pushTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth
Protocol putWithCert = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
putWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
pushTransfer.getProtocols().add(putWithCert);

// Do the transfer
// negotiate the transfer
Transfer details = doTransfer(pushTransfer);
Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace,
Direction.pushToVoSpace, details.getDirection());
Assert.assertNotNull("expected > 0 protocols", details.getProtocols());
String endpoint = null;
Assert.assertNotNull(details.getProtocols());
log.info(pushTransfer.getDirection() + " results: " + details.getProtocols().size());
URL putURL = null;
for (Protocol p : details.getProtocols()) {
String endpoint = p.getEndpoint();
log.info("PUT endpoint: " + endpoint);
try {
endpoint = p.getEndpoint();
log.debug("endpoint: " + endpoint);
new URL(endpoint);

URL u = new URL(endpoint);
if (putURL == null) {
putURL = u; // first
}
} catch (MalformedURLException e) {
Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage()));
}
}
Assert.assertNotNull(putURL);

// put the bytes
Random rnd = new Random();
byte[] data = new byte[1024];
rnd.nextBytes(data);
FileContent content = new FileContent(data, "application/octet-stream");
HttpUpload put = new HttpUpload(content, putURL);
put.run();
log.info("put: " + put.getResponseCode() + " " + put.getThrowable());
Assert.assertTrue(PUT_OK.contains(put.getResponseCode()));
Assert.assertNull(put.getThrowable());

// Create a pull-from-vospace Transfer for the node
Transfer pullTransfer = new Transfer(nodeURI.getURI(), Direction.pullFromVoSpace);
pullTransfer.version = VOS.VOSPACE_21;
protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
pullTransfer.getProtocols().add(protocol);
pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth
Protocol getWithCert = new Protocol(VOS.PROTOCOL_HTTPS_GET);
getWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
pullTransfer.getProtocols().add(getWithCert);


// Do the transfer
details = doTransfer(pullTransfer);
Assert.assertEquals("expected transfer direction = " + Direction.pullFromVoSpace,
Direction.pullFromVoSpace, details.getDirection());
Assert.assertNotNull("expected > 0 protocols", details.getProtocols());
endpoint = null;
Assert.assertNotNull(details.getProtocols());
log.info(pullTransfer.getDirection() + " results: " + details.getProtocols().size());
URL getURL = null;
for (Protocol p : details.getProtocols()) {
String endpoint = p.getEndpoint();
log.info("GET endpoint: " + endpoint);
try {
endpoint = p.getEndpoint();
log.debug("endpoint: " + endpoint);
new URL(endpoint);
URL u = new URL(endpoint);
if (getURL == null) {
getURL = u; // first
}
} catch (MalformedURLException e) {
Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage()));
}
}

Assert.assertNotNull(getURL);

// get the bytes
ByteArrayOutputStream bos = new ByteArrayOutputStream();
HttpGet get = new HttpGet(getURL, bos);
get.run();
log.info("get: " + get.getResponseCode() + " " + get.getContentType() + " " + get.getThrowable());
Assert.assertEquals(200, get.getResponseCode());
Assert.assertNull(get.getThrowable());
Assert.assertEquals(content.getBytes().length, get.getContentLength());
Assert.assertEquals(content.getContentType(), get.getContentType());
byte[] actual = bos.toByteArray();
Assert.assertArrayEquals(content.getBytes(), actual);

// Delete the node
delete(nodeURL, false);

Expand Down
2 changes: 1 addition & 1 deletion cadc-vos-server/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '2.0.4'
version = '2.0.5'

description = 'OpenCADC VOSpace server'
def git_url = 'https://github.com/opencadc/vos'
Expand Down
9 changes: 9 additions & 0 deletions ...os-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,15 @@
import org.opencadc.vospace.transfer.TransferReader;
import org.opencadc.vospace.transfer.TransferWriter;

/**
* Servlet to support output of a transfer result from
* /transfers/{jobID}/results/transferDetails. Implementation
* detail/requirement: This servlet MUST be deployed as a
* sibling of the /nodes endpoint and MUST be named /xfer
* in the servlet mapping.
*
* @author pdowler
*/
public class TransferDetailsServlet extends HttpServlet {

private static final long serialVersionUID = 2022026164700L;
Expand Down
7 changes: 4 additions & 3 deletions cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,10 @@ protected void updateTransferJob(Node target, URI resolvedPath, ExecutionPhase e
Subject s = AuthenticationUtil.getCurrentSubject();
AuthMethod authMethod = AuthenticationUtil.getAuthMethod(s);
// HACK: self-lookup
URL serviceURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_XFER_20, authMethod);
String path = "/" + job.getID();
URL url = new URL(serviceURL.toExternalForm() + path);
URL nodesURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_NODES_20, authMethod);
String xferURL = nodesURL.toExternalForm().replace("/nodes", "/xfer"); // hard coded ugh
String surl = xferURL + "/" + job.getID();
URL url = new URL(surl);
log.debug("transfer URL: " + url);

uri = url.toURI();
Expand Down
2 changes: 1 addition & 1 deletion cavern/VERSION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor
# build version tag: timestamp
VER=0.6.1
VER=0.6.2
TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")"
unset VER
9 changes: 7 additions & 2 deletions cavern/src/main/java/org/opencadc/cavern/files/GetAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,9 @@

package org.opencadc.cavern.files;

import ca.nrc.cadc.io.ByteCountOutputStream;
import ca.nrc.cadc.net.ResourceNotFoundException;
import java.io.FileNotFoundException;
import java.io.OutputStream;
import java.nio.file.AccessDeniedException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
Expand All @@ -94,10 +94,11 @@ public GetAction() {

@Override
public void doAction() throws Exception {
ByteCountOutputStream out = null;
try {
Path source = resolveAndSetMetadata();

OutputStream out = syncOutput.getOutputStream();
out = new ByteCountOutputStream(syncOutput.getOutputStream());
log.debug("Starting copy of file " + source);
Files.copy(source, out);
log.debug("Completed copy of file " + source);
Expand All @@ -112,6 +113,10 @@ public void doAction() throws Exception {
} catch (AccessControlException | AccessDeniedException e) {
log.debug(e);
throw new AccessControlException(e.getMessage());
} finally {
if (out != null && out.getByteCount() > 0L) {
logInfo.setBytes(out.getByteCount());
}
}
}
}
38 changes: 31 additions & 7 deletions cavern/src/main/java/org/opencadc/cavern/files/PutAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@

import ca.nrc.cadc.auth.AuthenticationUtil;
import ca.nrc.cadc.auth.HttpPrincipal;
import ca.nrc.cadc.io.ByteCountOutputStream;
import ca.nrc.cadc.io.ByteLimitExceededException;
import ca.nrc.cadc.io.MultiBufferIO;
import ca.nrc.cadc.net.ResourceNotFoundException;
Expand Down Expand Up @@ -97,7 +98,7 @@
import org.opencadc.vospace.VOSURI;

/**
*
* @author pdowler
* @author majorb
* @author jeevesh
*/
Expand Down Expand Up @@ -132,6 +133,7 @@ public void doAction() throws Exception {
boolean putStarted = false;
boolean successful = false;

long bytesWritten = 0L;
try {
log.debug("put: start " + nodeURI.getURI().toASCIIString());

Expand Down Expand Up @@ -172,6 +174,13 @@ public void doAction() throws Exception {
throw new IllegalArgumentException("not a data node");
}
node = (DataNode) n;
if (node == null) {
log.warn("target node: " + node + ": creating");
node = new DataNode(nodeURI.getName());
node.owner = caller;
node.parent = cn;
nodePersistence.put(node);
}

// check write permission
if (!preauthGranted) {
Expand All @@ -195,12 +204,15 @@ public void doAction() throws Exception {
//Files.copy(vis, target, StandardCopyOption.REPLACE_EXISTING);

// truncate: do not recreate file with wrong owner
StandardOpenOption openOption = StandardOpenOption.TRUNCATE_EXISTING;
DigestOutputStream out = new DigestOutputStream(
Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), md);
Files.newOutputStream(target, StandardOpenOption.WRITE, openOption), md);
ByteCountOutputStream bcos = new ByteCountOutputStream(out);
MultiBufferIO io = new MultiBufferIO();
io.copy(in, out);
out.flush();
io.copy(in, bcos);
bcos.flush();
log.debug("copy: done " + target);
bytesWritten = bcos.getByteCount();

URI expectedMD5 = syncInput.getDigest();
byte[] md5 = md.digest();
Expand All @@ -212,16 +224,14 @@ public void doAction() throws Exception {
OutputStream trunc = Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
trunc.close();
actualMD5 = null;
//PosixFileAttributes attrs = Files.readAttributes(target, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
//log.warn("after truncate, size: " + attrs.size());
}

// re-read node from filesystem
node = (DataNode) nodePersistence.get(cn, nodeURI.getName());

// update Node
node.owner = caller;
node.ownerID = identityManager.toPosixPrincipal(caller);
node.ownerID = null; // just in case

log.debug(nodeURI + " MD5: " + propValue);
NodeProperty csp = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5);
Expand All @@ -240,6 +250,17 @@ public void doAction() throws Exception {
csp.setValue(actualMD5.toASCIIString());
}
}
// set/update content-type attr
String contentType = syncInput.getHeader("content-type");
if (contentType != null) {
NodeProperty ctp = node.getProperty(VOS.PROPERTY_URI_TYPE);
if (ctp == null) {
ctp = new NodeProperty(VOS.PROPERTY_URI_TYPE, contentType);
node.getProperties().add(ctp);
} else {
ctp.setValue(contentType);
}
}

nodePersistence.put(node);
successful = true;
Expand Down Expand Up @@ -279,6 +300,9 @@ public void doAction() throws Exception {

throw ex;
} finally {
if (bytesWritten > 0L) {
logInfo.setBytes(bytesWritten);
}
if (successful) {
log.debug("put: done " + nodeURI.getURI().toASCIIString());
} else if (putStarted) {
Expand Down

0 comments on commit 5ecca60

Please sign in to comment.