Skip to content

Commit

Permalink
Merge pull request #217 from andamian/CADC-12561
Browse files Browse the repository at this point in the history 
Added files end point to vault (CADC-12561)
  • Loading branch information
@pdowler
pdowler authored Feb 8, 2024
2 parents 63807e5 + 6393ee1 commit 55a22fc
Show file tree
Hide file tree
Showing 19 changed files with 205 additions and 108 deletions.
4 changes: 2 additions & 2 deletions cadc-test-vos/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '2.1.6'
version = '2.1.7'

description = 'OpenCADC VOSpace test library'
def git_url = 'https://github.com/opencadc/vos'
Expand All @@ -26,7 +26,7 @@ dependencies {
implementation 'org.opencadc:cadc-gms:[1.0.5,)'
implementation 'org.opencadc:cadc-vos:[2.0,)'
implementation 'org.opencadc:cadc-uws:[1.0,2.0)'
implementation 'org.opencadc:cadc-registry:[1.7.4,2.0)'
implementation 'org.opencadc:cadc-registry:[1.7.6,2.0)'

implementation 'junit:junit:[4.0,)'
implementation 'org.apache.commons:commons-compress:[1.12,)'
Expand Down
101 changes: 97 additions & 4 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/FilesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,26 @@
import ca.nrc.cadc.net.HttpPost;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.reg.client.RegistryClient;
import ca.nrc.cadc.util.HexUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.URI;
import java.net.URL;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDateTime;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.NodeProperty;
import org.opencadc.vospace.VOS;
import org.opencadc.vospace.VOSURI;
import org.opencadc.vospace.transfer.Direction;
Expand All @@ -103,8 +113,8 @@ protected FilesTest(URI resourceID, File testCert) {
super(resourceID, testCert);

RegistryClient regClient = new RegistryClient();
this.filesServiceURL = regClient.getServiceURL(resourceID, Standards.VOSPACE_FILES_20, AuthMethod.ANON);
log.info(String.format("%s: %s", Standards.VOSPACE_FILES_20, filesServiceURL));
this.filesServiceURL = regClient.getServiceURL(resourceID, Standards.VOSPACE_FILES, AuthMethod.ANON);
log.info(String.format("%s: %s", Standards.VOSPACE_FILES, filesServiceURL));
}

@Test
Expand All @@ -116,6 +126,9 @@ public void fileTest() {
VOSURI nodeURI = getVOSURI(name);
log.debug("files-data-node URL: " + nodeURL);

// cleanup
delete(nodeURL, false);

// Create a Transfer
Transfer transfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace);
transfer.version = VOS.VOSPACE_21;
Expand Down Expand Up @@ -165,27 +178,107 @@ public void fileTest() {
ByteArrayInputStream is = new ByteArrayInputStream(expected.getBytes());
put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE);

// get the file using files endpoint
URL fileURL = getNodeURL(filesServiceURL, name);

// test HEAD
log.info("HEAD: " + fileURL);

HttpGet headFile = new HttpGet(fileURL, out);
headFile.setHeadOnly(true);
Subject.doAs(authSubject, new RunnableAction(headFile));
log.info("GET response: " + headFile.getResponseCode() + " " + headFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, headFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", headFile.getThrowable());
Assert.assertEquals(expected.getBytes().length, headFile.getContentLength());
String contentDisposition = "inline; filename=\"" + name + "\"";
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
if (headFile.getDigest() != null) {
Assert.assertTrue(computeChecksumURI(expected.getBytes()).equals(headFile.getDigest()));
}
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());
Assert.assertEquals(VOSTest.TEXT_CONTENT_TYPE, headFile.getContentType());

log.info("GET: " + fileURL);
out = new ByteArrayOutputStream();
HttpGet getFile = new HttpGet(fileURL, out);
Subject.doAs(authSubject, new RunnableAction(getFile));
log.info("GET response: " + getFile.getResponseCode() + " " + getFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, getFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", getFile.getThrowable());
Assert.assertEquals(expected.getBytes().length, headFile.getContentLength());
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
if (headFile.getDigest() != null) {
Assert.assertTrue(computeChecksumURI(expected.getBytes()).equals(headFile.getDigest()));
}
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());
Assert.assertEquals(VOSTest.TEXT_CONTENT_TYPE, headFile.getContentType());


String actual = out.toString();
log.debug("file content: " + actual);
Assert.assertEquals("expected file content to match", expected, actual);

// Delete the node
delete(nodeURL);

} catch (Exception e) {
log.error("Unexpected error", e);
Assert.fail("Unexpected error: " + e);
}
}

@Test
public void emptyFileTest() throws Exception {
// Put an empty DataNode
String name = "empty-files-data-node";
URL nodeURL = getNodeURL(nodesServiceURL, name);
VOSURI nodeURI = getVOSURI(name);
log.debug("empty-files-data-node URL: " + nodeURL);
// cleanup
delete(nodeURL, false);

// PUT the node
log.info("put: " + nodeURI + " -> " + nodeURL);
DataNode testNode = new DataNode(name);
put(nodeURL, nodeURI, testNode);

URL fileURL = getNodeURL(filesServiceURL, name);

// test HEAD
log.info("HEAD: " + fileURL);
OutputStream out = new ByteArrayOutputStream();
HttpGet headFile = new HttpGet(fileURL, out);
headFile.setHeadOnly(true);
Subject.doAs(authSubject, new RunnableAction(headFile));
log.info("GET response: " + headFile.getResponseCode() + " " + headFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, headFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", headFile.getThrowable());
Assert.assertEquals(0, headFile.getContentLength());
String contentDisposition = "inline; filename=\"" + name + "\"";
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());

log.info("GET: " + fileURL);
out = new ByteArrayOutputStream();
HttpGet getFile = new HttpGet(fileURL, out);
Subject.doAs(authSubject, new RunnableAction(getFile));
log.info("GET response: " + getFile.getResponseCode() + " " + getFile.getThrowable());
Assert.assertEquals("expected GET response code = 204", 204, getFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", getFile.getThrowable());
Assert.assertEquals(0, headFile.getContentLength());
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());


// Delete the node
delete(nodeURL);

}

protected static URI computeChecksumURI(byte[] input) throws NoSuchAlgorithmException, IOException {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(input);
byte[] digest = md.digest();
return URI.create("md5:" + HexUtil.toHex(digest));
}

}
52 changes: 19 additions & 33 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/NodesTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
Expand Down Expand Up @@ -224,7 +225,7 @@ public void testContainerNode() {
put(subDirURL, subDirURI, subDirNode);
Assert.fail("New node should fail when parent is locked");
} catch (AssertionError ex) {
Assert.assertEquals("expected PUT response code = 200 expected:<200> but was:<403>",
Assert.assertEquals("expected PUT response code in [200, 201]",
ex.getMessage());
}
}
Expand Down Expand Up @@ -909,45 +910,30 @@ public void testDetail() {
}
}

//@Test
public void dataViewTest() {
@Test
public void testDataView() {
try {
// upload test file
String name = "view-data-node";
URL nodeURL = getNodeURL(nodesServiceURL, name);
VOSURI nodeURI = getVOSURI(name);
DataNode node = new DataNode(name);

// cleanup
delete(nodeURL, false);

put(nodeURL, nodeURI, node);

// get the node with view=data
URL getURL = new URL(nodeURL + "?view=data");
ByteArrayOutputStream out = new ByteArrayOutputStream();
HttpGet get = new HttpGet(getURL, out);
get.setFollowRedirects(false);
Subject.doAs(authSubject, new RunnableAction(get));
Assert.assertNotNull(get.getThrowable());
URL redirectURL = get.getRedirectURL();
Assert.assertNotNull(redirectURL);
log.debug("location = " + redirectURL);

String query = redirectURL.getQuery();
String[] params = query.split("&");
Assert.assertEquals(3, params.length);
for (String p : params) {
String[] pv = p.split("=");
String key = pv[0];
String val = NetUtil.decode(pv[1]);
Assert.assertEquals(2, pv.length);
if ("target".equalsIgnoreCase(key)) {
Assert.assertEquals(nodeURI.getURI().toASCIIString(), val);
} else if ("protocol".equalsIgnoreCase(key)) {
Assert.assertEquals(VOS.PROTOCOL_HTTPS_GET.toASCIIString(), val);
} else if ("direction".equalsIgnoreCase(key)) {
Assert.assertEquals(Direction.pullFromVoSpace.getValue(), val);
} else {
Assert.fail(String.format("unexpected transfer parameter: %s = %s", key, val));
}
}
URL viewDataNodeUrl = getNodeURL(nodesServiceURL, name + "?view=data");
HttpGet getRequest = new HttpGet(viewDataNodeUrl, false);
log.debug("GET: " + viewDataNodeUrl);
Subject.doAs(authSubject, new RunnableAction(getRequest));
log.debug("GET responseCode: " + getRequest.getResponseCode() + " " + getRequest.getThrowable());
Assert.assertEquals(HttpURLConnection.HTTP_SEE_OTHER, getRequest.getResponseCode());
Assert.assertNull(getRequest.getThrowable());
String expectedFilesLocation = nodeURL.toString().replace("/nodes/", "/files/");
log.debug("view=data redirects " + expectedFilesLocation + " vs "
+ getRequest.getResponseHeader("Location"));
Assert.assertTrue(expectedFilesLocation.equalsIgnoreCase(getRequest.getResponseHeader("Location")));

if (cleanupOnSuccess) {
delete(nodeURL);
Expand Down
22 changes: 11 additions & 11 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ public void multipleTargetTest() {
expected.add(file2);

doTest(targets, expected, TAR_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);

// cleanup
delete(nodes);
Expand All @@ -327,14 +327,14 @@ public void multipleTargetTest() {
@Test
public void fullTest() {
try {
// /root-folder/
// /root-folder/file-1.txt
// /root-folder/folder-1/
// /root-folder/folder-2/
// /root-folder/folder-2/file-2.txt
// /root-folder/folder-2/file-3.txt
// /root-folder/folder-2/folder-3/
// /root-folder/folder-2/folder-3/link-1.txt
// /root-folder/
// /root-folder/file-1.txt
// /root-folder/folder-1/
// /root-folder/folder-2/
// /root-folder/folder-2/file-2.txt
// /root-folder/folder-2/file-3.txt
// /root-folder/folder-2/folder-3/
// /root-folder/folder-2/folder-3/link-1.txt

// nodes paths
String root = "full-root-folder";
Expand Down Expand Up @@ -387,7 +387,7 @@ public void fullTest() {
expected.add(file3);

doTest(targets, expected, TAR_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);

// cleanup
delete(nodes);
Expand Down Expand Up @@ -567,7 +567,7 @@ private File extractPackage(File packageFile, String contentType)
ArchiveInputStream archiveInputStream = archiveStreamFactory.createArchiveInputStream(
archiveType, inputStream);
ArchiveEntry entry;
while((entry = archiveInputStream.getNextEntry()) != null) {
while ((entry = archiveInputStream.getNextEntry()) != null) {
if (!archiveInputStream.canReadEntryData(entry)) {
log.debug("unable to read archive entry: " + entry.getName());
continue;
Expand Down
9 changes: 6 additions & 3 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/VOSTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.security.auth.Subject;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
Expand Down Expand Up @@ -205,9 +208,9 @@ public void put(URL nodeURL, InputStream is, String contentType) {
log.debug("PUT " + nodeURL);
Subject.doAs(authSubject, new RunnableAction(put));
log.debug("PUT responseCode: " + put.getResponseCode());
// TODO revert response code back to expected 201
Assert.assertEquals("expected PUT response code = 200",
200, put.getResponseCode());
Integer[] validCodes = new Integer[]{200, 201};
Assert.assertTrue("expected PUT response code in [200, 201]",
Arrays.asList(validCodes).contains(put.getResponseCode()));
Assert.assertNull("expected PUT throwable == null", put.getThrowable());
}

Expand Down
18 changes: 17 additions & 1 deletion cadc-vos-server/src/main/java/org/opencadc/vospace/server/actions/GetNodeAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@
import ca.nrc.cadc.net.HttpTransfer;
import ca.nrc.cadc.util.StringUtil;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URI;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.opencadc.vospace.ContainerNode;
Expand Down Expand Up @@ -118,10 +120,24 @@ public GetNodeAction() {
@Override
public void doAction() throws Exception {
VOSURI target = getTargetURI();

String view = syncInput.getParameter(QUERY_PARAM_VIEW);
if ("data".equalsIgnoreCase(view)) {
// makes the assumption that /files endpoint is a sibling of /nodes
URI requestURI = URI.create(syncInput.getRequestURI());
String filesPath = syncInput.getContextPath() + "/files" + target.getPath();
// query params are not passed through
URI filesURI = new URI(requestURI.getScheme(), requestURI.getHost(), filesPath, null);
String location = filesURI.toASCIIString();
log.debug("Redirecting view=data request to " + location);
syncOutput.setHeader("Location", location);
syncOutput.setCode(HttpURLConnection.HTTP_SEE_OTHER);
return;
}

final String detailLevel = syncInput.getParameter(QUERY_PARAM_DETAIL);

// get parent node
// TBD: resolveLinks=true?
PathResolver pathResolver = new PathResolver(nodePersistence, voSpaceAuthorizer);
Node serverNode = pathResolver.getNode(target.getPath(), false);
if (serverNode == null) {
Expand Down
4 changes: 1 addition & 3 deletions cadc-vos-server/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,8 @@ protected URL getURL(VOSURI nodeURI)

// Use a temporary Job with just the remote IP to avoid the original Job in
// a transfer URL which may close the Job.
Job pkgJob = new Job();
pkgJob.setRemoteIP(remoteIP);
TransferGenerator transferGenerator = nodePersistence.getTransferGenerator();
List<Protocol> protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null);
List<Protocol> protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, null);
log.debug("num transfer protocols: " + protocols.size());

// Get the node endpoint from the first protocol
Expand Down
Loading

0 comments on commit 55a22fc

Please sign in to comment.