Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added files end point to vault (CADC-12561) #217

Merged
merged 5 commits into from
Feb 8, 2024
Merged

Added files end point to vault (CADC-12561) #217

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ad3d824
Added files end point to vault
Feb 5, 2024
f2394c0
Re-work after code review
Feb 6, 2024
c5ce211
Re-work after code review
Feb 6, 2024
c594710
Rework after code review
Feb 8, 2024
6393ee1
Small changes
Feb 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cadc-test-vos/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '2.1.6'
version = '2.1.7'

description = 'OpenCADC VOSpace test library'
def git_url = 'https://github.com/opencadc/vos'
Expand All @@ -26,7 +26,7 @@ dependencies {
implementation 'org.opencadc:cadc-gms:[1.0.5,)'
implementation 'org.opencadc:cadc-vos:[2.0,)'
implementation 'org.opencadc:cadc-uws:[1.0,2.0)'
implementation 'org.opencadc:cadc-registry:[1.7.4,2.0)'
implementation 'org.opencadc:cadc-registry:[1.7.6,2.0)'

implementation 'junit:junit:[4.0,)'
implementation 'org.apache.commons:commons-compress:[1.12,)'
Expand Down
252 changes: 173 additions & 79 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/FilesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,16 +76,26 @@
import ca.nrc.cadc.net.HttpPost;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.reg.client.RegistryClient;
import ca.nrc.cadc.util.HexUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.URI;
import java.net.URL;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDateTime;
import javax.security.auth.Subject;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;
import org.opencadc.vospace.DataNode;
import org.opencadc.vospace.NodeProperty;
import org.opencadc.vospace.VOS;
import org.opencadc.vospace.VOSURI;
import org.opencadc.vospace.transfer.Direction;
Expand All @@ -103,89 +113,173 @@ protected FilesTest(URI resourceID, File testCert) {
super(resourceID, testCert);

RegistryClient regClient = new RegistryClient();
this.filesServiceURL = regClient.getServiceURL(resourceID, Standards.VOSPACE_FILES_20, AuthMethod.ANON);
log.info(String.format("%s: %s", Standards.VOSPACE_FILES_20, filesServiceURL));
this.filesServiceURL = regClient.getServiceURL(resourceID, Standards.VOSPACE_FILES, AuthMethod.ANON);
log.info(String.format("%s: %s", Standards.VOSPACE_FILES, filesServiceURL));
}

@Test
public void fileTest() {
try {
// Put a DataNode
String name = "files-data-node";
URL nodeURL = getNodeURL(nodesServiceURL, name);
VOSURI nodeURI = getVOSURI(name);
log.debug("files-data-node URL: " + nodeURL);

// Create a Transfer
Transfer transfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace);
transfer.version = VOS.VOSPACE_21;
transfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth
Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
transfer.getProtocols().add(protocol);

// Get the transfer document
TransferWriter writer = new TransferWriter();
StringWriter sw = new StringWriter();
writer.write(transfer, sw);
log.debug("files-data-node transfer XML: " + sw);

// POST the transfer document
FileContent fileContent = new FileContent(sw.toString().getBytes(), VOSTest.XML_CONTENT_TYPE);
URL transferURL = getNodeURL(synctransServiceURL, name);
log.debug("transfer URL: " + transferURL);
HttpPost post = new HttpPost(synctransServiceURL, fileContent, false);
Subject.doAs(authSubject, new RunnableAction(post));
Assert.assertEquals("expected POST response code = 303", 303, post.getResponseCode());
Assert.assertNull("expected PUT throwable == null", post.getThrowable());

// Get the transfer details
ByteArrayOutputStream out = new ByteArrayOutputStream();
HttpGet get = new HttpGet(post.getRedirectURL(), out);
log.debug("GET: " + post.getRedirectURL());
Subject.doAs(authSubject, new RunnableAction(get));
log.debug("GET responseCode: " + get.getResponseCode());
Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode());
Assert.assertNull("expected GET throwable == null", get.getThrowable());
Assert.assertTrue("expected GET Content-Type starting with " + VOSTest.XML_CONTENT_TYPE,
get.getContentType().startsWith(VOSTest.XML_CONTENT_TYPE));

// Get the endpoint from the transfer details
log.debug("transfer details XML: " + out);
TransferReader transferReader = new TransferReader();
Transfer details = transferReader.read(out.toString(), "vos");
Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace,
Direction.pushToVoSpace, details.getDirection());
Assert.assertTrue("expected >0 endpoints", details.getProtocols().size() > 0);
URL endpoint = new URL(details.getProtocols().get(0).getEndpoint());

// PUT a file to the endpoint
log.info("PUT: " + endpoint);
String expected = "test content for files endpoint\n";
ByteArrayInputStream is = new ByteArrayInputStream(expected.getBytes());
put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE);

// get the file using files endpoint
URL fileURL = getNodeURL(filesServiceURL, name);
log.info("GET: " + fileURL);
out = new ByteArrayOutputStream();
HttpGet getFile = new HttpGet(fileURL, out);
Subject.doAs(authSubject, new RunnableAction(getFile));
log.info("GET response: " + getFile.getResponseCode() + " " + getFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, getFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", getFile.getThrowable());

String actual = out.toString();
log.debug("file content: " + actual);
Assert.assertEquals("expected file content to match", expected, actual);

// Delete the node
delete(nodeURL);

} catch (Exception e) {
log.error("Unexpected error", e);
Assert.fail("Unexpected error: " + e);
public void fileTest() throws Exception {
andamian marked this conversation as resolved.
Show resolved Hide resolved
// Put a DataNode
String name = "files-data-node";
URL nodeURL = getNodeURL(nodesServiceURL, name);
VOSURI nodeURI = getVOSURI(name);
log.debug("files-data-node URL: " + nodeURL);

// cleanup
delete(nodeURL, false);

// Create a Transfer
Transfer transfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace);
transfer.version = VOS.VOSPACE_21;
transfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth
Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
transfer.getProtocols().add(protocol);

// Get the transfer document
TransferWriter writer = new TransferWriter();
StringWriter sw = new StringWriter();
writer.write(transfer, sw);
log.debug("files-data-node transfer XML: " + sw);

// POST the transfer document
FileContent fileContent = new FileContent(sw.toString().getBytes(), VOSTest.XML_CONTENT_TYPE);
URL transferURL = getNodeURL(synctransServiceURL, name);
log.debug("transfer URL: " + transferURL);
HttpPost post = new HttpPost(synctransServiceURL, fileContent, false);
Subject.doAs(authSubject, new RunnableAction(post));
Assert.assertEquals("expected POST response code = 303", 303, post.getResponseCode());
Assert.assertNull("expected PUT throwable == null", post.getThrowable());

// Get the transfer details
ByteArrayOutputStream out = new ByteArrayOutputStream();
HttpGet get = new HttpGet(post.getRedirectURL(), out);
log.debug("GET: " + post.getRedirectURL());
Subject.doAs(authSubject, new RunnableAction(get));
log.debug("GET responseCode: " + get.getResponseCode());
Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode());
Assert.assertNull("expected GET throwable == null", get.getThrowable());
Assert.assertTrue("expected GET Content-Type starting with " + VOSTest.XML_CONTENT_TYPE,
get.getContentType().startsWith(VOSTest.XML_CONTENT_TYPE));

// Get the endpoint from the transfer details
log.debug("transfer details XML: " + out);
TransferReader transferReader = new TransferReader();
Transfer details = transferReader.read(out.toString(), "vos");
Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace,
Direction.pushToVoSpace, details.getDirection());
Assert.assertTrue("expected >0 endpoints", details.getProtocols().size() > 0);
URL endpoint = new URL(details.getProtocols().get(0).getEndpoint());

// PUT a file to the endpoint
log.info("PUT: " + endpoint);
String expected = "test content for files endpoint\n";
ByteArrayInputStream is = new ByteArrayInputStream(expected.getBytes());
put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE);

URL fileURL = getNodeURL(filesServiceURL, name);

// test HEAD
log.info("HEAD: " + fileURL);

HttpGet headFile = new HttpGet(fileURL, out);
headFile.setHeadOnly(true);
Subject.doAs(authSubject, new RunnableAction(headFile));
log.info("GET response: " + headFile.getResponseCode() + " " + headFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, headFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", headFile.getThrowable());
Assert.assertEquals(expected.getBytes().length, headFile.getContentLength());
String contentDisposition = "inline; filename=\"" + name + "\"";
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
if (headFile.getDigest() != null) {
Assert.assertTrue(computeChecksumURI(expected.getBytes()).equals(headFile.getDigest()));
}
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());
Assert.assertEquals(VOSTest.TEXT_CONTENT_TYPE, headFile.getContentType());

log.info("GET: " + fileURL);
out = new ByteArrayOutputStream();
HttpGet getFile = new HttpGet(fileURL, out);
Subject.doAs(authSubject, new RunnableAction(getFile));
log.info("GET response: " + getFile.getResponseCode() + " " + getFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, getFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", getFile.getThrowable());
Assert.assertEquals(expected.getBytes().length, headFile.getContentLength());
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
if (headFile.getDigest() != null) {
Assert.assertTrue(computeChecksumURI(expected.getBytes()).equals(headFile.getDigest()));
}
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());
Assert.assertEquals(VOSTest.TEXT_CONTENT_TYPE, headFile.getContentType());


String actual = out.toString();
log.debug("file content: " + actual);
Assert.assertEquals("expected file content to match", expected, actual);

// Delete the node
delete(nodeURL);
}

@Test
public void emptyFileTest() throws Exception {
// Put an empty DataNode
String name = "empty-files-data-node";
URL nodeURL = getNodeURL(nodesServiceURL, name);
VOSURI nodeURI = getVOSURI(name);
log.debug("empty-files-data-node URL: " + nodeURL);
// cleanup
delete(nodeURL, false);

// PUT the node
log.info("put: " + nodeURI + " -> " + nodeURL);
DataNode testNode = new DataNode(name);
put(nodeURL, nodeURI, testNode);

URL fileURL = getNodeURL(filesServiceURL, name);

// test HEAD
log.info("HEAD: " + fileURL);
OutputStream out = new ByteArrayOutputStream();
HttpGet headFile = new HttpGet(fileURL, out);
headFile.setHeadOnly(true);
Subject.doAs(authSubject, new RunnableAction(headFile));
log.info("GET response: " + headFile.getResponseCode() + " " + headFile.getThrowable());
Assert.assertEquals("expected GET response code = 200", 200, headFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", headFile.getThrowable());
Assert.assertEquals(0, headFile.getContentLength());
String contentDisposition = "inline; filename=\"" + name + "\"";
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());

log.info("GET: " + fileURL);
out = new ByteArrayOutputStream();
HttpGet getFile = new HttpGet(fileURL, out);
Subject.doAs(authSubject, new RunnableAction(getFile));
log.info("GET response: " + getFile.getResponseCode() + " " + getFile.getThrowable());
Assert.assertEquals("expected GET response code = 204", 204, getFile.getResponseCode());
Assert.assertNull("expected GET throwable == null", getFile.getThrowable());
Assert.assertEquals(0, headFile.getContentLength());
Assert.assertTrue(contentDisposition.equals(headFile.getResponseHeader("Content-Disposition")));
Assert.assertTrue(System.currentTimeMillis() > headFile.getLastModified().getTime());


// Delete the node
delete(nodeURL);

}

protected static URI computeChecksumURI(byte[] input) throws NoSuchAlgorithmException, IOException {
MessageDigest md = MessageDigest.getInstance("MD5");
InputStream in = new ByteArrayInputStream(input);
andamian marked this conversation as resolved.
Show resolved Hide resolved
DigestInputStream dis = new DigestInputStream(in, md);
int bytesRead = dis.read();
byte[] buf = new byte[512];
while (bytesRead > 0) {
bytesRead = dis.read(buf);
}
byte[] digest = md.digest();
return URI.create("md5:" + HexUtil.toHex(digest));
}

}
16 changes: 15 additions & 1 deletion cadc-test-vos/src/main/java/org/opencadc/conformance/vos/NodesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.security.PrivilegedExceptionAction;
Expand Down Expand Up @@ -224,7 +225,7 @@ public void testContainerNode() {
put(subDirURL, subDirURI, subDirNode);
Assert.fail("New node should fail when parent is locked");
} catch (AssertionError ex) {
Assert.assertEquals("expected PUT response code = 200 expected:<200> but was:<403>",
Assert.assertEquals("expected PUT response code in [200, 201]",
ex.getMessage());
}
}
Expand Down Expand Up @@ -307,6 +308,19 @@ public void testDataNode() {
Assert.fail("immutable prop test: " + len);
}

// test view=data
andamian marked this conversation as resolved.
Show resolved Hide resolved
URL viewDataNodeUrl = getNodeURL(nodesServiceURL, name + "?view=data");
HttpGet getRequest = new HttpGet(viewDataNodeUrl, false);
log.debug("GET: " + viewDataNodeUrl);
Subject.doAs(authSubject, new RunnableAction(getRequest));
log.debug("GET responseCode: " + getRequest.getResponseCode() + " " + getRequest.getThrowable());
Assert.assertEquals(HttpURLConnection.HTTP_SEE_OTHER, getRequest.getResponseCode());
Assert.assertNull(getRequest.getThrowable());
String expectedFilesLocation = nodeURL.toString().replace("/nodes/", "/files/");
log.debug("view=data redirects " + expectedFilesLocation + " vs "
+ getRequest.getResponseHeader("Location"));
Assert.assertTrue(expectedFilesLocation.equalsIgnoreCase(getRequest.getResponseHeader("Location")));

// fail to update with a sketch property URI
URI illegal = new URI(VOS.VOSPACE_URI_NAMESPACE + "core#make-stuff-up");
NodeProperty illegalProp = new NodeProperty(illegal, "that should not work");
Expand Down
22 changes: 11 additions & 11 deletions cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,7 @@ public void multipleTargetTest() {
expected.add(file2);

doTest(targets, expected, TAR_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);

// cleanup
delete(nodes);
Expand All @@ -327,14 +327,14 @@ public void multipleTargetTest() {
@Test
public void fullTest() {
try {
// /root-folder/
// /root-folder/file-1.txt
// /root-folder/folder-1/
// /root-folder/folder-2/
// /root-folder/folder-2/file-2.txt
// /root-folder/folder-2/file-3.txt
// /root-folder/folder-2/folder-3/
// /root-folder/folder-2/folder-3/link-1.txt
// /root-folder/
// /root-folder/file-1.txt
// /root-folder/folder-1/
// /root-folder/folder-2/
// /root-folder/folder-2/file-2.txt
// /root-folder/folder-2/file-3.txt
// /root-folder/folder-2/folder-3/
// /root-folder/folder-2/folder-3/link-1.txt

// nodes paths
String root = "full-root-folder";
Expand Down Expand Up @@ -387,7 +387,7 @@ public void fullTest() {
expected.add(file3);

doTest(targets, expected, TAR_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);
// doTest(targets, expected, ZIP_CONTENT_TYPE);

// cleanup
delete(nodes);
Expand Down Expand Up @@ -567,7 +567,7 @@ private File extractPackage(File packageFile, String contentType)
ArchiveInputStream archiveInputStream = archiveStreamFactory.createArchiveInputStream(
archiveType, inputStream);
ArchiveEntry entry;
while((entry = archiveInputStream.getNextEntry()) != null) {
while ((entry = archiveInputStream.getNextEntry()) != null) {
if (!archiveInputStream.canReadEntryData(entry)) {
log.debug("unable to read archive entry: " + entry.getName());
continue;
Expand Down
Loading
Loading