Skip to content

Commit

Permalink
cavern: add QuotaPlugin API
Browse files Browse the repository at this point in the history
added a no-op impl to use as default
augmented ExtendedFileAttributes API to include other attr namespaces
but not yet implemented
  • Loading branch information
pdowler committed May 8, 2024
1 parent b84f2d3 commit ac00d60
Show file tree
Hide file tree
Showing 8 changed files with 458 additions and 94 deletions.
2 changes: 1 addition & 1 deletion cadc-util-fs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.1.0'
version = '1.1.1'

description = 'OpenCADC file system utility library'
def git_url = 'https://github.com/opencadc/vos'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,63 +90,115 @@ public class ExtendedFileAttributes {
private static final Logger log = Logger.getLogger(ExtendedFileAttributes.class);

/**
* Set the attribute for the current path. If attributeValue is null, the attribute will be deleted.
* Set the attribute for the specified path. This method defaults to the "user" namespace.
*
* @param path The path where the attribute will be set. Must not be null
* @param attrName The name of the attribute to be set. Must not be null
* @param attrValue The value of the attribute to be set. A null value means the attribute is to be deleted
*
* @throws IOException if setting attribute failed
* @see #setFileAttribute(java.nio.file.Path, java.lang.String, java.lang.String, java.lang.String)
*/
public static void setFileAttribute(Path path, String attrName, String attrValue) throws IOException {
setFileAttribute(path, attrName, attrValue, null);
}

/**
* Set an attribute on a path in the specified namespace.
* If attributeValue is null, the attribute will be deleted.
* If the specified namespace is null, it defaults to the "user" namespace.
* The specified attribyteKey should not include the namespace; the namespace will be set by this method.
*
* @param path The path where the attribute will be set. Must not be null
* @param attrName The name of the attribute to be set. Must not be null
* @param attrValue The value of the attribute to be set. A null value means the attribute is to be deleted
* @param namespace attribute namespace, null defaults to the user namespace
*
* @param path The path where the attribute will be set. Must not be null.
* @param attributeKey The name of the attribute to be set. Must not be null.
* @param attributeValue The value of the attribute to be set. A null value means the attribute is to be deleted.
* @throws IOException if setting attribute failed
*/
public static void setFileAttribute(Path path, String attributeKey, String attributeValue) throws IOException {
if (path == null || attributeKey == null) {
public static void setFileAttribute(Path path, String attrName, String attrValue, String namespace)
throws IOException {
if (path == null || attrName == null) {
throw new IllegalArgumentException("path or attributeKey cannot be null");
}

log.debug("setFileAttribute: " + path);
UserDefinedFileAttributeView udv = Files.getFileAttributeView(path,
UserDefinedFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);
if (attributeValue != null) {
attributeValue = attributeValue.trim();
log.debug("attribute: " + attributeKey + " = " + attributeValue);
ByteBuffer buf = ByteBuffer.wrap(attributeValue.getBytes(Charset.forName("UTF-8")));
udv.write(attributeKey, buf);
} else {
try {
log.debug("attribute: " + attributeKey + " (delete)");
udv.delete(attributeKey);
} catch (FileSystemException ex) {
log.debug("assume no such attr: " + ex);

if (namespace == null || "user".equals(namespace)) {
log.debug("setFileAttribute: " + path);
UserDefinedFileAttributeView udv = Files.getFileAttributeView(path,
UserDefinedFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);
if (attrValue != null) {
attrValue = attrValue.trim();
log.debug("attribute: " + attrName + " = " + attrValue);
ByteBuffer buf = ByteBuffer.wrap(attrValue.getBytes(Charset.forName("UTF-8")));
udv.write(attrName, buf);
} else {
try {
log.debug("attribute: " + attrName + " (delete)");
udv.delete(attrName);
} catch (FileSystemException ex) {
log.debug("assume no such attr: " + ex);
}
}
return;
}

// TODO: support non-user space attrs by execing setfattr
String key = namespace + "." + attrName;
// setfattr -n $key -v $attrValue $path
throw new UnsupportedOperationException("attribute namespace '" + namespace + "' not supported");
}

/**
* Get the value of specified attribute for the current path.
* Get the value of specified attribute for the current path. This method defaults to the "user" namespace.
*
* @param path The path where the attribute resides in. Must not be null
* @param attrName The name of the attribute to get. Must not be null
*
* @param path The path where the attribute resides in. Must not be null.
* @param attributeName The name of the attribute to get. Must not be null.
* @return attribute value or null if not set
* @throws IOException if reading attribute failed
* @see #getFileAttribute(java.nio.file.Path, java.lang.String, java.lang.String)
*/
public static String getFileAttribute(Path path, String attrName) throws IOException {
return getFileAttribute(path, attrName, null);
}

/**
* Get the value of specified attribute for the current path from the specified attribute namespace.
*
* @param path The path where the attribute resides in. Must not be null
* @param attrName The name of the attribute to get. Must not be null
* @param namespace attribute namespace, null defaults to the user namespace
*
* @return attribute value or null if not set
*
* @throws IOException if reading attribute failed
*/
public static String getFileAttribute(Path path, String attributeName) throws IOException {
if (path == null || attributeName == null) {
public static String getFileAttribute(Path path, String attrName, String namespace) throws IOException {
if (path == null || attrName == null) {
throw new IllegalArgumentException("path or attributeName cannot be null");
}

try {
UserDefinedFileAttributeView udv = Files.getFileAttributeView(path,
UserDefinedFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);
if (namespace == null || "user".equals(namespace)) {
try {
UserDefinedFileAttributeView udv = Files.getFileAttributeView(path,
UserDefinedFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);

int sz = udv.size(attributeName);
ByteBuffer buf = ByteBuffer.allocate(2 * sz);
udv.read(attributeName, buf);
return new String(buf.array(), Charset.forName("UTF-8")).trim();
} catch (FileSystemException ex) {
log.debug("assume no such attr: " + ex);
return null;
int sz = udv.size(attrName);
ByteBuffer buf = ByteBuffer.allocate(2 * sz);
udv.read(attrName, buf);
return new String(buf.array(), Charset.forName("UTF-8")).trim();
} catch (FileSystemException ex) {
log.debug("assume no such attr: " + ex);
return null;
}
}

// TODO: support non-user space attrs by execing setfattr
String key = namespace + "." + attrName;
// getfattr -n $key $path
throw new UnsupportedOperationException("attribute namespace '" + namespace + "' not supported");
}

/**
* Get all user-defined attributes.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/*
************************************************************************
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2024. (c) 2024.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
* All rights reserved Tous droits réservés
*
* NRC disclaims any warranties, Le CNRC dénie toute garantie
* expressed, implied, or énoncée, implicite ou légale,
* statutory, of any kind with de quelque nature que ce
* respect to the software, soit, concernant le logiciel,
* including without limitation y compris sans restriction
* any warranty of merchantability toute garantie de valeur
* or fitness for a particular marchande ou de pertinence
* purpose. NRC shall not be pour un usage particulier.
* liable in any event for any Le CNRC ne pourra en aucun cas
* damages, whether direct or être tenu responsable de tout
* indirect, special or general, dommage, direct ou indirect,
* consequential or incidental, particulier ou général,
* arising from the use of the accessoire ou fortuit, résultant
* software. Neither the name de l'utilisation du logiciel. Ni
* of the National Research le nom du Conseil National de
* Council of Canada nor the Recherches du Canada ni les noms
* names of its contributors may de ses participants ne peuvent
* be used to endorse or promote être utilisés pour approuver ou
* products derived from this promouvoir les produits dérivés
* software without specific prior de ce logiciel sans autorisation
* written permission. préalable et particulière
* par écrit.
*
* This file is part of the Ce fichier fait partie du projet
* OpenCADC project. OpenCADC.
*
* OpenCADC is free software: OpenCADC est un logiciel libre ;
* you can redistribute it and/or vous pouvez le redistribuer ou le
* modify it under the terms of modifier suivant les termes de
* the GNU Affero General Public la “GNU Affero General Public
* License as published by the License” telle que publiée
* Free Software Foundation, par la Free Software Foundation
* either version 3 of the : soit la version 3 de cette
* License, or (at your option) licence, soit (à votre gré)
* any later version. toute version ultérieure.
*
* OpenCADC is distributed in the OpenCADC est distribué
* hope that it will be useful, dans l’espoir qu’il vous
* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE
* without even the implied GARANTIE : sans même la garantie
* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ
* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF
* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence
* General Public License for Générale Publique GNU Affero
* more details. pour plus de détails.
*
* You should have received Vous devriez avoir reçu une
* a copy of the GNU Affero copie de la Licence Générale
* General Public License along Publique GNU Affero avec
* with OpenCADC. If not, see OpenCADC ; si ce n’est
* <http://www.gnu.org/licenses/>. pas le cas, consultez :
* <http://www.gnu.org/licenses/>.
*
************************************************************************
*/

package org.opencadc.util.fs;

import ca.nrc.cadc.util.Log4jInit;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;

/**
*
* @author pdowler
*/
public class ExtendedFileAttributesTest {
private static final Logger log = Logger.getLogger(ExtendedFileAttributesTest.class);

static {
Log4jInit.setLevel("org.opencadc.util.fs", Level.DEBUG);
Log4jInit.setLevel("ca.nrc.cadc.exec", Level.DEBUG);
}

static final String ROOT = "build/tmp/attr-tests";

static {
try {
Path root = FileSystems.getDefault().getPath(ROOT);
if (!Files.exists(root)) {
Files.createDirectory(root);
}
} catch (IOException ex) {
throw new RuntimeException("TEST SETUP: failed to create test dir: " + ROOT, ex);
}
}

public ExtendedFileAttributesTest() {
}

@Test
public void testSimpleAttrs() throws Exception {
Path root = FileSystems.getDefault().getPath(ROOT);
Set<PosixFilePermission> perms = new HashSet<>();
perms.add(PosixFilePermission.OWNER_READ);
perms.add(PosixFilePermission.OWNER_WRITE);
perms.add(PosixFilePermission.GROUP_READ);
perms.add(PosixFilePermission.GROUP_WRITE);
perms.add(PosixFilePermission.OWNER_EXECUTE);
perms.add(PosixFilePermission.GROUP_EXECUTE);
Path target = root.resolve("simple-test");
if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) {
Files.delete(target);
}
Files.createDirectory(target, PosixFilePermissions.asFileAttribute(perms));

Map<String,String> attrs = ExtendedFileAttributes.getAttributes(target);
Assert.assertTrue(attrs.isEmpty());

log.info("** simple **");
String key = "foo";
String val = "foo-val";
ExtendedFileAttributes.setFileAttribute(target, key, val);
attrs = ExtendedFileAttributes.getAttributes(target);
for (Map.Entry<String,String> me : attrs.entrySet()) {
log.info(me.getKey() + " = " + me.getValue());
}
Assert.assertEquals(1, attrs.size());
Assert.assertEquals(val, attrs.get(key));
}
}
2 changes: 1 addition & 1 deletion cavern/VERSION
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor
# build version tag: timestamp
VER=0.7.5
VER=0.7.6
TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")"
unset VER
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ public class FileSystemNodePersistence implements NodePersistence {
private final PosixIdentityManager identityManager;
private final PosixMapperClient posixMapper;
private final GroupCache groupCache;
private final QuotaPlugin quotaImpl = new NoQuotaPlugin(); // TODO: configurable

private final ContainerNode root;
private final Set<ContainerNode> allocationParents = new TreeSet<>();
Expand Down Expand Up @@ -285,7 +286,7 @@ public TransferGenerator getTransferGenerator() {
public Node get(ContainerNode parent, String name) throws TransientException {
identityManager.addToCache(AuthenticationUtil.getCurrentSubject());
try {
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache);
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache, quotaImpl);
Node ret = nut.get(parent, name);
if (ret == null) {
return null;
Expand Down Expand Up @@ -313,7 +314,7 @@ public ResourceIterator<Node> iterator(ContainerNode parent, Integer limit, Stri
}

try {
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache);
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache, quotaImpl);
// this is a complicated way to get the Path
LocalServiceURI loc = new LocalServiceURI(getResourceID());
VOSURI vu = loc.getURI(parent);
Expand Down Expand Up @@ -431,7 +432,7 @@ public Node put(Node node) throws NodeNotSupportedException, TransientException
}
}

NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache);
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache, quotaImpl);

if (node instanceof LinkNode) {
LinkNode ln = (LinkNode) node;
Expand Down Expand Up @@ -463,7 +464,7 @@ public void move(Node node, ContainerNode dest, String newName) {
throw new IllegalArgumentException("args must both be peristent nodes before move");
}

NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache);
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache, quotaImpl);
Subject caller = AuthenticationUtil.getCurrentSubject();
PosixPrincipal owner = identityManager.addToCache(caller);

Expand All @@ -480,7 +481,7 @@ public void move(Node node, ContainerNode dest, String newName) {

@Override
public void delete(Node node) throws TransientException {
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache);
NodeUtil nut = new NodeUtil(rootPath, rootURI, groupCache, quotaImpl);
Subject caller = AuthenticationUtil.getCurrentSubject();
identityManager.addToCache(caller);
try {
Expand Down
Loading

0 comments on commit ac00d60

Please sign in to comment.