Viewfs support (#37)

* Add support for creating a client based on the defaultFS setting

* Add viewfs support

README.md

@@ -16,6 +16,7 @@ Here is a list of currently supported and unsupported but possible future featur
 ### HDFS Features
 - [x] Name Services
 - [ ] Observer reads (state ID tracking is supported, but needs improvements on tracking Observer/Active NameNode)
+- [x] ViewFS
 - [ ] Federated router
 - [x] Erasure coded reads 
     - RS schema only, no support for RS-Legacy or XOR
@@ -42,8 +43,11 @@ Here is a list of currently supported and unsupported but possible future featur
 
 ## Supported HDFS Settings
 The client will attempt to read Hadoop configs `core-site.xml` and `hdfs-site.xml` in the directories `$HADOOP_CONF_DIR` or if that doesn't exist, `$HADOOP_HOME/etc/hadoop`. Currently the supported configs that are used are:
+- `fs.defaultFS` - Client::default() support
 - `dfs.ha.namenodes` - name service support
 - `dfs.namenode.rpc-address.*` - name service support
+- `fs.viewfs.mounttable.*.link.*` - ViewFS links
+- `fs.viewfs.mounttable.*.linkFallback` - ViewFS link fallback
 
 All other settings are generally assumed to be the defaults currently. For instance, security is assumed to be enabled and SASL negotiation is always done, but on insecure clusters this will just do SIMPLE authentication. Any setups that require other customized Hadoop client configs may not work correctly. 
 

rust/Cargo.toml

@@ -17,9 +17,9 @@ bytes = "1"
 chrono = { version = "0.4", optional = true }
 crc = "3"
 futures = "0.3"
-gsasl-sys = { version = "0.2", default_features = false, optional = true }
+gsasl-sys = { version = "0.2", default-features = false, optional = true }
 libc = "0.2"
-libgssapi = { version = "0.6", default_features = false, optional = true }
+libgssapi = { version = "0.6", default-features = false, optional = true }
 log = "0.4"
 object_store = { version = "0.7", optional = true, features = ["cloud"] }
 prost = "0.11"
@@ -30,7 +30,7 @@ socket2 = "0.5"
 thiserror = "1"
 tokio = { version = "1", features = ["rt", "net", "io-util", "macros", "sync"] }
 url = "2"
-users = { version = "0.11", default_features = false }
+users = { version = "0.11", default-features = false }
 uuid = { version = "1", features = ["v4"] }
 which = { version = "4", optional = true }
 

rust/minidfs/src/main/java/Main.java

@@ -22,6 +22,7 @@
 import org.apache.hadoop.security.token.Token;
 
 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+import static org.apache.hadoop.fs.viewfs.Constants.*;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.*;
 
@@ -57,13 +58,7 @@ public static void main(String args[]) throws Exception {
             conf.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true");
         }
 
-        MiniDFSNNTopology nnTopology = null;
-        if (flags.contains("ha")) {
-            nnTopology = MiniDFSNNTopology.simpleHATopology(3);
-            conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + ".minidfs-ns", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
-            conf.set(DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY, "true");
-            conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, "true");
-        }
+        MiniDFSNNTopology nnTopology = generateTopology(flags, conf);
 
         int numDataNodes = 1;
         if (flags.contains("ec")) {
@@ -79,11 +74,23 @@ public static void main(String args[]) throws Exception {
             .numDataNodes(numDataNodes)
             .build();
 
+        if (flags.contains("viewfs")) {
+            hdfsConf.set(FS_DEFAULT_NAME_KEY, "viewfs://minidfs-viewfs");
+        } else if (flags.contains("ha")) {
+            hdfsConf.set(FS_DEFAULT_NAME_KEY, "hdfs://minidfs-ns");
+        } else {
+            hdfsConf.set(FS_DEFAULT_NAME_KEY, "hdfs://127.0.0.1:9000");
+        }
+
         hdfsConf.writeXml(new FileOutputStream("target/test/core-site.xml"));
         dfs.waitActive();
 
         int activeNamenode = 0;
-        if (flags.contains("ha")) {
+        if (flags.contains("viewfs")) {
+            // Each name services has two namenodes
+            dfs.transitionToActive(0);
+            dfs.transitionToActive(2);
+        } else if (flags.contains("ha")) {
             activeNamenode = 2;
             // dfs.transitionToObserver(1);
             dfs.transitionToActive(activeNamenode);
@@ -133,4 +140,23 @@ public static void main(String args[]) throws Exception {
             kdc.stop();
         }
     }
+
+    public static MiniDFSNNTopology generateTopology(Set<String> flags, Configuration conf) {
+        MiniDFSNNTopology nnTopology = null;
+        if (flags.contains("viewfs")) {
+            nnTopology = MiniDFSNNTopology.simpleHAFederatedTopology(2);
+            conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + ".ns0", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+            conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + ".ns1", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+            conf.set(DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY, "true");
+            conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, "true");
+            conf.set(CONFIG_VIEWFS_PREFIX + ".minidfs-viewfs.link./mount1", "hdfs://ns0/nested");
+            conf.set(CONFIG_VIEWFS_PREFIX + ".minidfs-viewfs.linkFallback", "hdfs://ns1/nested");
+        } else if (flags.contains("ha")) {
+            nnTopology = MiniDFSNNTopology.simpleHATopology(3);
+            conf.set(HdfsClientConfigKeys.Failover.PROXY_PROVIDER_KEY_PREFIX + ".minidfs-ns", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
+            conf.set(DFSConfigKeys.DFS_NAMENODE_STATE_CONTEXT_ENABLED_KEY, "true");
+            conf.set(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, "true");
+        }
+        return nnTopology;
+    }
 }