diff --git a/python/hdfs_native/__init__.py b/python/hdfs_native/__init__.py index 91bf871..90b3cfc 100644 --- a/python/hdfs_native/__init__.py +++ b/python/hdfs_native/__init__.py @@ -88,14 +88,18 @@ def __exit__(self, *_args): class Client: - def __init__(self, url: str, config: Optional[Dict[str, str]] = None): + def __init__( + self, + url: Optional[str] = None, + config: Optional[Dict[str, str]] = None, + ): self.inner = RawClient(url, config) def get_file_info(self, path: str) -> "FileStatus": """Gets the file status for the file at `path`""" return self.inner.get_file_info(path) - def list_status(self, path: str, recursive: bool) -> Iterator["FileStatus"]: + def list_status(self, path: str, recursive: bool = False) -> Iterator["FileStatus"]: """Gets the status of files rooted at `path`. If `recursive` is true, lists all files recursively.""" return self.inner.list_status(path, recursive) @@ -103,15 +107,27 @@ def read(self, path: str) -> FileReader: """Opens a file for reading at `path`""" return FileReader(self.inner.read(path)) - def create(self, path: str, write_options: WriteOptions) -> FileWriter: + def create( + self, + path: str, + write_options: Optional[WriteOptions] = None, + ) -> FileWriter: """Creates a new file and opens it for writing at `path`""" + if not write_options: + write_options = WriteOptions() + return FileWriter(self.inner.create(path, write_options)) def append(self, path: str) -> FileWriter: """Opens an existing file to append to at `path`""" return FileWriter(self.inner.append(path)) - def mkdirs(self, path: str, permission: int, create_parent: bool) -> None: + def mkdirs( + self, + path: str, + permission: int = 0o0755, + create_parent: bool = False, + ) -> None: """ Creates a directory at `path` with unix permissions `permission`. If `create_parent` is true, any parent directories that don't exist will also be created. Otherwise this will fail if @@ -119,7 +135,7 @@ def mkdirs(self, path: str, permission: int, create_parent: bool) -> None: """ return self.inner.mkdirs(path, permission, create_parent) - def rename(self, src: str, dst: str, overwrite: bool) -> None: + def rename(self, src: str, dst: str, overwrite: bool = False) -> None: """ Moves a file or directory from `src` to `dst`. If `overwrite` is True, the destination will be overriden if it already exists, otherwise the operation will fail if the destination @@ -127,7 +143,7 @@ def rename(self, src: str, dst: str, overwrite: bool) -> None: """ return self.inner.rename(src, dst, overwrite) - def delete(self, path: str, recursive: bool) -> bool: + def delete(self, path: str, recursive: bool = False) -> bool: """ Deletes a file or directory at `path`. If `recursive` is True and the target is a directory, this will delete all contents underneath the directory. If `recursive` is False and the target diff --git a/python/hdfs_native/_internal.pyi b/python/hdfs_native/_internal.pyi index b2212b4..6147da7 100644 --- a/python/hdfs_native/_internal.pyi +++ b/python/hdfs_native/_internal.pyi @@ -54,7 +54,11 @@ class RawFileWriter: """Closes the file and saves the final metadata to the NameNode""" class RawClient: - def __init__(self, url: str, config: Optional[Dict[str, str]]) -> None: ... + def __init__( + self, + url: Optional[str], + config: Optional[Dict[str, str]], + ) -> None: ... def get_file_info(self, path: str) -> FileStatus: ... def list_status(self, path: str, recursive: bool) -> Iterator[FileStatus]: ... def read(self, path: str) -> RawFileReader: ... diff --git a/python/src/lib.rs b/python/src/lib.rs index b7866a2..e71c4a8 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -206,13 +206,20 @@ struct RawClient { impl RawClient { #[new] #[pyo3(signature = (url, config))] - pub fn new(url: &str, config: Option>) -> PyResult { + pub fn new(url: Option<&str>, config: Option>) -> PyResult { // Initialize logging, ignore errors if this is called multiple times let _ = env_logger::try_init(); + let config = config.unwrap_or_default(); + + let inner = if let Some(url) = url { + Client::new_with_config(url, config).map_err(PythonHdfsError::from)? + } else { + Client::default_with_config(config).map_err(PythonHdfsError::from)? + }; + Ok(RawClient { - inner: Client::new_with_config(url, config.unwrap_or_default()) - .map_err(PythonHdfsError::from)?, + inner, rt: Arc::new( tokio::runtime::Runtime::new() .map_err(|err| PyRuntimeError::new_err(err.to_string()))?, diff --git a/rust/src/client.rs b/rust/src/client.rs index ba8410f..ec0c0bb 100644 --- a/rust/src/client.rs +++ b/rust/src/client.rs @@ -161,6 +161,17 @@ impl Client { Self::with_config(&parsed_url, Configuration::new_with_config(config)?) } + pub fn default_with_config(config: HashMap) -> Result { + let config = Configuration::new_with_config(config)?; + let url = config + .get(config::DEFAULT_FS) + .ok_or(HdfsError::InvalidArgument(format!( + "No {} setting found", + config::DEFAULT_FS + )))?; + Self::with_config(&Url::parse(&url)?, config) + } + fn with_config(url: &Url, config: Configuration) -> Result { if !url.has_host() { return Err(HdfsError::InvalidArgument( @@ -493,19 +504,7 @@ impl Default for Client { /// Creates a new HDFS Client based on the fs.defaultFS setting. Panics if the config files fail to load, /// no defaultFS is defined, or the defaultFS is invalid. fn default() -> Self { - let config = Configuration::new().expect("Failed to load configuration"); - let url = config - .get(config::DEFAULT_FS) - .ok_or(HdfsError::InvalidArgument(format!( - "No {} setting found", - config::DEFAULT_FS - ))) - .expect("No fs.defaultFS config defined"); - Self::with_config( - &Url::parse(&url).expect("Failed to parse fs.defaultFS"), - config, - ) - .expect("Failed to create default client") + Self::default_with_config(Default::default()).expect("Failed to create default client") } }