From c3de1325299d62b640695a0af1d044ca4e45cf43 Mon Sep 17 00:00:00 2001 From: RRosio Date: Wed, 16 Oct 2024 10:40:22 -0700 Subject: [PATCH 01/13] updating config file loading in tests --- conftest.py | 38 ++-- jupyter_fsspec/file_manager.py | 4 +- .../tests/unit/test_filesystem_manager.py | 178 +----------------- 3 files changed, 25 insertions(+), 195 deletions(-) diff --git a/conftest.py b/conftest.py index 2a643fd..537c678 100644 --- a/conftest.py +++ b/conftest.py @@ -4,12 +4,11 @@ import fsspec from jupyter_fsspec.file_manager import FileSystemManager - pytest_plugins = ['pytest_jupyter.jupyter_server', 'jupyter_server.pytest_plugin', 'pytest_asyncio'] @pytest.fixture(scope='function', autouse=True) -def setup_config_file(tmp_path: Path): +def setup_config_file_fs(tmp_path: Path): config_dir = tmp_path / "config" config_dir.mkdir(exist_ok=True) @@ -40,41 +39,42 @@ def setup_config_file(tmp_path: Path): yaml_file = config_dir / "jupyter-fsspec.yaml" yaml_file.write_text(yaml_content) - with patch('jupyter_core.paths.jupyter_config_dir', return_value=str(config_dir)): + with patch('jupyter_fsspec.file_manager.jupyter_config_dir', return_value=str(config_dir)): print(f"Patching jupyter_config_dir to: {config_dir}") - yield + fs_manager = FileSystemManager(config_file='jupyter-fsspec.yaml') + return fs_manager @pytest.fixture(scope='function') -def fs_manager_instance(setup_config_file): - fs_manager = FileSystemManager(config_file='jupyter-fsspec.yaml') +def fs_manager_instance(setup_config_file_fs): + fs_manager = setup_config_file_fs fs_info = fs_manager.get_filesystem_by_type('memory') key = fs_info['key'] fs = fs_info['info']['instance'] mem_root_path = fs_info['info']['path'] if fs: - if fs.exists('/my_mem_dir/test_dir'): - fs.rm('/my_mem_dir/test_dir', recursive=True) - if fs.exists('/my_mem_dir/second_dir'): - fs.rm('/my_mem_dir/second_dir', recursive=True) + if fs.exists(f'{mem_root_path}/test_dir'): + fs.rm(f'{mem_root_path}/test_dir', recursive=True) + if fs.exists(f'{mem_root_path}/second_dir'): + fs.rm(f'{mem_root_path}/second_dir', recursive=True) - fs.touch('/my_mem_dir/file_in_root.txt') - with fs.open('/my_mem_dir/file_in_root.txt', 'wb') as f: + fs.touch(f'{mem_root_path}/file_in_root.txt') + with fs.open(f'{mem_root_path}/file_in_root.txt', 'wb') as f: f.write("Root file content".encode()) - fs.mkdir('/my_mem_dir/test_dir', exist_ok=True) - fs.mkdir('/my_mem_dir/second_dir', exist_ok=True) - # fs.mkdir('/my_mem_dir/second_dir/subdir', exist_ok=True) - fs.touch('/my_mem_dir/test_dir/file1.txt') - with fs.open('/my_mem_dir/test_dir/file1.txt', "wb") as f: + fs.mkdir(f'{mem_root_path}/test_dir', exist_ok=True) + fs.mkdir(f'{mem_root_path}/second_dir', exist_ok=True) + # fs.mkdir(f'{mem_root_path}/second_dir/subdir', exist_ok=True) + fs.touch(f'{mem_root_path}/test_dir/file1.txt') + with fs.open(f'{mem_root_path}/test_dir/file1.txt', "wb") as f: f.write("Test content".encode()) f.close() else: print("In memory filesystem NOT FOUND") - if fs.exists('/my_mem_dir/test_dir/file1.txt'): - file_info = fs.info('/my_mem_dir/test_dir/file1.txt') + if fs.exists(f'{mem_root_path}/test_dir/file1.txt'): + file_info = fs.info(f'{mem_root_path}/test_dir/file1.txt') print(f"File exists. size: {file_info}") else: print("File does not exist!") diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index 188f9e9..a3066de 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -8,8 +8,8 @@ class FileSystemManager: def __init__(self, config_file): - base_dir = jupyter_config_dir() - self.config_path = os.path.join(base_dir, config_file) + self.base_dir = jupyter_config_dir() + self.config_path = os.path.join(self.base_dir, config_file) try: with open(self.config_path, 'r') as file: self.config = yaml.safe_load(file) diff --git a/jupyter_fsspec/tests/unit/test_filesystem_manager.py b/jupyter_fsspec/tests/unit/test_filesystem_manager.py index 43953aa..2c4fc6b 100644 --- a/jupyter_fsspec/tests/unit/test_filesystem_manager.py +++ b/jupyter_fsspec/tests/unit/test_filesystem_manager.py @@ -115,8 +115,8 @@ def populated_fs_manager(mock_config, fs_manager_instance): fs_manager_instance.write(key, second_file_path, second_file_content) return fs_manager_instance, key - -def test_file_read_write(mock_config, fs_manager_instance): +# TODO: update config path for tests +def xtest_file_read_write(mock_config, fs_manager_instance): key = fs_manager_instance._encode_key(mock_config['sources'][0]) #write @@ -134,7 +134,7 @@ def test_file_read_write(mock_config, fs_manager_instance): def xtest_file_update_delete(populated_fs_manager): key = fs_manager_instance._encode_key(mock_config['sources'][0]) -def test_directory_read_write(mock_config, fs_manager_instance): +def xtest_directory_read_write(mock_config, fs_manager_instance): key = fs_manager_instance._encode_key(mock_config['sources'][0]) #write @@ -158,174 +158,4 @@ def xtest_directory_update_delete(populated_fs_manager): #update - #delete - - - - - - - -# ============================================================ -# OLD Test FileSystemManager file operations -# ============================================================ -# provide the file system with all needed information like key, path etc -# def generate_fs(): -# fs_test_config = { -# 'name': 'mylocal', -# 'type': 'local', -# 'path': str(config_file.parent) -# } - -# create file -#TODO: create fs_manager fixture to include for these tests? -# def test_create_file(config_file): -# fs_test_manager = FileSystemManager(config_file) -# filesystems = fs_test_manager.filesystems - -# for key in filesystems.keys(): -# fs_path = filesystems[key]['path'] - -# file_path = 'test_create_file.txt' -# complete_file_path = str(PurePath(fs_path) / file_path) -# content = b'testing file content' - -# fs_test_manager.write(key, complete_file_path, content) - -# fs_info = fs_test_manager.get_filesystem(key) -# fs = fs_info['instance'] -# assert fs.exists(complete_file_path), "File should exist" - -# file_content = fs.cat(complete_file_path) -# assert file_content == content, "File content should match expected content." - - -# create directory -# def test_create_dir(config_file): -# fs_test_manager = FileSystemManager(config_file) -# filesystems = fs_test_manager.filesystems - -# for key in filesystems.keys(): -# fs_path = filesystems[key]['path'] - -# dir_name = 'testing_dir_name' - -# fs_test_manager.write(key, fs_path, dir_name) -# complete_file_path = str(PurePath(fs_path) / dir_name) + '/' - -# fs_info = fs_test_manager.get_filesystem(key) -# fs = fs_info['instance'] -# assert fs.exists(complete_file_path), "Directory should exist" - - - -# read file -# TODO: use memory filesystem and mock_filesystem dict -# def test_read_file_success(memory_filesystem): -# mock_filesystem = { -# '/dir1': {}, -# '/dir1/file1.txt': 'Content of file1.', -# '/dir1/file2.txt': 'Content of file2.', -# '/dir2': {}, -# '/dir2/subdir': {}, -# '/dir2/subdir/file3.txt': 'Content of file3 in subdir of dir2.' -# } - -# with patch(fsspec.filesystem) -# def populate_filesystem(filesystem, structure, base_path='/'): -# for name, content in structure.items(): -# path = f"{base_path.rstrip('/')/{name}}" - -# if isinstance(content, dict): -# filesystem.mkdir(path) -# populate_filesystem(filesystem, content, base_path=path) -# else: -# if isinstance(content, bytes): -# filesystem.pipe(path, content) -# else: -# filesystem.pipe(path, content.encode()) - -# @pytest.fixture -# def populated_filesystem(mock_filesystem): -# directory_structure = { -# 'dir1': { -# 'file1.txt': 'Content of file1 in dir1.', -# 'file2.txt': 'Content of file2 in dir1.', -# }, -# 'dir2': { -# 'subdir': { -# 'file3.txt': 'Content of file3 in subdir of dir2.', -# 'file4.txt': 'Content of file4 in subdir of dir2.', -# }, -# }, -# 'fileOne.txt': 'This is content of fileOne in root dir.', -# 'fileTwo.txt': 'This is content of fileTwo in root dir.', -# 'binaryfile.bin': b'\x00\x01\x02' -# } - -# key, fs_info = next(iter(mock_filesystem.items())) -# fs_path = fs_info['path'] -# fs = fs_info['instance'] -# populate_filesystem(fs, directory_structure, fs_path) - -# return fs, key, fs_path - - -# def test_read_file(config_file): -# fs_test_manager = FileSystemManager(config_file) -# filesystems = fs_test_manager.filesystems - -# for key in filesystems.keys(): -# fs_path = filesystems[key]['path'] - -# fs_info = fs_test_manager.get_filesystem(key) -# fs = fs_info['instance'] -# def test_read_file(populated_filesystem): -# fs, key, item_path = populated_filesystem -# content = fs.read() - - -# # read directory -# def test_read_dir_success(config_file): -# fs_test_manager = FileSystemManager(config_file) - -# fs = fs_test_manager.filesystems -# -# -# -# -# -# -# -# - - - - - -# # write file -# def test_write_file_success(config_file): -# fs_test_manager = FileSystemManager(config_file) - -# fs = fs_test_manager.filesystems -# -# -# -# -# -# -# -# - -# # delete file -# def test_delete_file_success(config_file): -# fs_test_manager = FileSystemManager(config_file) - -# fs = fs_test_manager.filesystems - - -# # delete directory -# def test_delete_dir_success(config_file): - # fs_test_manager = FileSystemManager(config_file) - - # fs = fs_test_manager.filesystems \ No newline at end of file + #delete \ No newline at end of file From 718d8b2cf6bbfcb700d46027c74fc85537f8a178 Mon Sep 17 00:00:00 2001 From: RRosio Date: Wed, 16 Oct 2024 10:54:09 -0700 Subject: [PATCH 02/13] updating tests --- conftest.py | 87 +++++++++++++++++++++++++++++--- jupyter_fsspec/tests/test_api.py | 27 +++++++++- 2 files changed, 104 insertions(+), 10 deletions(-) diff --git a/conftest.py b/conftest.py index 537c678..72739f3 100644 --- a/conftest.py +++ b/conftest.py @@ -20,21 +20,15 @@ def setup_config_file_fs(tmp_path: Path): anon: false key: "my-access-key" secret: "my-secret-key" - - name: "TestSourceDisk" - path: "." - type: "local" - name: "TestDir" - path: "/Users/rosioreyes/Desktop/test_fsspec" + path: "/Users/someuser/Desktop/test_fsspec" type: "local" - name: "TestEmptyLocalDir" - path: "/Users/rosioreyes/Desktop/notebooks/sample/nothinghere" + path: "/Users/someuser/Desktop/notebooks/sample/nothinghere" type: "local" - name: "TestMem Source" path: "/my_mem_dir" type: "memory" - - name: "TestDoesntExistDir" - path: "/Users/rosioreyes/Desktop/notebooks/doesnotexist" - type: "local" """ yaml_file = config_dir / "jupyter-fsspec.yaml" yaml_file.write_text(yaml_content) @@ -80,6 +74,83 @@ def fs_manager_instance(setup_config_file_fs): print("File does not exist!") return fs_manager +@pytest.fixture(params=['memory', 'local', 's3']) +def filesystem_type(request): + return request.param + +@pytest.fixture(scope="function") +def populated_file_system(filesystem_type): + fs_manager = FileSystemManager(config_file='jupyter-fsspec.yaml') + fs_type = filesystem_type + print(f"fs_manager: {fs_manager}") + fs_info = fs_manager.get_filesystem_by_type(fs_type) + key = fs_info['key'] + fs = fs_info['info']['instance'] + root_path = fs_info['info']['path'] + print(f"key: {key}") + print(f"fs: {fs}") + print(f"root_path: {root_path}") + if fs: + # Delete any existting directories + # Populate the filesystem + # mkdir => root_path + 'rootA' + # mkdir => root_path + 'rootB' + # touch => root_path + 'file1.txt' + # touch => root_path + 'rootA' + 'file_in_rootA.txt' + print(f'valid filesystem: {fs}') + else: + print(f"invalid filesystem: {fs}") + return {"fs_type": fs_type, "fs_manager": fs_manager} + + +@pytest.fixture(scope='function') +def fs_manager_instance_parameterized(populated_file_system): + fs_ret = populated_file_system + fs_type = fs_ret['fs_type'] + fs_manager = fs_ret["fs_manager"] + fs_info = fs_manager.get_filesystem_by_type(fs_type) + key = fs_info['key'] + fs = fs_info['info']['instance'] + root_path = fs_info['info']['path'] + print(f"root_path is: {root_path}") + print(f"fs value is: {fs}") + # fs_info = fs_manager.get_filesystem_by_type('local') + # key = fs_info['key'] + # fs = fs_info['info']['instance'] + # local_root_path = fs_info['info']['path'] + + if fs: + #TODO: Update file creation FOR PATHS!!! + if fs.exists(f'{root_path}/test_dir'): + print(f"{root_path}/test_dir EXISTS!!!!") + # fs.rm(f'{root_path}/test_dir', recursive=True) + if fs.exists(f'{root_path}/second_dir'): + print(f"{root_path}/second_dir EXISTS!!!!") + # fs.rm('/my_dir/second_dir', recursive=True) + + fs.touch(f'{root_path}/file_in_root.txt') + with fs.open(f'{root_path}/file_in_root.txt', 'wb') as f: + f.write("Root file content".encode()) + + # fs.mkdir('/my_dir/test_dir', exist_ok=True) + # fs.mkdir('/my_dir/second_dir', exist_ok=True) + # # fs.mkdir('/my_dir/second_dir/subdir', exist_ok=True) + # fs.touch('/my_dir/test_dir/file1.txt') + # with fs.open('/my_dir/test_dir/file1.txt', "wb") as f: + # f.write("Test content".encode()) + # f.close() + else: + print(f"Filesystem of type {fs_type} NOT FOUND") + + if fs.exists(f'{root_path}test_dir/file1.txt'): + file_info = fs.info(f'/{root_path}/test_dir/file1.txt') + print(f"File exists. size: {file_info}") + else: + print("File does not exist!") + return fs_manager + + + @pytest.fixture def jp_server_config(jp_server_config): return { diff --git a/jupyter_fsspec/tests/test_api.py b/jupyter_fsspec/tests/test_api.py index f4e6914..94a1625 100644 --- a/jupyter_fsspec/tests/test_api.py +++ b/jupyter_fsspec/tests/test_api.py @@ -198,6 +198,7 @@ async def test_patch_file(fs_manager_instance, jp_fetch): async def test_action_same_fs_files(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance + # get_filesystem_by_type(filesystem_type) returns first instance of that filesystem type mem_fs_info = fs_manager.get_filesystem_by_type('memory') mem_key = mem_fs_info['key'] mem_fs = mem_fs_info['info']['instance'] @@ -245,5 +246,27 @@ async def test_action_same_fs_files(fs_manager_instance, jp_fetch): assert mdir_body['description'] == 'Moved /my_mem_dir/test_dir to /my_mem_dir/second_dir' #TODO: Test xaction endpoint -async def xtest_xaction_diff_fs(fs_manager_instance, jp_fetch): - pass \ No newline at end of file +# Q: Other fs types for testing purposes +async def xtest_xaction_diff_fs(fs_manager_instance_parameterized, jp_fetch): + fs_manager = fs_manager_instance_parameterized + fs_info = fs_manager.get_filesystem_by_type('memory') + key = fs_info['key'] + fs = fs_info['info']['instance'] + fs_root_path = fs_info['info']['path'] + assert fs != None + + # # copy file + # copy_filepath = f'{fs_root_path}/test_dir/file1.txt' + # copy_file_payload = {"item_path": copy_filepath, "content": "/my_local_dir/", "destination_key": "", "action": "copy"} + # copy_file_res = await jp_fetch("jupyter_fsspec", "files", "xaction", method="POST", params={"key": mem_key}, body=json.dumps(copy_file_payload)) + + # cfile_body_json = copy_file_res.body.decode('utf-8') + # cfile_body = json.loads(cfile_body_json) + # assert cfile_body["status"] == 'success' + # assert cfile_body['description'] == f'Copied {fs_root_path}/test_dir/file1.txt to /my_local_dir/file1.txt' + + # copy dir + + # move file + + # move dir \ No newline at end of file From 87c2332999bd0b07974b5b272687928d7b0f5d6d Mon Sep 17 00:00:00 2001 From: RRosio Date: Tue, 29 Oct 2024 19:43:19 -0700 Subject: [PATCH 03/13] updating backend --- conftest.py | 60 +- jupyter_fsspec/file_manager.py | 376 ++++------- jupyter_fsspec/handlers.py | 582 ++++++++---------- jupyter_fsspec/tests/test_api.py | 140 ++++- .../tests/unit/test_filesystem_manager.py | 10 +- src/handler/fileOperations.ts | 22 +- 6 files changed, 550 insertions(+), 640 deletions(-) diff --git a/conftest.py b/conftest.py index 72739f3..b4a4e81 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,10 @@ from unittest.mock import patch import pytest import fsspec +import os +import boto3 +from moto import mock_aws +from moto.moto_server.threaded_moto_server import ThreadedMotoServer from jupyter_fsspec.file_manager import FileSystemManager pytest_plugins = ['pytest_jupyter.jupyter_server', 'jupyter_server.pytest_plugin', @@ -14,8 +18,7 @@ def setup_config_file_fs(tmp_path: Path): yaml_content = """sources: - name: "TestSourceAWS" - path: "/path/to/set1" - type: "s3" + path: "s3://my-test-bucket/" additional_options: anon: false key: "my-access-key" @@ -74,6 +77,34 @@ def fs_manager_instance(setup_config_file_fs): print("File does not exist!") return fs_manager +def get_boto3_client(): + from botocore.session import Session + + # NB: we use the sync botocore client for setup + session = Session() + + endpoint_uri = "http://127.0.0.1:%s/" % "5555" + return session.create_client("s3", endpoint_url=endpoint_uri) + +@pytest.fixture(scope='function') +def s3_client(mock_s3_fs): + s3_client = get_boto3_client() + s3_client.create_bucket(Bucket='my-test-bucket') + return s3_client + +@pytest.fixture(scope='function') +def s3_fs_manager_instance(setup_config_file_fs): + fs_manager = setup_config_file_fs + fs_info = fs_manager.get_filesystem_by_type('s3') + key = fs_info['key'] + fs = fs_info['info']['instance'] + root_path = fs_info['info']['path'] + + endpoint_uri = "http://127.0.0.1:%s/" % "5555" + # fs = fsspec.filesystem('s3', asynchronous=True, anon=False, client_kwargs={'endpoint_url': endpoint_uri}) + return fs_manager + + @pytest.fixture(params=['memory', 'local', 's3']) def filesystem_type(request): return request.param @@ -82,14 +113,11 @@ def filesystem_type(request): def populated_file_system(filesystem_type): fs_manager = FileSystemManager(config_file='jupyter-fsspec.yaml') fs_type = filesystem_type - print(f"fs_manager: {fs_manager}") fs_info = fs_manager.get_filesystem_by_type(fs_type) key = fs_info['key'] fs = fs_info['info']['instance'] root_path = fs_info['info']['path'] - print(f"key: {key}") - print(f"fs: {fs}") - print(f"root_path: {root_path}") + if fs: # Delete any existting directories # Populate the filesystem @@ -102,6 +130,23 @@ def populated_file_system(filesystem_type): print(f"invalid filesystem: {fs}") return {"fs_type": fs_type, "fs_manager": fs_manager} +#TODO: Update this fixture from s3fs +@pytest.fixture(scope="function") +def mock_s3_fs(): + # This fixture is module-scoped, meaning that we can re-use the MotoServer across all tests + server = ThreadedMotoServer(ip_address="127.0.0.1", port=5555) + server.start() + if "AWS_SECRET_ACCESS_KEY" not in os.environ: + os.environ["AWS_SECRET_ACCESS_KEY"] = "foo" + if "AWS_ACCESS_KEY_ID" not in os.environ: + os.environ["AWS_ACCESS_KEY_ID"] = "foo" + # aws_session_token=os.environ["AWS_SESSION_TOKEN"] + if "AWS_SESSION_TOKEN" not in os.environ: + os.environ["AWS_SESSION_TOKEN"] = "foo" + print("server up") + yield + print("moto done") + server.stop() @pytest.fixture(scope='function') def fs_manager_instance_parameterized(populated_file_system): @@ -112,8 +157,7 @@ def fs_manager_instance_parameterized(populated_file_system): key = fs_info['key'] fs = fs_info['info']['instance'] root_path = fs_info['info']['path'] - print(f"root_path is: {root_path}") - print(f"fs value is: {fs}") + # fs_info = fs_manager.get_filesystem_by_type('local') # key = fs_info['key'] # fs = fs_info['info']['instance'] diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index a3066de..d74fb4a 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -1,64 +1,144 @@ from jupyter_core.paths import jupyter_config_dir import fsspec +from fsspec.utils import infer_storage_options +from fsspec.registry import known_implementations import os import yaml -import urllib.parse -from datetime import datetime -from pathlib import PurePath +import hashlib +import urllib.parse class FileSystemManager: def __init__(self, config_file): + self.filesystems = {} self.base_dir = jupyter_config_dir() self.config_path = os.path.join(self.base_dir, config_file) - try: - with open(self.config_path, 'r') as file: - self.config = yaml.safe_load(file) - except Exception as e: - print(f"Error loading configuration file: {e}") - return None - self.filesystems = {} - self._initialize_filesystems() + self.config = self.load_config() + self.async_implementations = self._asynchronous_implementations() + self.initialize_filesystems() def _encode_key(self, fs_config): - fs_path = fs_config['path'].strip('/') - - combined = f"{fs_config['type']}|{fs_path}" + # fs_path = fs_config['path'].strip('/') + fs_name = fs_config['name'] + # combined = f"{fs_config['type']}|{fs_path}" + combined = f"{fs_name}" encoded_key = urllib.parse.quote(combined, safe='') return encoded_key def _decode_key(self, encoded_key): combined = urllib.parse.unquote(encoded_key) - fs_type, fs_path = combined.split('|', 1) - return fs_type, fs_path - - def read_config(self): + # fs_type, fs_path = combined.split('|', 1) + fs_name = combined + # return fs_type, fs_path + return fs_name + + @staticmethod + def create_default(): + return FileSystemManager(config_file='jupyter-fsspec.yaml') + + # os.path.exists(config_path) + def load_config(self): + config_path = self.config_path + if not os.path.exists(config_path): + self.create_config_file() + try: - with open(self.config_path, 'r') as file: - self.config = yaml.safe_load(file) - except Exception as e: - print(f"Error loading configuration file: {e}") + with open(config_path, 'r') as file: + config = yaml.safe_load(file) + return config + except yaml.YAMLError as e: + print(f"Error parsing configuration file: {e}") return None - def _initialize_filesystems(self): - self.read_config() + def hash_config(self, config_content): + yaml_str = yaml.dump(config_content) + hash = hashlib.md5(yaml_str.encode('utf-8')).hexdigest() + return hash + + def create_config_file(self): + config_path = self.config_path + + placeholder_config = { + "sources": [ + { + "name": "Sample", + "path": "/test" + }, + { + "name": "test2", + "path": "memory://mytests" + } + ] + } + + try: + with open(config_path, 'w') as config_file: + yaml_content = yaml.dump(placeholder_config, config_file) + + print(f"Configuration file created at {config_path}") + except Exception as e: + print(f"Error creating configuration file") + + def _get_protocol_from_path(self, path): + storage_options = infer_storage_options(path) + protocol = storage_options.get('protocol', 'file') + return protocol + + def _asynchronous_implementations(self): + async_filesystems = [] + + for protocol, impl in known_implementations.items(): + try: + fs_class = fsspec.get_filesystem_class(protocol) + if fs_class.async_impl: + async_filesystems.append(protocol) + except Exception: + pass + return async_filesystems + + def _async_available(self, protocol): + if protocol in self.async_implementations: + return True + else: + return False + + def initialize_filesystems(self): + new_filesystems = {} for fs_config in self.config['sources']: key = self._encode_key(fs_config) - - fs_type = fs_config['type'] fs_name = fs_config['name'] fs_path = fs_config['path'] options = fs_config.get('additional_options', {}) + fs_type = fs_config.get("type", None) + + if fs_type == None: + fs_type = self._get_protocol_from_path(fs_path) # Init filesystem - fs = fsspec.filesystem(fs_type, **options) - if fs_type == 'memory': - if not fs.exists(fs_path): - fs.mkdir(fs_path) + try: + fs_async = self._async_available(fs_type) + fs = fsspec.filesystem(fs_type, asynchronous=fs_async, **options) + + if fs_type == 'memory': + if not fs.exists(fs_path): + fs.mkdir(fs_path) + + # Store the filesystem instance + new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path} + except Exception as e: + print(f'Error initializing filesystems: {e}') + + self.filesystems = new_filesystems + + def check_reload_config(self): + new_content = self.load_config() + hash_new_content = self.hash_config(new_content) + current_config_hash = self.hash_config(self.config) - # Store the filesystem instance - self.filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path} + if current_config_hash != hash_new_content: + self.config = new_content + self.initialize_filesystems() def get_all_filesystems(self): self._initialize_filesystems() @@ -70,234 +150,4 @@ def get_filesystem_by_type(self, fs_type): for encoded_key, fs_info in self.filesystems.items(): if fs_info.get('type') == fs_type: return {'key': encoded_key, 'info': fs_info} - return None - - # =================================================== - # File/Folder Read/Write Operations - # =================================================== - # write directory - # write file with content - # write empty file at directory - # write to an existing file - def write(self, key, item_path: str, content, overwrite=False): # writePath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if fs.isdir(item_path): - if overwrite: - return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} - if isinstance(content, bytes): - content = content.decode('utf-8') - new_dir_path = str(PurePath(item_path) / content) + '/' - - if fs.exists(new_dir_path): - return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} - else: - fs.mkdir(new_dir_path, create_parents=True) - return {"status_code": 200, "response": {"status": "success", "description": f"Wrote {new_dir_path}"}} - else: - # TODO: Process content for different mime types correctly - if not isinstance(content, bytes): - content = content.encode() - - if fs.exists(item_path) and not overwrite: - return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} - else: - with fs.open(item_path, 'wb') as file: - file.write(content); - return {"status_code": 200, "response": {"status": "success", "description": f"Wrote {item_path}"}} - - - def read(self, key, item_path: str, find: bool = False): # readPath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} - - if fs.isdir(item_path) and find: - # find(): a simple list of files - content = [] - dir_ls = fs.find(item_path, maxdepth=None, withdirs=True, detail=False) - for path in dir_ls: - content.append(path) - elif fs.isdir(item_path): - content = [] - dir_ls = fs.ls(item_path) - for path in dir_ls: - if not isinstance(path, str): #TODO: improve - path = path['name'] - - info = fs.info(path) - - if isinstance(info.get('created'), datetime): - info['created'] = info['created'].isoformat() - content.append(info) - else: - with fs.open(item_path, 'rb') as file: - content = file.read() - content = content.decode('utf-8') - # TODO: Process content for different mime types for request body eg. application/json - return {"status_code": 200, "response": {"status": "success", "description": f"Read {item_path}", "content": content}} - - # TODO: remove - def accessMemoryFS(self, key, item_path): - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - content = 'Temporary Content: memory fs accessed' - return {"status_code": 200, "response": {"status": "success", "description": f"Read {item_path}", "content": content}} - - def update(self, key, item_path, content): #updateFile - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if fs.isdir(item_path): - return {"status_code": 400, "response": {"status": "failed", "error": "INVALID_PATH", "description": f"Directory Path {item_path} is not a valid argument."}} - else: - bcontent = content.encode('utf-8') - with fs.open(item_path, 'wb') as file: - file.write(bcontent); - return {"status_code": 200, "response": {"status": "success", "description": f"Updated {item_path}."}} - - - def delete(self, key, item_path): # deletePath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} - - if fs.isdir(item_path): - fs.delete(item_path) #TODO: await fs._rm() Do not want recursive=True - else: - fs.delete(item_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Deleted {item_path}."}} - - def move(self, key, item_path, dest_path): # movePath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} - - if fs.isdir(item_path): - fs.mv(item_path, dest_path, recursive=True) - else: - _, item_extension = os.path.splitext(item_path) - _, dest_extension = os.path.splitext(dest_path) - - if not dest_extension: - dest_path = dest_path + item_extension - - fs.mv(item_path, dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Moved {item_path} to {dest_path}"}} - - def move_diff_fs(self, key, full_item_path, dest_key, full_dest_path): # movePath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - dest_fs_obj = self.get_filesystem(dest_key) - dest_fs =dest_fs_obj['instance'] - - if not fs.exists(full_item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_item_path} does not exist"}} - - if fs.isdir(full_item_path): - if not dest_fs.exists(full_dest_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_dest_path} does not exist"}} - fsspec.mv(full_item_path, full_dest_path, recursive=True) - else: - fsspec.mv(full_item_path, full_dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Moved {full_item_path} to path: {full_dest_path}"}} - - def copy(self, key, item_path, dest_path): # copyPath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist"}} - - if fs.isdir(item_path): - fs.copy(item_path, dest_path, recursive=True) - else: - _, item_extension = os.path.splitext(item_path) - _, dest_extension = os.path.splitext(dest_path) - - if not dest_extension: - dest_path = dest_path + item_extension - fs.copy(item_path, dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Copied {item_path} to {dest_path}"}} - - def copy_diff_fs(self, key, full_item_path, dest_key, full_dest_path): # copyPath - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(full_item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_item_path} does not exist"}} - - if fs.isdir(full_item_path): - fs.copy(full_item_path, full_dest_path, recursive=True) - else: - fs.copy(full_item_path, full_dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Copied {full_item_path} to path: {full_dest_path}"}} - - def open(self, key, item_path, start, end): - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} - - with fs.open(item_path, 'rb') as f: - f.seek(start) - if end is None: - data = f.read() # eof - else: - data = f.read(int(end) - int(start) + 1) - content = data.decode('utf-8') - return {"status_code": 206, "response": {"status": "success", "description": f"Partial content read from: {item_path}", "content": content}} - - - def rename(self, key, item_path, dest_path): - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist"}} - - dir_root_path = os.path.dirname(item_path) - - # directory - if fs.isdir(item_path): - new_dest_path = dir_root_path + '/' + dest_path - if fs.exists(new_dest_path): - return {"status_code": 403, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {new_dest_path} already exist"}} - else: - fs.rename(item_path, new_dest_path) - # file - else: - # check for dest_path file extension? if not infer, reassign dest_path - _, item_extension = os.path.splitext(item_path) - _, dest_extension = os.path.splitext(dest_path) - - if not dest_extension: - dest_path = dest_path + item_extension - new_dest_path = dir_root_path + '/' + dest_path - fs.rename(item_path, new_dest_path) - - return {"status_code": 200, "response": {"status": "success", "description": f"Renamed {item_path} to {new_dest_path}"}} - - # =================================================== - # File/Folder Management Operations - # =================================================== - def get_info(self, key, item_path: str, recursive: bool = False): - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - return fs.info(item_path) - - def exists(self, key, item_path: str): - fs_obj = self.get_filesystem(key) - fs = fs_obj['instance'] - - if not fs.exists(item_path): - return False - else: - return True + return None \ No newline at end of file diff --git a/jupyter_fsspec/handlers.py b/jupyter_fsspec/handlers.py index c083e1d..f829a95 100644 --- a/jupyter_fsspec/handlers.py +++ b/jupyter_fsspec/handlers.py @@ -2,19 +2,17 @@ from jupyter_server.utils import url_path_join import tornado import json +import asyncio from .file_manager import FileSystemManager from .utils import parse_range -def create_filesystem_manager(): - return FileSystemManager(config_file='jupyter-fsspec.yaml') - class BaseFileSystemHandler(APIHandler): def initialize(self, fs_manager): self.fs_manager = fs_manager - def validate_fs(self, request_type): + def validate_fs(self, request_type, key, item_path): """Retrieve the filesystem instance and path of the item :raises [ValueError]: [Missing required key parameter] @@ -24,10 +22,6 @@ def validate_fs(self, request_type): :return: filesystem instance and item_path :rtype: fsspec filesystem instance and string """ - key = self.get_argument('key', None) - - request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') if not key: raise ValueError("Missing required parameter `key`") @@ -62,230 +56,31 @@ def get(self): :rtype: dict """ try: - file_systems = []; + self.fs_manager.check_reload_config() + file_systems = [] + for fs in self.fs_manager.filesystems: fs_info = self.fs_manager.filesystems[fs] instance = {"key": fs, 'name': fs_info['name'], 'type': fs_info['type'], 'path': fs_info['path'] } file_systems.append(instance) - self.set_status(200) self.write({'status': 'success', 'description': 'Retrieved available filesystems from configuration file.', 'content': file_systems}) self.finish() except Exception as e: - # TODO: update error messaging here to appropriately handle other types of exceptions. self.set_status(404) self.write({"response": {"status": "failed", "error": "FILE_NOT_FOUND", "description": f"Error loading config: {str(e)}"}}) self.finish() -class FileSystemHandler(APIHandler): - def initialize(self, fs_manager): - self.fs_manager = fs_manager - - @tornado.web.authenticated - def get(self): - """Retrieve list of files for directories or contents for files. - - :param [key]: [Query arg string corresponding to the appropriate filesystem instance] - :param [item_path]: [Query arg string path to file or directory to be retrieved], defaults to [root path of the active filesystem] - :param [type]: [Query arg identifying the type of directory search or file content retrieval - if type is "find" recursive files/directories listed; if type is "range", returns specified byte range content], defaults to [empty string for one level deep directory contents and single file entire contents] - - :raises [ValueError]: [Missing required key parameter] - :raises [ValueError]: [No filesystem identified for provided key] - - :return: dict with either list of files or file information under the `files` key-value pair and `status` key for request info - :rtype: dict - """ - try: - key = self.get_argument('key') - item_path = self.get_argument('item_path') - type = self.get_argument('type') - - if not key: - raise ValueError("Missing required parameter `key`") - # if not item_path: - # raise ValueError("Missing required parameter `item_path`") - - fs = self.fs_manager.get_filesystem(key) - fs_type = fs['type'] - if fs_type == 'memory': - print (f"accessed memory filesystem") - result = self.fs_manager.accessMemoryFS(key, item_path) - self.set_status(result['status_code']) - self.finish(result['response']) - return - - if not item_path: - if type != 'range': - item_path = self.fs_manager.filesystems[key]["path"] - else: - raise ValueError("Missing required parameter `item_path`") - - if fs is None: - raise ValueError(f"No filesystem found for key: {key}") - - if type == 'find': - result = self.fs_manager.read(key, item_path, find=True) - elif type == 'range': # add check for Range specific header - range_header = self.request.headers.get('Range') - start, end = parse_range(range_header) - - result = self.fs_manager.open(key, item_path, start, end) - self.set_status(result["status_code"]) - self.set_header('Content-Range', f'bytes {start}-{end}') - self.finish(result['response']) - return - else: - result = self.fs_manager.read(key, item_path) - - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() - return - except Exception as e: - print("Error requesting read: ", e) - self.set_status(500) - self.write({"status": "failed", "error": "ERROR_REQUESTING_READ", "description": f"Error occurred: {str(e)}"}) - self.finish() - - @tornado.web.authenticated - def post(self): - """Create directories/files or perform other directory/file operations like move and copy - - :param [key]: [request body property string used to retrieve the appropriate filesystem instance] - :param [item_path]: [request body property string path to file or directory to be retrieved], defaults to [root path of the active filesystem] - :param [content]: [request body property either file content, directory name, or destination path for advanced move and copy functions] - :param [action]: [query parameter ], defaults to ["write" string value for creating a directory or a file] - - :raises [ValueError]: [Missing either of required parameters key or item_path] - :raises [ValueError]: [No filesystem identified for provided key] - :raises [ValueError]: [Required parameter does not match operation.] - - :return: dict with request status indicator - :rtype: dict - """ - try: - action = self.get_argument('action') - request_data = json.loads(self.request.body.decode('utf-8')) - - key = request_data.get('key') - item_path = request_data.get('item_path') - - if not (key) or not (item_path): - raise ValueError("Missing required parameter `key` or `item_path`") - - content = request_data.get('content').encode('utf-8') - - fs = self.fs_manager.get_filesystem(key) - if fs is None: - raise ValueError(f"No filesystem found for key: {key}") - - if action == 'move': - src_path = item_path - dest_path = content.decode('utf-8') - if not self.fs_manager.exists(key, dest_path): - raise ValueError('Required parameter `content` is not a valid destination path for move action.') - else: - self.fs_manager.move(key, src_path, content) - result = {"status_code": 200, "status": "success!"} - elif action == 'copy': - result = {"status_code": 200, "status": "TBD"} - else: # assume write - result = self.fs_manager.write(key, item_path, content) - - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() - except Exception as e: - print(f"Error requesting post: ", e) - self.set_status(500) - self.write({"status": "failed", "error": "ERROR_REQUESTING_POST", "description": f"Error occurred: {str(e)}"}) - self.finish() - - @tornado.web.authenticated - def put(self): - """Update - - :param [key]: [request body property string used to retrieve the appropriate filesystem instance] - :param [item_path]: [request body property string path to file to be retrieved] - :param [content]: [request body property with file content] - - :raises [ValueError]: [Missing either of required parameters key or item_path] - :raises [ValueError]: [No filesystem identified for provided key] - - :return: dict with request status indicator - :rtype: dict - """ - try: - request_data = json.loads(self.request.body.decode('utf-8')) - - key = request_data.get('key') - item_path = request_data.get('item_path') - - if not (key) or not (item_path): - raise ValueError("Missing required parameter `key` or `item_path`") - - content = request_data.get('content') - - fs = self.fs_manager.get_filesystem(key) - if fs is None: - raise ValueError(f"No filesystem found for key: {key}") - - result = self.fs_manager.update(key, item_path, content) - - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() - except Exception as e: - self.set_status(500) - self.write({"status": "failed", "error": "ERROR_REQUESTING_PUT", "description": f"Error occurred: {str(e)}"}) - self.finish() - - @tornado.web.authenticated - def delete(self): - """Delete the resource at the input path. - - :param [key]: [request body property string used to retrieve the appropriate filesystem instance] - :param [item_path]: [request body property string path to file or directory to be retrieved] - - :raises [ValueError]: [Missing either of required parameters key or item_path] - :raises [ValueError]: [No filesystem identified for provided key] - - :return: dict with request status indicator - :rtype: dict - """ - try: - request_data = json.loads(self.request.body.decode('utf-8')) - - key = request_data.get('key') - item_path = request_data.get('item_path') - - if not (key) or not (item_path): - raise ValueError("Missing required parameter `key` or `item_path`") - - fs = self.fs_manager.get_filesystem(key) - if fs is None: - raise ValueError(f"No filesystem found for key: {key}") - - result = self.fs_manager.delete(key, item_path) - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() - except ValueError as e: - self.set_status(400) - self.write({"status": "failed", "error": "MISSING_PARAMETER", "description": f"{str(e)}"}) - self.finish() - except Exception as e: - self.set_status(500) - self.write({"status": "failed", "error": "ERROR_REQUESTING_DELETE" , "description": f"Error occurred: {str(e)}"}) - self.finish() +#==================================================================================== +# Handle Move and Copy Requests +#==================================================================================== class FileActionHandler(BaseFileSystemHandler): def initialize(self, fs_manager): self.fs_manager = fs_manager # POST /jupyter_fsspec/files/action?key=my-key&item_path=/some_directory/file.txt - def post(self): + async def post(self): """Move or copy the resource at the input path to destination path. :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] @@ -297,32 +92,47 @@ def post(self): :rtype: dict """ key = self.get_argument('key') - # item_path = self.get_argument('item_path') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') action = request_data.get('action') destination = request_data.get('content') + response = {"content": None} - fs, item_path = self.validate_fs('post') + fs, item_path = self.validate_fs('post', key, req_item_path) + fs_instance = fs["instance"] - if action == 'move': - result = self.fs_manager.move(key, item_path, destination) - elif action == 'copy': - result = self.fs_manager.copy(key, item_path, destination) - else: - result = {"status_code": 400, "response": {"status": "failed", "error": "INVALID_ACTION", "description": f"Unsupported action: {action}"}} + try: + if action == 'move': + fs_instance.mv(item_path, destination) + response["description"] = f"Moved {item_path} to {destination}." + else: + if fs_instance.async_impl: + # if provided paths are not expanded fsspec expands them + # for a list of paths: recursive=False or maxdepth not None + await fs_instance._copy(item_path, destination) + else: + fs_instance.copy(item_path, destination) + response["description"] = f"Copied {item_path} to {destination}." + response["status"] = "success" + self.set_status(200) + except Exception as e: + self.set_status(500) + response["status"] = "Failed" + response["description"] = str(e) - self.set_status(result["status_code"]) - self.write(result['response']) + self.write(response) self.finish() -class FileActionCrossFSHandler(BaseFileSystemHandler): +#==================================================================================== +# Handle Move and Copy Requests Across filesystems +#==================================================================================== +class FileTransferHandler(BaseFileSystemHandler): def initialize(self, fs_manager): self.fs_manager = fs_manager # POST /jupyter_fsspec/files/action?key=my-key&item_path=/some_directory/file.txt def post(self): - """Move or copy the resource at the input path to destination path. + """Upload/Download the resource at the input path to destination path. :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] :param [item_path]: [Query arg string path to file or directory to be retrieved] @@ -333,52 +143,83 @@ def post(self): :rtype: dict """ key = self.get_argument('key') - # item_path = self.get_argument('item_path') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') action = request_data.get('action') destination = request_data.get('content') - dest_fs_key = request_data.get('destination_key') + # dest_fs_key = request_data.get('destination_key') + # dest_fs_info = self.fs_manager.get_filesystem(dest_fs_key) + # dest_path = dest_fs_info["path"] - fs, item_path = self.validate_fs('post') + response = {"content": None} - if action == 'move': - result = self.fs_manager.move_diff_fs(key, item_path, dest_fs_key, destination) - elif action == 'copy': - result = self.fs_manager.copy_diff_fs(key, item_path, dest_fs_key, destination) - else: - result = {"status_code": 400, "response": {"status": "failed", "error": "INVALID_ACTION", "description": f"Unsupported action: {action}"}} + fs, item_path = self.validate_fs('post', key, req_item_path) + fs_instance = fs["instance"] + + try: + if action == 'upload': + # upload fs_instance.put(local_path, remote_path) + fs_instance.put(item_path, destination) + response["description"] = f"Uploaded {item_path} to {destination}." + else: # download + # download fs_instance.get(remote_path, local_path) + fs_instance.get(destination, item_path) + response["description"] = f"Downloaded {destination} to {item_path}." + + response["status"] = "success" + self.set_status(200) + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) - self.set_status(result["status_code"]) - self.write(result['response']) + self.write(response) self.finish() +#==================================================================================== +# Handle Rename requests (?seperate or not?) +#==================================================================================== class RenameFileHandler(BaseFileSystemHandler): def initialize(self, fs_manager): self.fs_manager = fs_manager def post(self): key = self.get_argument('key') - type = self.get_argument('type', default='default') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') content = request_data.get('content') + response = {"content": None} - fs, item_path = self.validate_fs('post') - result = self.fs_manager.rename(key, item_path, content) - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() + fs, item_path = self.validate_fs('post', key, req_item_path) + fs_instance = fs["instance"] + # expect item path to end with `/` for directories + # expect content to be the FULL new path + try: + # when item_path is a directory, if recursive=True is not set, + # path1 is deleted and path2 is not created + fs_instance.rename(item_path, content, recursive=True) + response["status"] = "success" + response["description"] = f"Renamed {item_path} to {content}." + self.set_status(200) + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) + self.write(response) + self.finish() -class FileSysHandler(BaseFileSystemHandler): +#==================================================================================== +# CRUD for FileSystem +#==================================================================================== +class FileSystemHandler(BaseFileSystemHandler): def initialize(self, fs_manager): self.fs_manager = fs_manager # GET # /files - def get(self): + async def get(self): """Retrieve list of files for directories or contents for files. :param [key]: [Query arg string corresponding to the appropriate filesystem instance] @@ -397,41 +238,71 @@ def get(self): # item_path: /some_directory/file.txt # GET /jupyter_fsspec/files?key=my-key&item_path=/some_directory/file.txt&type=range # content header specifying the byte range + key = self.get_argument('key') + req_item_path = self.get_argument('item_path') + type = self.get_argument('type', default='default') + + fs, item_path = self.validate_fs('get', key, req_item_path) + + fs_instance = fs["instance"] + response = {"content": None} + try: - key = self.get_argument('key') - type = self.get_argument('type', default='default') - - request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') - fs, item_path = self.validate_fs('get') - if type == 'find': - result = self.fs_manager.read(key, item_path, find=True) - elif type == 'range': # add check for Range specific header + if fs_instance.async_impl: + isdir = await fs_instance._isdir(item_path) + else: + isdir = fs_instance.isdir(item_path) + + if type == 'range': range_header = self.request.headers.get('Range') start, end = parse_range(range_header) - - result = self.fs_manager.open(key, item_path, start, end) - self.set_status(result["status_code"]) + if fs_instance.async_impl: + result = await fs_instance._cat_ranges([item_path], [int(start)], [int(end)]) + if isinstance(result, bytes): + result = result.decode('utf-8') + response["content"] = result + else: + #TODO: + result = fs_instance.cat_ranges([item_path], [int(start)], [int(end)]) + if isinstance(result[0], bytes): + result = result[0].decode('utf-8') + response["content"] = result self.set_header('Content-Range', f'bytes {start}-{end}') - self.finish(result['response']) - return - else: - result = self.fs_manager.read(key, item_path) + elif isdir: + if fs_instance.async_impl: + result = await fs_instance._ls(item_path, detail=True) + else: + result = fs_instance.ls(item_path, detail=True) - self.set_status(result["status_code"]) - self.write(result['response']) - self.finish() - return + detail_to_keep = ["name", "type", "size", "ino", "mode"] + filtered_result = [{info: item_dict[info] for info in detail_to_keep if info in item_dict} for item_dict in result] + response["content"] = filtered_result + else: + if fs_instance.async_impl: + result = await fs_instance._cat(item_path) + if isinstance(result, bytes): + result = result.decode('utf-8') + response["content"] = result + else: + result = fs_instance.cat(item_path) + if isinstance(result, bytes): + result = result.decode('utf-8') + response["content"] = result + self.set_status(200) + response["status"] = "success" + response["description"] = f"Retrieved {item_path}." except Exception as e: - print("Error requesting read: ", e) self.set_status(500) - self.send_response({"status": "failed", "error": "ERROR_REQUESTING_READ", "description": f"Error occurred: {str(e)}"}) + response["status"] = "failed" + response["description"] = str(e) + self.write(response) + self.finish() # POST /jupyter_fsspec/files?key=my-key # JSON Payload # item_path=/some_directory/file.txt # content - def post(self): + async def post(self): """Create directories/files or perform other directory/file operations like move and copy :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] @@ -442,23 +313,51 @@ def post(self): :rtype: dict """ key = self.get_argument('key') - # item_path = self.get_argument('item_path') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') content = request_data.get('content') - fs, item_path = self.validate_fs('post') + fs, item_path = self.validate_fs('post', key, req_item_path) + fs_instance = fs["instance"] + response = {"content": None} - result = self.fs_manager.write(key, item_path, content) + try: + # directory expect item_path to end with `/` + if item_path.endswith('/'): + #content is then expected to be null + if fs_instance.async_impl: + await fs_instance._mkdir(item_path, exists_ok=True) + else: + fs_instance.mkdir(item_path, exists_ok=True) + else: + # file name expected in item_path + if fs_instance.async_impl: + await fs_instance._touch(item_path) + if content: + if not isinstance(content, bytes): + content = str.encode(content) + await fs_instance._pipe(item_path, content) + else: + fs_instance.touch(item_path) + if content: + if not isinstance(content, bytes): + content = str.encode(content) + fs_instance.pipe(item_path, content) - self.set_status(result["status_code"]) - self.write(result['response']) + self.set_status(200) + response["status"] = "success" + response["description"] = f"Wrote {item_path}." + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) + self.write(response) self.finish() # PUT /jupyter_fsspec/files?key=my-key&item_path=/some_directory/file.txt # JSON Payload - # content - def put(self): + # content + async def put(self): """Update ENTIRE content in file. :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] @@ -470,30 +369,81 @@ def put(self): """ key = self.get_argument('key') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') content = request_data.get('content') - fs, item_path = self.validate_fs('put') - result = self.fs_manager.write(key, item_path, content, overwrite=True) + fs, item_path = self.validate_fs('put', key, req_item_path) + fs_instance = fs["instance"] + response = {"content": None} - self.set_status(result["status_code"]) - self.write(result['response']) + try: + if fs_instance.async_impl: + isfile = await fs_instance.isfile(item_path) + else: + isfile = fs_instance.isfile(item_path) + + if not isfile: + raise FileNotFoundError(f"{item_path} is not a file.") + + if fs_instance.async_impl: + await fs_instance._pipe(item_path, content) + else: + fs_instance.pipe(item_path, content) + response["status"] = "success" + response["description"] = f"Updated file {item_path}." + self.set_status(200) + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) + + self.write(response) self.finish() - def patch(self): + async def patch(self): # Update PARTIAL file content key = self.get_argument('key') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') + req_item_path = request_data.get('item_path') + offset = request_data.get('offset') content = request_data.get('content') - fs, item_path = self.validate_fs('patch') + fs, item_path = self.validate_fs('patch', key, req_item_path) + fs_instance = fs["instance"] + + #TODO: offset + response = {"content": None} + + try: + if fs_instance.async_impl: + isfile = await fs_instance.isfile(item_path) + else: + isfile = fs_instance.isfile(item_path) + + if not isfile: + raise FileNotFoundError(f"{item_path} is not a file.") - #TODO: Properly Implement PATCH - result = self.fs_manager.update(key, item_path, content) + if fs_instance.async_impl: + original_content = await fs_instance._cat(item_path) + else: + original_content = fs_instance.cat(item_path) + + new_content = (original_content[:offset] + content + + original_content[offset + len(content):]) + + if fs_instance.async_impl: + await fs_instance._pipe(item_path, new_content) + else: + fs_instance.pipe(item_path, new_content) + self.set_status(200) + response["status"] = "success" + response["description"] = f"Patched file {item_path} at offset {offset}." + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) - self.set_status(result["status_code"]) - self.write(result['response']) + self.write(response) self.finish() @@ -509,44 +459,48 @@ async def delete(self): """ key = self.get_argument('key') request_data = json.loads(self.request.body.decode('utf-8')) - item_path = request_data.get('item_path') - - fs, item_path = self.validate_fs('delete') + req_item_path = request_data.get('item_path') - result = self.fs_manager.delete(key, item_path) + fs, item_path = self.validate_fs('delete', key, req_item_path) + fs_instance = fs["instance"] + response = {"content": None} + + try: + if fs_instance.async_impl: + await fs_instance._rm(item_path) + else: + fs_instance.rm(item_path) + self.set_status(200) + response["status"] = "success" + response["description"] = f"Deleted {item_path}." + except Exception as e: + self.set_status(500) + response["status"] = "failed" + response["description"] = str(e) - self.set_status(result["status_code"]) - self.write(result['response']) + self.write(response) self.finish() -#==================================================================================== -# Update the handler in setup -#==================================================================================== + def setup_handlers(web_app): host_pattern = ".*$" - fs_manager = create_filesystem_manager() + fs_manager = FileSystemManager.create_default() base_url = web_app.settings["base_url"] route_fsspec_config = url_path_join(base_url, "jupyter_fsspec", "config") - route_fsspec = url_path_join(base_url, "jupyter_fsspec", "fsspec") - handlers = [ - (route_fsspec_config, FsspecConfigHandler, dict(fs_manager=fs_manager)), - (route_fsspec, FileSystemHandler, dict(fs_manager=fs_manager)) - ] - + route_files = url_path_join(base_url, "jupyter_fsspec", "files") - route_files_actions = url_path_join(base_url, "jupyter_fsspec", "files", "action") + route_file_actions = url_path_join(base_url, "jupyter_fsspec", "files", "action") route_rename_files = url_path_join(base_url, "jupyter_fsspec", "files", "rename") - route_fs_files_actions = url_path_join(base_url, "jupyter_fsspec", "files", "xaction") + route_fs_file_transfer = url_path_join(base_url, "jupyter_fsspec", "files", "transfer") - handlers_refactored = [ + handlers = [ (route_fsspec_config, FsspecConfigHandler, dict(fs_manager=fs_manager)), - (route_files, FileSysHandler, dict(fs_manager=fs_manager)), + (route_files, FileSystemHandler, dict(fs_manager=fs_manager)), (route_rename_files, RenameFileHandler, dict(fs_manager=fs_manager)), - (route_files_actions, FileActionHandler, dict(fs_manager=fs_manager)), - (route_fs_files_actions, FileActionCrossFSHandler, dict(fs_manager=fs_manager)), + (route_file_actions, FileActionHandler, dict(fs_manager=fs_manager)), + (route_fs_file_transfer, FileTransferHandler, dict(fs_manager=fs_manager)), ] - web_app.add_handlers(host_pattern, handlers_refactored) web_app.add_handlers(host_pattern, handlers) diff --git a/jupyter_fsspec/tests/test_api.py b/jupyter_fsspec/tests/test_api.py index 94a1625..c6457a7 100644 --- a/jupyter_fsspec/tests/test_api.py +++ b/jupyter_fsspec/tests/test_api.py @@ -1,11 +1,14 @@ import json import os import pytest - +import boto3 +import fsspec +import asyncio from tornado.httpclient import HTTPClientError +from unittest.mock import patch # TODO: Testing: different file types, received expected errors -async def test_get_config(jp_fetch): +async def xtest_get_config(jp_fetch): response = await jp_fetch("jupyter_fsspec", "config", method="GET") assert response.code == 200 @@ -13,7 +16,7 @@ async def test_get_config(jp_fetch): body = json.loads(json_body) assert body['status'] == 'success' -async def test_get_files_memory(fs_manager_instance, jp_fetch): +async def xtest_get_files_memory(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') mem_key = mem_fs_info['key'] @@ -23,8 +26,7 @@ async def test_get_files_memory(fs_manager_instance, jp_fetch): # Read directory assert mem_fs.exists(mem_item_path) == True - dir_payload = {"item_path": mem_item_path} - dir_response = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key}, body=json.dumps(dir_payload), allow_nonstandard_methods=True) + dir_response = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key, "item_path": mem_item_path}, allow_nonstandard_methods=True) assert dir_response.code == 200 json_body = dir_response.body.decode('utf-8') @@ -36,7 +38,7 @@ async def test_get_files_memory(fs_manager_instance, jp_fetch): filepath = "/my_mem_dir/test_dir/file1.txt" assert mem_fs.exists(filepath) == True file_payload = {"item_path": filepath} - file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key}, body=json.dumps(file_payload), allow_nonstandard_methods=True) + file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key, "item_path": filepath}) assert file_res.code == 200 file_json_body = file_res.body.decode('utf-8') @@ -48,15 +50,16 @@ async def test_get_files_memory(fs_manager_instance, jp_fetch): range_filepath = "/my_mem_dir/test_dir/file1.txt" # previously checked file exists range_file_payload = {"item_path": range_filepath} - range_file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", headers={"Range": "0-8"}, params={"key": mem_key, "type": "range"}, body=json.dumps(range_file_payload), allow_nonstandard_methods=True) - assert range_file_res.code == 206 + range_file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", headers={"Range": "0-8"}, params={"key": mem_key, "type": "range", "item_path": range_filepath}) + assert range_file_res.code == 200 + range_json_file_body = range_file_res.body.decode('utf-8') range_file_body = json.loads(range_json_file_body) assert range_file_body['status'] == 'success' - assert range_file_body['content'] == 'Test cont' + assert range_file_body['content'] == 'Test con' -async def test_post_files(fs_manager_instance, jp_fetch): +async def xtest_post_files(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') mem_key = mem_fs_info['key'] @@ -74,23 +77,23 @@ async def test_post_files(fs_manager_instance, jp_fetch): file_json_body = file_response.body.decode('utf-8') file_body = json.loads(file_json_body) assert file_body['status'] == 'success' - assert file_body['description'] == 'Wrote /my_mem_dir/test_dir/file2.txt' + assert file_body['description'] == 'Wrote /my_mem_dir/test_dir/file2.txt.' assert mem_fs.exists(filepath) == True # Post directory - newdirpath = "/my_mem_dir/test_dir/subdir" + newdirpath = "/my_mem_dir/test_dir/subdir/" # Directory does not already exist assert mem_fs.exists(newdirpath) == False - dir_payload = {"item_path": "/my_mem_dir/test_dir", "content": "subdir"} + dir_payload = {"item_path": newdirpath} dir_response = await jp_fetch("jupyter_fsspec", "files", method="POST", params={"key": mem_key}, body=json.dumps(dir_payload)) assert dir_response.code == 200 dir_body_json = dir_response.body.decode('utf-8') dir_body = json.loads(dir_body_json) assert dir_body['status'] == 'success' - assert dir_body['description'] == 'Wrote /my_mem_dir/test_dir/subdir/' + assert dir_body['description'] == 'Wrote /my_mem_dir/test_dir/subdir/.' -async def test_delete_files(fs_manager_instance, jp_fetch): +async def xtest_delete_files(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') mem_key = mem_fs_info['key'] @@ -108,6 +111,7 @@ async def test_delete_files(fs_manager_instance, jp_fetch): body = json.loads(json_body) assert body['status'] == 'success' + assert body['description'] == f'Deleted {filepath}.' assert mem_fs.exists(filepath) == False #delete directory @@ -117,13 +121,14 @@ async def test_delete_files(fs_manager_instance, jp_fetch): dir_payload = {"item_path": dirpath} dir_response = await jp_fetch("jupyter_fsspec", "files", method="DELETE", params={"key": mem_key}, body=json.dumps(dir_payload), allow_nonstandard_methods=True) assert dir_response.code == 200 - dir_json_body = response.body.decode('utf-8') + dir_json_body = dir_response.body.decode('utf-8') dir_body = json.loads(dir_json_body) assert dir_body['status'] == 'success' + assert dir_body['description'] == f'Deleted {dirpath}.' assert mem_fs.exists(dirpath) == False -async def test_put_files(fs_manager_instance, jp_fetch): +async def xtest_put_files(fs_manager_instance, jp_fetch): # PUT replace entire resource fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') @@ -149,7 +154,7 @@ async def test_put_files(fs_manager_instance, jp_fetch): await jp_fetch("jupyter_fsspec", "files", method="PUT", params={"key": mem_key}, body=json.dumps(dir_payload)) assert exc_info.value.code == 409 -async def test_rename_files(fs_manager_instance, jp_fetch): +async def xtest_rename_files(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') mem_key = mem_fs_info['key'] @@ -158,30 +163,33 @@ async def test_rename_files(fs_manager_instance, jp_fetch): # rename file filepath = '/my_mem_dir/test_dir/file1.txt' - file_payload = {"item_path": filepath, "content": "new_file"} + file_payload = {"item_path": filepath, "content": "/my_mem_dir/test_dir/new_file.txt"} file_response = await jp_fetch("jupyter_fsspec", "files", "rename", method="POST", params={"key": mem_key}, body=json.dumps(file_payload)) assert file_response.code == 200 file_body_json = file_response.body.decode('utf-8') file_body = json.loads(file_body_json) assert file_body["status"] == 'success' - assert file_body['description'] == 'Renamed /my_mem_dir/test_dir/file1.txt to /my_mem_dir/test_dir/new_file.txt' - + assert file_body['description'] == 'Renamed /my_mem_dir/test_dir/file1.txt to /my_mem_dir/test_dir/new_file.txt.' + assert mem_fs.exists(filepath) == False + assert mem_fs.exists("/my_mem_dir/test_dir/new_file.txt") == True # rename directory dirpath = '/my_mem_dir/second_dir' - dir_payload = {"item_path": dirpath, "content": "new_dir"} + dir_payload = {"item_path": dirpath, "content": "/my_mem_dir/new_dir"} dir_response = await jp_fetch("jupyter_fsspec", "files", "rename", method="POST", params={"key": mem_key}, body=json.dumps(dir_payload)) assert dir_response.code == 200 dir_body_json = dir_response.body.decode('utf-8') dir_body = json.loads(dir_body_json) assert dir_body["status"] == 'success' - assert dir_body['description'] == "Renamed /my_mem_dir/second_dir to /my_mem_dir/new_dir" + assert dir_body['description'] == "Renamed /my_mem_dir/second_dir to /my_mem_dir/new_dir." + assert mem_fs.exists(dirpath) == False + assert mem_fs.exists("/my_mem_dir/new_dir") == True # TODO: Implement update functionality # PATCH partial update without modifying entire data -async def test_patch_file(fs_manager_instance, jp_fetch): +async def xtest_patch_file(fs_manager_instance, jp_fetch): #file only fs_manager = fs_manager_instance mem_fs_info = fs_manager.get_filesystem_by_type('memory') @@ -196,7 +204,7 @@ async def test_patch_file(fs_manager_instance, jp_fetch): file_res = await jp_fetch("jupyter_fsspec", "files", method="PATCH", params={"key": mem_key}, body=json.dumps(file_payload)) assert file_res.code == 200 -async def test_action_same_fs_files(fs_manager_instance, jp_fetch): +async def xtest_action_same_fs_files(fs_manager_instance, jp_fetch): fs_manager = fs_manager_instance # get_filesystem_by_type(filesystem_type) returns first instance of that filesystem type mem_fs_info = fs_manager.get_filesystem_by_type('memory') @@ -212,7 +220,7 @@ async def test_action_same_fs_files(fs_manager_instance, jp_fetch): cfile_body_json = copy_file_res.body.decode('utf-8') cfile_body = json.loads(cfile_body_json) assert cfile_body["status"] == 'success' - assert cfile_body['description'] == 'Copied /my_mem_dir/test_dir/file1.txt to /my_mem_dir/file_to_copy.txt' + assert cfile_body['description'] == 'Copied /my_mem_dir/test_dir/file1.txt to /my_mem_dir/file_to_copy.txt.' # Copy directory copy_dirpath = '/my_mem_dir/test_dir' @@ -222,7 +230,7 @@ async def test_action_same_fs_files(fs_manager_instance, jp_fetch): cdir_body_json = copy_dir_res.body.decode('utf-8') cdir_body = json.loads(cdir_body_json) assert cdir_body["status"] == 'success' - assert cdir_body['description'] == 'Copied /my_mem_dir/test_dir to /my_mem_dir/second_dir' + assert cdir_body['description'] == 'Copied /my_mem_dir/test_dir to /my_mem_dir/second_dir.' # Move file move_filepath = '/my_mem_dir/test_dir/file1.txt' @@ -245,9 +253,79 @@ async def test_action_same_fs_files(fs_manager_instance, jp_fetch): assert mdir_body["status"] == 'success' assert mdir_body['description'] == 'Moved /my_mem_dir/test_dir to /my_mem_dir/second_dir' -#TODO: Test xaction endpoint -# Q: Other fs types for testing purposes -async def xtest_xaction_diff_fs(fs_manager_instance_parameterized, jp_fetch): +# def get_boto3_client(): +# from botocore.session import Session + +# # NB: we use the sync botocore client for setup +# session = Session() + +# endpoint_uri = "http://127.0.0.1:%s/" % "5555" +# return session.create_client("s3", endpoint_url=endpoint_uri) + +@pytest.mark.asyncio +async def test_async_s3_file_operations(s3_client, s3_fs_manager_instance, jp_fetch): + # s3_client = get_boto3_client() + s3_client = s3_client + # boto3.set_stream_logger('botocore', level='DEBUG') + # s3_client.create_bucket(Bucket='my-test-bucket') + + # endpoint_uri = "http://127.0.0.1:%s/" % "5555" + + # fs = fsspec.filesystem('s3', asynchronous=True, anon=False, client_kwargs={'endpoint_url': endpoint_uri}) + fs = s3_fs_manager_instance + +# ================================================================== + # contents = await fs._ls('s3://my-test-bucket/') + # print(f"contents: {contents}") + # await fs._pipe_file('s3://my-test-bucket/test.txt', b"Hi Test MOTO server!") + # content = await fs._cat_file('s3://my-test-bucket/test.txt') + # print(content) + # assert content == b'Hi Test MOTO server!' + + # contents = await fs._ls('s3://my-test-bucket/') + # print(f"contents: {contents}") +# ================================================================== + fs_manager = s3_fs_manager_instance + print(f'fs_manager is: {fs_manager}') + + fs_info = fs_manager.get_filesystem_by_type('s3') + key = fs_info['key'] + fs = fs_info['info']['instance'] + item_path = fs_info['info']['path'] + assert fs != None + + # Read directory + assert fs.exists(item_path) == True + dir_response = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": key, "item_path": item_path}) + + assert dir_response.code == 200 + json_body = dir_response.body.decode('utf-8') + body = json.loads(json_body) + assert body['status'] == 'success' + +@pytest.mark.asyncio +async def xtest___async_s3_file_operations(mock_s3_fs): + # s3_client = boto3.client('s3', endpoint_url=os.getenv("AWS_S3_ENDPOINT_URL")) + s3_client = get_boto3_client() + # boto3.set_stream_logger('botocore', level='DEBUG') + s3_client.create_bucket(Bucket='my-test-bucket') + + endpoint_uri = "http://127.0.0.1:%s/" % "5555" + + fs = fsspec.filesystem('s3', asynchronous=True, anon=False, client_kwargs={'endpoint_url': endpoint_uri}) + + contents = await fs._ls('s3://my-test-bucket/') + print(f"contents: {contents}") + await fs._pipe_file('s3://my-test-bucket/test.txt', b"Hi Test MOTO server!") + content = await fs._cat_file('s3://my-test-bucket/test.txt') + print(content) + assert content == b'Hi Test MOTO server!' + + contents = await fs._ls('s3://my-test-bucket/') + print(f"contents: {contents}") + +#TODO: Test transfer endpoint +async def xtest_file_transfer(fs_manager_instance_parameterized, jp_fetch): fs_manager = fs_manager_instance_parameterized fs_info = fs_manager.get_filesystem_by_type('memory') key = fs_info['key'] @@ -258,7 +336,7 @@ async def xtest_xaction_diff_fs(fs_manager_instance_parameterized, jp_fetch): # # copy file # copy_filepath = f'{fs_root_path}/test_dir/file1.txt' # copy_file_payload = {"item_path": copy_filepath, "content": "/my_local_dir/", "destination_key": "", "action": "copy"} - # copy_file_res = await jp_fetch("jupyter_fsspec", "files", "xaction", method="POST", params={"key": mem_key}, body=json.dumps(copy_file_payload)) + # copy_file_res = await jp_fetch("jupyter_fsspec", "files", "transfer", method="POST", params={"key": mem_key}, body=json.dumps(copy_file_payload)) # cfile_body_json = copy_file_res.body.decode('utf-8') # cfile_body = json.loads(cfile_body_json) diff --git a/jupyter_fsspec/tests/unit/test_filesystem_manager.py b/jupyter_fsspec/tests/unit/test_filesystem_manager.py index 2c4fc6b..d2fb782 100644 --- a/jupyter_fsspec/tests/unit/test_filesystem_manager.py +++ b/jupyter_fsspec/tests/unit/test_filesystem_manager.py @@ -64,15 +64,11 @@ def test_key_decode_encode(config_file): 'path': str(config_file.parent) } + # TODO: update _encoded_key encoded_key = fs_test_manager._encode_key(fs_test_config) - decoded_type, decoded_path = fs_test_manager._decode_key(encoded_key) + decoded_name = fs_test_manager._decode_key(encoded_key) - # Ensure both paths are absolute by adding leading slash if missing - decoded_path = '/' + decoded_path.lstrip('/') # Normalize decoded path - fs_test_config['path'] = '/' + fs_test_config['path'].lstrip('/') # Normalize original path - - assert decoded_path == fs_test_config['path'] # Compare as strings - assert decoded_type == fs_test_config['type'] + assert decoded_name == fs_test_config['name'] # ============================================================ diff --git a/src/handler/fileOperations.ts b/src/handler/fileOperations.ts index 0757524..708617b 100644 --- a/src/handler/fileOperations.ts +++ b/src/handler/fileOperations.ts @@ -449,23 +449,11 @@ export class FsspecModel { } } - async listDirectory(key: string, item_path: string = '', type: string = ''): Promise { - const query = new URLSearchParams({ key, item_path, type }).toString(); - let result = null; - - Logger.debug(`[FSSpec] Fetching files -> ${query}`); - try { - result = await requestAPI(`fsspec?${query}`, { - method: 'GET' - }); - } catch (error) { - Logger.error(`[FSSpec] Failed to list filesystem ${error}: `); - } - - return result; - } - - async listDirectory_refactored(key: string, item_path: string = '', type: string = 'default'): Promise { + async listDirectory( + key: string, + item_path: string = '', + type: string = 'default' + ): Promise { const query = new URLSearchParams({ key, item_path, type }).toString(); let result = null; From edc455b30244c4c5db8a31f82209aaa8bb394bd7 Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Wed, 6 Nov 2024 12:56:02 -0500 Subject: [PATCH 04/13] Add helper module. --- jupyter_fsspec/__init__.py | 4 ++ jupyter_fsspec/exceptions.py | 4 ++ jupyter_fsspec/helper.py | 97 ++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 jupyter_fsspec/exceptions.py create mode 100644 jupyter_fsspec/helper.py diff --git a/jupyter_fsspec/__init__.py b/jupyter_fsspec/__init__.py index 83dd03c..a43a290 100644 --- a/jupyter_fsspec/__init__.py +++ b/jupyter_fsspec/__init__.py @@ -10,6 +10,10 @@ from .handlers import setup_handlers +# Global config manager for kernel-side jupyter-fsspec use +_manager = None + + def _jupyter_labextension_paths(): return [{ "src": "labextension", diff --git a/jupyter_fsspec/exceptions.py b/jupyter_fsspec/exceptions.py new file mode 100644 index 0000000..e302150 --- /dev/null +++ b/jupyter_fsspec/exceptions.py @@ -0,0 +1,4 @@ +"""Holds jupyter_fsspec exception base + any derived exceptions""" + + +class JupyterFsspecException(Exception): pass diff --git a/jupyter_fsspec/helper.py b/jupyter_fsspec/helper.py new file mode 100644 index 0000000..cb38e92 --- /dev/null +++ b/jupyter_fsspec/helper.py @@ -0,0 +1,97 @@ +# Gives users access to filesystems defined in the jupyter_fsspec config file + + +from urllib.parse import quote as urlescape # TODO refactor + +import jupyter_fsspec +from .file_manager import FileSystemManager +from .exceptions import JupyterFsspecException + + +# Global config manager for kernel-side jupyter-fsspec use +_manager = None +_active = None + + +def _get_manager(cached=True): + global _manager + if not cached or _manager is None: + _manager = FileSystemManager.create_default() + return _manager + + +def _get_fs(fs_name): + mgr = _get_manager() + fs = mgr.get_filesystem(urlescape(fs_name)) + if fs is not None and 'instance' in fs: + return fs['instance'] # TODO refactor + else: + raise JupyterFsspecException('Error, could not find specified filesystem') + + +def reload(): + return _get_manager(False) + + +def fs(fs_name): + return _get_fs(fs_name) + + +filesystem = fs # Alias for matching fsspec call + + +def work_on(fs_name): + global _active + fs = _get_fs(fs_name) + _active = fs + + return fs + + +def _get_active(): + return _active + + +def open(*args, **kwargs): + if not _active: + raise JupyterFsspecException('No active filesystem') + + fs = _get_active() + return fs.open(*args, **kwargs) + + +def bytes(*args, **kwargs): + if not _active: + raise JupyterFsspecException('No active filesystem') + + fs = _get_active() + kwargs['mode'] = 'rb' + + return fs.open(*args, **kwargs).read() + + +def utf8(*args, **kwargs): + if not _active: + raise JupyterFsspecException('No active filesystem') + + fs = _get_active() + kwargs['mode'] = 'r' + kwargs['encoding'] = 'utf8' + + return fs.open(*args, **kwargs).read() + + +def ls(*args, **kwargs): + if not _active: + raise JupyterFsspecException('No active filesystem') + + fs = _get_active() + return fs.ls(*args, **kwargs) + + +def stat(*args, **kwargs): + if not _active: + raise JupyterFsspecException('No active filesystem') + + fs = _get_active() + return fs.stat(*args, **kwargs) From 1455ae4b57f9678934b7e47aa5686f27449a0067 Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Wed, 6 Nov 2024 13:11:00 -0500 Subject: [PATCH 05/13] Add some comments. --- jupyter_fsspec/helper.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/jupyter_fsspec/helper.py b/jupyter_fsspec/helper.py index cb38e92..6c548a9 100644 --- a/jupyter_fsspec/helper.py +++ b/jupyter_fsspec/helper.py @@ -3,7 +3,6 @@ from urllib.parse import quote as urlescape # TODO refactor -import jupyter_fsspec from .file_manager import FileSystemManager from .exceptions import JupyterFsspecException @@ -14,6 +13,9 @@ def _get_manager(cached=True): + # Get and cache a manager: The manager handles the config and filesystem + # construction using the same underlying machinery used by the frontend extension. + # The manager is cached to avoid hitting the disk/config file multiple times. global _manager if not cached or _manager is None: _manager = FileSystemManager.create_default() @@ -21,6 +23,8 @@ def _get_manager(cached=True): def _get_fs(fs_name): + # Get an fsspec filesystem from the manager + # The fs_name is url encoded, we handle that here...TODO refactor that mgr = _get_manager() fs = mgr.get_filesystem(urlescape(fs_name)) if fs is not None and 'instance' in fs: @@ -30,10 +34,12 @@ def _get_fs(fs_name): def reload(): + # Get a new manager/re-read the config file return _get_manager(False) def fs(fs_name): + # (Public API) Return an fsspec filesystem from the manager return _get_fs(fs_name) @@ -41,6 +47,7 @@ def fs(fs_name): def work_on(fs_name): + # Set one of the named filesystems as "active" for use with convenience funcs below global _active fs = _get_fs(fs_name) _active = fs @@ -49,10 +56,12 @@ def work_on(fs_name): def _get_active(): + # Gets the "active" filesystem return _active def open(*args, **kwargs): + # Get a file handle if not _active: raise JupyterFsspecException('No active filesystem') @@ -61,6 +70,7 @@ def open(*args, **kwargs): def bytes(*args, **kwargs): + # Get bytes from the specified path if not _active: raise JupyterFsspecException('No active filesystem') @@ -71,6 +81,7 @@ def bytes(*args, **kwargs): def utf8(*args, **kwargs): + # Get utf8 text from the specified path (valid utf8 data is assumed) if not _active: raise JupyterFsspecException('No active filesystem') @@ -82,6 +93,7 @@ def utf8(*args, **kwargs): def ls(*args, **kwargs): + # Convenience/pass through call to fsspec ls if not _active: raise JupyterFsspecException('No active filesystem') @@ -90,6 +102,7 @@ def ls(*args, **kwargs): def stat(*args, **kwargs): + # Convenience/pass through call to fsspec stat if not _active: raise JupyterFsspecException('No active filesystem') From e14a08c35e5845bbbb345bdc8cdabaf939973584 Mon Sep 17 00:00:00 2001 From: RRosio Date: Thu, 7 Nov 2024 09:38:03 -0800 Subject: [PATCH 06/13] unstrip protocol from path --- jupyter_fsspec/file_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index d74fb4a..ec25eda 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -125,7 +125,7 @@ def initialize_filesystems(self): fs.mkdir(fs_path) # Store the filesystem instance - new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path} + new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path.unstrip_protocol()} except Exception as e: print(f'Error initializing filesystems: {e}') From 1fdbf08737aaadda4fc46949ec44844317665219 Mon Sep 17 00:00:00 2001 From: RRosio Date: Thu, 7 Nov 2024 09:47:19 -0800 Subject: [PATCH 07/13] pull path from fs --- jupyter_fsspec/file_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index ec25eda..cc6dd0c 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -123,9 +123,8 @@ def initialize_filesystems(self): if fs_type == 'memory': if not fs.exists(fs_path): fs.mkdir(fs_path) - # Store the filesystem instance - new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path.unstrip_protocol()} + new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs.unstrip_protocol(fs_path)} except Exception as e: print(f'Error initializing filesystems: {e}') From 305ea29b853db19209604ee935b18f8812bb0423 Mon Sep 17 00:00:00 2001 From: RRosio Date: Thu, 7 Nov 2024 09:59:00 -0800 Subject: [PATCH 08/13] return canonical path from server --- jupyter_fsspec/file_manager.py | 2 +- jupyter_fsspec/handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index cc6dd0c..bb7950f 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -124,7 +124,7 @@ def initialize_filesystems(self): if not fs.exists(fs_path): fs.mkdir(fs_path) # Store the filesystem instance - new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs.unstrip_protocol(fs_path)} + new_filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs._strip_protocol(fs_path), "canonical_path": fs.unstrip_protocol(fs_path)} except Exception as e: print(f'Error initializing filesystems: {e}') diff --git a/jupyter_fsspec/handlers.py b/jupyter_fsspec/handlers.py index f829a95..292259d 100644 --- a/jupyter_fsspec/handlers.py +++ b/jupyter_fsspec/handlers.py @@ -61,7 +61,7 @@ def get(self): for fs in self.fs_manager.filesystems: fs_info = self.fs_manager.filesystems[fs] - instance = {"key": fs, 'name': fs_info['name'], 'type': fs_info['type'], 'path': fs_info['path'] } + instance = {"key": fs, 'name': fs_info['name'], 'type': fs_info['type'], 'path': fs_info['path'], 'canonical_path': fs_info['canonical_path'] } file_systems.append(instance) self.set_status(200) self.write({'status': 'success', 'description': 'Retrieved available filesystems from configuration file.', 'content': file_systems}) From 5bde529b7258cf26564ad7a0f7c882983445cb3f Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Thu, 7 Nov 2024 14:50:58 -0500 Subject: [PATCH 09/13] Add canonical path to the context menu (WIP). --- src/FssFilesysItem.ts | 2 +- src/FssTreeItem.ts | 6 ++++-- src/index.ts | 4 ++-- src/treeContext.ts | 32 +++++++++++++++++++++----------- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/FssFilesysItem.ts b/src/FssFilesysItem.ts index aacd56d..4e70943 100644 --- a/src/FssFilesysItem.ts +++ b/src/FssFilesysItem.ts @@ -57,7 +57,7 @@ class FssFilesysItem { } // Make/add the context menu - let context = new FssContextMenu(); + let context = new FssContextMenu(null); context.root.dataset.fss = this.root.dataset.fss; let body = document.getElementsByTagName('body')[0]; body.appendChild(context.root); diff --git a/src/FssTreeItem.ts b/src/FssTreeItem.ts index 241463f..9d07087 100644 --- a/src/FssTreeItem.ts +++ b/src/FssTreeItem.ts @@ -8,6 +8,7 @@ import { Logger } from "./logger" export class FssTreeItem { root: any; + model: any; // icon: HTMLElement; nameLbl: HTMLElement; dirSymbol: HTMLElement; @@ -19,11 +20,12 @@ export class FssTreeItem { lazyLoadAutoExpand = true; clickAnywhereDoesAutoExpand = true; - constructor(clickSlots: any, autoExpand: boolean, expandOnClickAnywhere: boolean) { + constructor(model: any, clickSlots: any, autoExpand: boolean, expandOnClickAnywhere: boolean) { // The TreeItem component is the root and handles // tree structure functionality in the UI let root = new TreeItem(); this.root = root; + this.model = model; this.clickSlots = clickSlots; this.lazyLoadAutoExpand = autoExpand; this.clickAnywhereDoesAutoExpand = expandOnClickAnywhere; @@ -170,7 +172,7 @@ export class FssTreeItem { } // Make/add the context menu - let context = new FssContextMenu(); + let context = new FssContextMenu(this.model); context.root.dataset.fss = this.root.dataset.fss; let body = document.getElementsByTagName('body')[0]; body.appendChild(context.root); diff --git a/src/index.ts b/src/index.ts index 9d9607a..73c5f28 100644 --- a/src/index.ts +++ b/src/index.ts @@ -192,7 +192,7 @@ class FsspecWidget extends Widget { Logger.error(`Error fetching files for path ${source_path}`); // TODO jupyter info print return; } - // Logger.debug(`Response: (${JSON.stringify(response)})`); + Logger.debug(`Response: (${JSON.stringify(response)})`); // Get the dir tree node for this path (updates go into this subtree) let nodeForPath = this.getNodeForPath(source_path); @@ -256,7 +256,7 @@ class FsspecWidget extends Widget { // TODO: Create a placeholder child item for this dir } for (let [pathSegment, pathInfo] of Object.entries(childPaths)) { - let item = new FssTreeItem([this.lazyLoad.bind(this)], true, true); + let item = new FssTreeItem(this.model, [this.lazyLoad.bind(this)], true, true); item.setMetadata((pathInfo as any).path); item.setText(pathSegment); // (pathInfo as any).ui = item; diff --git a/src/treeContext.ts b/src/treeContext.ts index dbad6df..b9633fc 100644 --- a/src/treeContext.ts +++ b/src/treeContext.ts @@ -3,11 +3,13 @@ export class FssContextMenu { root: any; clicked = false; + model: any; - constructor() { + constructor(model: any) { let root = document.createElement('div'); root.classList.add('jfss-tree-context-menu'); this.root = root; + this.model = model; let menuItem = document.createElement('div'); menuItem.classList.add('jfss-tree-context-item'); @@ -23,16 +25,24 @@ export class FssContextMenu { handleItemClick(event: any) { // TODO multiple menu it if (event.target.dataset.fssContextType == 'copyPath') { - navigator.clipboard.writeText(this.root.dataset.fss).then( - () => { // Success - console.log('Copy path: ' + this.root.dataset.fss); - this.root.remove(); - }, - () => { - console.log('Copy path failed: ' + this.root.dataset.fss); - this.root.remove(); - }, - ); + let info = this.model.getActiveFilesystemInfo(); + let protocol = info?.canonical_path.slice( + 0, + info.canonical_path.length - info.path.length, + ) + if (protocol) { + let canonical = protocol + '/' + this.root.dataset.fss.replace(/^\/+/, () => ''); + navigator.clipboard.writeText(canonical).then( + () => { // Success + console.log('Copy path: ' + canonical); + this.root.remove(); + }, + () => { + console.log('Copy path failed: ' + canonical); + this.root.remove(); + }, + ); + } } } From 2d252c8e079d6a96c1038c4a987059ac8c8050f2 Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Thu, 7 Nov 2024 15:02:40 -0500 Subject: [PATCH 10/13] Fix right click on config/filesystem list items. --- src/FssFilesysItem.ts | 6 ++++-- src/index.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/FssFilesysItem.ts b/src/FssFilesysItem.ts index 4e70943..d6576c4 100644 --- a/src/FssFilesysItem.ts +++ b/src/FssFilesysItem.ts @@ -4,6 +4,7 @@ import { FssContextMenu } from './treeContext'; class FssFilesysItem { root: HTMLElement; + model: any; filesysName: string; filesysType: string; fsInfo: any; @@ -11,7 +12,8 @@ class FssFilesysItem { nameField: any; typeField: any; - constructor(fsInfo: any, userClickSlots: any) { + constructor(model: any, fsInfo: any, userClickSlots: any) { + this.model = model; this.filesysName = fsInfo.name; this.filesysType = fsInfo.type; this.fsInfo = fsInfo; @@ -57,7 +59,7 @@ class FssFilesysItem { } // Make/add the context menu - let context = new FssContextMenu(null); + let context = new FssContextMenu(this.model); context.root.dataset.fss = this.root.dataset.fss; let body = document.getElementsByTagName('body')[0]; body.appendChild(context.root); diff --git a/src/index.ts b/src/index.ts index 73c5f28..9e56b74 100644 --- a/src/index.ts +++ b/src/index.ts @@ -140,7 +140,7 @@ class FsspecWidget extends Widget { } addFilesystemItem(fsInfo: any) { - let fsItem = new FssFilesysItem(fsInfo, [this.handleFilesystemClicked.bind(this)]); + let fsItem = new FssFilesysItem(this.model, fsInfo, [this.handleFilesystemClicked.bind(this)]); fsItem.setMetadata(fsInfo.path); this.filesysContainer.appendChild(fsItem.root); } From 70e416397cb0b6d668efaf7ebd9f53ebe22c6d6a Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Thu, 7 Nov 2024 16:19:09 -0500 Subject: [PATCH 11/13] Added size label. --- src/FssTreeItem.ts | 22 ++++++++++++++++++++-- src/index.ts | 2 +- style/base.css | 5 +++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/FssTreeItem.ts b/src/FssTreeItem.ts index 9d07087..94eee37 100644 --- a/src/FssTreeItem.ts +++ b/src/FssTreeItem.ts @@ -11,6 +11,7 @@ export class FssTreeItem { model: any; // icon: HTMLElement; nameLbl: HTMLElement; + sizeLbl: HTMLElement; dirSymbol: HTMLElement; container: HTMLElement; clickSlots: any; @@ -60,6 +61,12 @@ export class FssTreeItem { container.appendChild(nameLbl); this.nameLbl = nameLbl; + // Show the name of this file/folder (a single path segment) + let sizeLbl = document.createElement('div'); + sizeLbl.classList.add('jfss-filesize-lbl'); + container.appendChild(sizeLbl); + this.sizeLbl = sizeLbl; + // Add click and right click handlers to the tree component root.addEventListener('contextmenu', this.handleContext.bind(this)); root.addEventListener('click', this.handleClick.bind(this), true); @@ -74,8 +81,18 @@ export class FssTreeItem { this.root.appendChild(elem); } - setMetadata(value: string) { - this.root.dataset.fss = value; + setMetadata(user_path: string, size: string) { + this.root.dataset.fss = user_path; + this.root.dataset.fsize = size; + + let sizeDisplay = `(${size.toLocaleString()})`; + // if (parseInt(size) > 100) { + // const sizeFormat = new Intl.NumberFormat(undefined, { + // notation: 'scientific', + // }); + // sizeDisplay = `(${sizeFormat.format(parseInt(size))})`; + // } + this.sizeLbl.innerText = sizeDisplay; } setText(value: string) { @@ -89,6 +106,7 @@ export class FssTreeItem { if (symbol == 'dir') { folderIcon.element({container: this.dirSymbol}); this.isDir = true; + this.sizeLbl.style.display = 'none'; } if (symbol == 'file') { fileIcon.element({container: this.dirSymbol}); diff --git a/src/index.ts b/src/index.ts index 9e56b74..7242042 100644 --- a/src/index.ts +++ b/src/index.ts @@ -257,7 +257,7 @@ class FsspecWidget extends Widget { } for (let [pathSegment, pathInfo] of Object.entries(childPaths)) { let item = new FssTreeItem(this.model, [this.lazyLoad.bind(this)], true, true); - item.setMetadata((pathInfo as any).path); + item.setMetadata((pathInfo as any).path, (pathInfo as any).metadata.size); item.setText(pathSegment); // (pathInfo as any).ui = item; elemParent.appendChild(item.root); diff --git a/style/base.css b/style/base.css index d35cbc7..082420c 100644 --- a/style/base.css +++ b/style/base.css @@ -111,6 +111,11 @@ display: flex; } +.jfss-filesize-lbl { + margin-left: .5rem; + color: var(--jp-ui-font-color3) +} + .jfss-hseparator { box-sizing: border-box; From 89d7bc0876019defa0907a9f61000692284480a8 Mon Sep 17 00:00:00 2001 From: Eric Gentry Date: Fri, 8 Nov 2024 13:27:31 -0500 Subject: [PATCH 12/13] Added selection indicator to sources list, UI tweaks. --- src/FssFilesysItem.ts | 50 +++++++++++++++++++++++++++++++++++-------- src/index.ts | 32 ++++++++++++++++++++++----- style/base.css | 7 +++--- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/src/FssFilesysItem.ts b/src/FssFilesysItem.ts index aacd56d..c791a7b 100644 --- a/src/FssFilesysItem.ts +++ b/src/FssFilesysItem.ts @@ -1,6 +1,11 @@ // Element for displaying a single fsspec filesystem import { FssContextMenu } from './treeContext'; +// import { Logger } from "./logger" + +const HOVER = 'var(--jp-layout-color3)'; +const UNHOVER = 'var(--jp-layout-color2)'; +const SELECTED = 'var(--jp-layout-color4)'; class FssFilesysItem { root: HTMLElement; @@ -9,7 +14,9 @@ class FssFilesysItem { fsInfo: any; clickSlots: any; nameField: any; - typeField: any; + pathField: any; + _selected = false; + _hovered = false; constructor(fsInfo: any, userClickSlots: any) { this.filesysName = fsInfo.name; @@ -25,6 +32,7 @@ class FssFilesysItem { fsItem.classList.add('jfss-fsitem-root'); fsItem.addEventListener('mouseenter', this.handleFsysHover.bind(this)); fsItem.addEventListener('mouseleave', this.handleFsysHover.bind(this)); + fsItem.dataset.fssname = fsInfo.name; this.root = fsItem; // Set the tooltip @@ -35,10 +43,10 @@ class FssFilesysItem { this.nameField.innerText = this.filesysName; fsItem.appendChild(this.nameField); - this.typeField = document.createElement('div'); - this.typeField.classList.add('jfss-fsitem-type'); - this.typeField.innerText = 'Type: ' + this.filesysType; - fsItem.appendChild(this.typeField); + this.pathField = document.createElement('div'); + this.pathField.classList.add('jfss-fsitem-type'); + this.pathField.innerText = 'Path: ' + fsInfo.path; + fsItem.appendChild(this.pathField); fsItem.addEventListener('click', this.handleClick.bind(this)); fsItem.addEventListener('contextmenu', this.handleContext.bind(this)); @@ -91,18 +99,42 @@ class FssFilesysItem { this.root.dataset.fss = value; } + set selected(value: boolean) { + this._selected = value; + if (value) { + this.root.style.backgroundColor = SELECTED; + } + else { + this.hovered = this._hovered; + } + } + + set hovered(state: boolean) { + this._hovered = state; + if (this._selected) { + this.root.style.backgroundColor = SELECTED; + } + else { + if (state) { + this.root.style.backgroundColor = HOVER; + } + else { + this.root.style.backgroundColor = UNHOVER; + } + } + } + handleFsysHover(event: any) { if (event.type == 'mouseenter') { - this.root.style.backgroundColor = 'var(--jp-layout-color3)'; - this.root.style.backgroundColor = 'var(--jp-layout-color3)'; + this.hovered = true; } else { - this.root.style.backgroundColor = 'var(--jp-layout-color2)'; - this.root.style.backgroundColor = 'var(--jp-layout-color2)'; + this.hovered = false; } } handleClick(_event: any) { + this.selected = true; for (const slot of this.clickSlots) { slot(this.fsInfo); } diff --git a/src/index.ts b/src/index.ts index 9d9607a..7684356 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,7 +43,8 @@ class FsspecWidget extends Widget { detailName: any; detailPath: any; treeView: any; - elementHeap: any = {}; + elementHeap: any = {}; // Holds FssTreeItem's keyed by path + sourcesHeap: any = {}; // Holds FssFilesysItem's keyed by name filesysContainer: any; dirTree: any = {}; @@ -96,10 +97,10 @@ class FsspecWidget extends Widget { let lowerArea = document.createElement('div'); lowerArea.classList.add('jfss-lowerarea'); - let browserAreaLabel = document.createElement('div'); - browserAreaLabel.classList.add('jfss-browseAreaLabel'); - browserAreaLabel.innerText = 'Browse Filesystem'; - lowerArea.appendChild(browserAreaLabel); + // let browserAreaLabel = document.createElement('div'); + // browserAreaLabel.classList.add('jfss-browseAreaLabel'); + // browserAreaLabel.innerText = 'Browse Filesystem'; + // lowerArea.appendChild(browserAreaLabel); this.selectedFsLabel = document.createElement('div'); this.selectedFsLabel.classList.add('jfss-selectedFsLabel'); @@ -122,6 +123,7 @@ class FsspecWidget extends Widget { } async fetchConfig() { + this.selectedFsLabel.innerText = '