diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..2a643fd --- /dev/null +++ b/conftest.py @@ -0,0 +1,91 @@ +from pathlib import Path +from unittest.mock import patch +import pytest +import fsspec +from jupyter_fsspec.file_manager import FileSystemManager + + +pytest_plugins = ['pytest_jupyter.jupyter_server', 'jupyter_server.pytest_plugin', + 'pytest_asyncio'] + +@pytest.fixture(scope='function', autouse=True) +def setup_config_file(tmp_path: Path): + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + + yaml_content = """sources: + - name: "TestSourceAWS" + path: "/path/to/set1" + type: "s3" + additional_options: + anon: false + key: "my-access-key" + secret: "my-secret-key" + - name: "TestSourceDisk" + path: "." + type: "local" + - name: "TestDir" + path: "/Users/rosioreyes/Desktop/test_fsspec" + type: "local" + - name: "TestEmptyLocalDir" + path: "/Users/rosioreyes/Desktop/notebooks/sample/nothinghere" + type: "local" + - name: "TestMem Source" + path: "/my_mem_dir" + type: "memory" + - name: "TestDoesntExistDir" + path: "/Users/rosioreyes/Desktop/notebooks/doesnotexist" + type: "local" + """ + yaml_file = config_dir / "jupyter-fsspec.yaml" + yaml_file.write_text(yaml_content) + + with patch('jupyter_core.paths.jupyter_config_dir', return_value=str(config_dir)): + print(f"Patching jupyter_config_dir to: {config_dir}") + yield + + +@pytest.fixture(scope='function') +def fs_manager_instance(setup_config_file): + fs_manager = FileSystemManager(config_file='jupyter-fsspec.yaml') + fs_info = fs_manager.get_filesystem_by_type('memory') + key = fs_info['key'] + fs = fs_info['info']['instance'] + mem_root_path = fs_info['info']['path'] + + if fs: + if fs.exists('/my_mem_dir/test_dir'): + fs.rm('/my_mem_dir/test_dir', recursive=True) + if fs.exists('/my_mem_dir/second_dir'): + fs.rm('/my_mem_dir/second_dir', recursive=True) + + fs.touch('/my_mem_dir/file_in_root.txt') + with fs.open('/my_mem_dir/file_in_root.txt', 'wb') as f: + f.write("Root file content".encode()) + + fs.mkdir('/my_mem_dir/test_dir', exist_ok=True) + fs.mkdir('/my_mem_dir/second_dir', exist_ok=True) + # fs.mkdir('/my_mem_dir/second_dir/subdir', exist_ok=True) + fs.touch('/my_mem_dir/test_dir/file1.txt') + with fs.open('/my_mem_dir/test_dir/file1.txt', "wb") as f: + f.write("Test content".encode()) + f.close() + else: + print("In memory filesystem NOT FOUND") + + if fs.exists('/my_mem_dir/test_dir/file1.txt'): + file_info = fs.info('/my_mem_dir/test_dir/file1.txt') + print(f"File exists. size: {file_info}") + else: + print("File does not exist!") + return fs_manager + +@pytest.fixture +def jp_server_config(jp_server_config): + return { + "ServerApp": { + "jpserver_extensions": { + "jupyter_fsspec": True + } + } + } diff --git a/jupyter_fsspec/file_manager.py b/jupyter_fsspec/file_manager.py index 3ac8768..188f9e9 100644 --- a/jupyter_fsspec/file_manager.py +++ b/jupyter_fsspec/file_manager.py @@ -3,14 +3,15 @@ import os import yaml import urllib.parse +from datetime import datetime from pathlib import PurePath class FileSystemManager: def __init__(self, config_file): base_dir = jupyter_config_dir() - config_path = os.path.join(base_dir, config_file) + self.config_path = os.path.join(base_dir, config_file) try: - with open(config_path, 'r') as file: + with open(self.config_path, 'r') as file: self.config = yaml.safe_load(file) except Exception as e: print(f"Error loading configuration file: {e}") @@ -22,17 +23,26 @@ def __init__(self, config_file): def _encode_key(self, fs_config): fs_path = fs_config['path'].strip('/') - combined = f"{fs_config['type']}|{fs_config['name']}|{fs_path}" + combined = f"{fs_config['type']}|{fs_path}" encoded_key = urllib.parse.quote(combined, safe='') return encoded_key - #TODO: verify def _decode_key(self, encoded_key): combined = urllib.parse.unquote(encoded_key) - fs_type, fs_name, fs_path = combined.split('|', 2) - return fs_type, fs_name, fs_path + fs_type, fs_path = combined.split('|', 1) + return fs_type, fs_path + + def read_config(self): + try: + with open(self.config_path, 'r') as file: + self.config = yaml.safe_load(file) + except Exception as e: + print(f"Error loading configuration file: {e}") + return None def _initialize_filesystems(self): + self.read_config() + for fs_config in self.config['sources']: key = self._encode_key(fs_config) @@ -43,35 +53,58 @@ def _initialize_filesystems(self): # Init filesystem fs = fsspec.filesystem(fs_type, **options) + if fs_type == 'memory': + if not fs.exists(fs_path): + fs.mkdir(fs_path) # Store the filesystem instance self.filesystems[key] = {"instance": fs, "name": fs_name, "type": fs_type, "path": fs_path} + def get_all_filesystems(self): + self._initialize_filesystems() + def get_filesystem(self, key): return self.filesystems.get(key) + + def get_filesystem_by_type(self, fs_type): + for encoded_key, fs_info in self.filesystems.items(): + if fs_info.get('type') == fs_type: + return {'key': encoded_key, 'info': fs_info} + return None # =================================================== # File/Folder Read/Write Operations # =================================================== - def write(self, key, item_path: str, content): # writePath + # write directory + # write file with content + # write empty file at directory + # write to an existing file + def write(self, key, item_path: str, content, overwrite=False): # writePath fs_obj = self.get_filesystem(key) fs = fs_obj['instance'] - if fs.exists(item_path) and not fs.isdir(item_path): - return {"status_code": 409, "status": f"Failed: Path {item_path} already exists."} - if fs.isdir(item_path): - content = content.decode('utf-8') + if overwrite: + return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} + if isinstance(content, bytes): + content = content.decode('utf-8') new_dir_path = str(PurePath(item_path) / content) + '/' - print(f"new_dir_path is: {new_dir_path}") + if fs.exists(new_dir_path): - return {"status_code": 409, "status": f"Failed: Path {item_path} already exists."} - fs.mkdir(new_dir_path, create_parents=True) - return {"status_code": 200, "response": {"status": "success", "description": f"Wrote {new_dir_path}."}} + return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} + else: + fs.mkdir(new_dir_path, create_parents=True) + return {"status_code": 200, "response": {"status": "success", "description": f"Wrote {new_dir_path}"}} else: # TODO: Process content for different mime types correctly - with fs.open(item_path, 'wb') as file: - file.write(content); + if not isinstance(content, bytes): + content = content.encode() + + if fs.exists(item_path) and not overwrite: + return {"status_code": 409, "response": {"status": "failed", "description": f"Failed: Path {item_path} already exists."}} + else: + with fs.open(item_path, 'wb') as file: + file.write(content); return {"status_code": 200, "response": {"status": "success", "description": f"Wrote {item_path}"}} @@ -83,15 +116,23 @@ def read(self, key, item_path: str, find: bool = False): # readPath return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} if fs.isdir(item_path) and find: + # find(): a simple list of files content = [] dir_ls = fs.find(item_path, maxdepth=None, withdirs=True, detail=False) for path in dir_ls: - content.append(fs.info(path)) + content.append(path) elif fs.isdir(item_path): content = [] dir_ls = fs.ls(item_path) for path in dir_ls: - content.append(fs.info(path)) + if not isinstance(path, str): #TODO: improve + path = path['name'] + + info = fs.info(path) + + if isinstance(info.get('created'), datetime): + info['created'] = info['created'].isoformat() + content.append(info) else: with fs.open(item_path, 'rb') as file: content = file.read() @@ -99,14 +140,13 @@ def read(self, key, item_path: str, find: bool = False): # readPath # TODO: Process content for different mime types for request body eg. application/json return {"status_code": 200, "response": {"status": "success", "description": f"Read {item_path}", "content": content}} - # TODO: + # TODO: remove def accessMemoryFS(self, key, item_path): fs_obj = self.get_filesystem(key) fs = fs_obj['instance'] content = 'Temporary Content: memory fs accessed' return {"status_code": 200, "response": {"status": "success", "description": f"Read {item_path}", "content": content}} - def update(self, key, item_path, content): #updateFile fs_obj = self.get_filesystem(key) fs = fs_obj['instance'] @@ -128,7 +168,7 @@ def delete(self, key, item_path): # deletePath return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} if fs.isdir(item_path): - fs.delete(item_path, recursive=True) #TODO: await fs._rm() + fs.delete(item_path) #TODO: await fs._rm() Do not want recursive=True else: fs.delete(item_path, recursive=False) return {"status_code": 200, "response": {"status": "success", "description": f"Deleted {item_path}."}} @@ -136,30 +176,68 @@ def delete(self, key, item_path): # deletePath def move(self, key, item_path, dest_path): # movePath fs_obj = self.get_filesystem(key) fs = fs_obj['instance'] - if not fs.exists(item_path): return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} if fs.isdir(item_path): - print(f"directory") fs.mv(item_path, dest_path, recursive=True) else: - print(f"file") + _, item_extension = os.path.splitext(item_path) + _, dest_extension = os.path.splitext(dest_path) + + if not dest_extension: + dest_path = dest_path + item_extension + fs.mv(item_path, dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Moved {item_path} to path: {dest_path}."}} + return {"status_code": 200, "response": {"status": "success", "description": f"Moved {item_path} to {dest_path}"}} + def move_diff_fs(self, key, full_item_path, dest_key, full_dest_path): # movePath + fs_obj = self.get_filesystem(key) + fs = fs_obj['instance'] + dest_fs_obj = self.get_filesystem(dest_key) + dest_fs =dest_fs_obj['instance'] + + if not fs.exists(full_item_path): + return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_item_path} does not exist"}} + + if fs.isdir(full_item_path): + if not dest_fs.exists(full_dest_path): + return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_dest_path} does not exist"}} + fsspec.mv(full_item_path, full_dest_path, recursive=True) + else: + fsspec.mv(full_item_path, full_dest_path, recursive=False) + return {"status_code": 200, "response": {"status": "success", "description": f"Moved {full_item_path} to path: {full_dest_path}"}} + def copy(self, key, item_path, dest_path): # copyPath fs_obj = self.get_filesystem(key) fs = fs_obj['instance'] if not fs.exists(item_path): - return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist."}} + return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist"}} if fs.isdir(item_path): fs.copy(item_path, dest_path, recursive=True) else: + _, item_extension = os.path.splitext(item_path) + _, dest_extension = os.path.splitext(dest_path) + + if not dest_extension: + dest_path = dest_path + item_extension fs.copy(item_path, dest_path, recursive=False) - return {"status_code": 200, "response": {"status": "success", "description": f"Copied {item_path} to path: {dest_path}."}} + return {"status_code": 200, "response": {"status": "success", "description": f"Copied {item_path} to {dest_path}"}} + + def copy_diff_fs(self, key, full_item_path, dest_key, full_dest_path): # copyPath + fs_obj = self.get_filesystem(key) + fs = fs_obj['instance'] + + if not fs.exists(full_item_path): + return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {full_item_path} does not exist"}} + + if fs.isdir(full_item_path): + fs.copy(full_item_path, full_dest_path, recursive=True) + else: + fs.copy(full_item_path, full_dest_path, recursive=False) + return {"status_code": 200, "response": {"status": "success", "description": f"Copied {full_item_path} to path: {full_dest_path}"}} def open(self, key, item_path, start, end): fs_obj = self.get_filesystem(key) @@ -177,6 +255,36 @@ def open(self, key, item_path, start, end): content = data.decode('utf-8') return {"status_code": 206, "response": {"status": "success", "description": f"Partial content read from: {item_path}", "content": content}} + + def rename(self, key, item_path, dest_path): + fs_obj = self.get_filesystem(key) + fs = fs_obj['instance'] + + if not fs.exists(item_path): + return {"status_code": 404, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {item_path} does not exist"}} + + dir_root_path = os.path.dirname(item_path) + + # directory + if fs.isdir(item_path): + new_dest_path = dir_root_path + '/' + dest_path + if fs.exists(new_dest_path): + return {"status_code": 403, "response": {"status": "failed", "error": "PATH_NOT_FOUND", "description": f"Path {new_dest_path} already exist"}} + else: + fs.rename(item_path, new_dest_path) + # file + else: + # check for dest_path file extension? if not infer, reassign dest_path + _, item_extension = os.path.splitext(item_path) + _, dest_extension = os.path.splitext(dest_path) + + if not dest_extension: + dest_path = dest_path + item_extension + new_dest_path = dir_root_path + '/' + dest_path + fs.rename(item_path, new_dest_path) + + return {"status_code": 200, "response": {"status": "success", "description": f"Renamed {item_path} to {new_dest_path}"}} + # =================================================== # File/Folder Management Operations # =================================================== diff --git a/jupyter_fsspec/handlers.py b/jupyter_fsspec/handlers.py index 3c7347a..c083e1d 100644 --- a/jupyter_fsspec/handlers.py +++ b/jupyter_fsspec/handlers.py @@ -6,7 +6,44 @@ from .file_manager import FileSystemManager from .utils import parse_range -fs_manager = FileSystemManager('jupyter-fsspec.yaml') + +def create_filesystem_manager(): + return FileSystemManager(config_file='jupyter-fsspec.yaml') + +class BaseFileSystemHandler(APIHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + + def validate_fs(self, request_type): + """Retrieve the filesystem instance and path of the item + + :raises [ValueError]: [Missing required key parameter] + :raises [ValueError]: [Missing required parameter item_path] + :raises [ValueError]: [No filesystem identified for provided key] + + :return: filesystem instance and item_path + :rtype: fsspec filesystem instance and string + """ + key = self.get_argument('key', None) + + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + + if not key: + raise ValueError("Missing required parameter `key`") + + fs = self.fs_manager.get_filesystem(key) + + if not item_path: + if type != 'range' and request_type == 'get': + item_path = self.fs_manager.filesystems[key]["path"] + else: + raise ValueError("Missing required parameter `item_path`") + + if fs is None: + raise ValueError(f"No filesystem found for key: {key}") + + return fs, item_path class FsspecConfigHandler(APIHandler): """ @@ -14,6 +51,9 @@ class FsspecConfigHandler(APIHandler): Args: APIHandler (_type_): _description_ """ + def initialize(self, fs_manager): + self.fs_manager = fs_manager + @tornado.web.authenticated def get(self): """Retrieve filesystems information from configuration file. @@ -23,8 +63,8 @@ def get(self): """ try: file_systems = []; - for fs in fs_manager.filesystems: - fs_info = fs_manager.filesystems[fs] + for fs in self.fs_manager.filesystems: + fs_info = self.fs_manager.filesystems[fs] instance = {"key": fs, 'name': fs_info['name'], 'type': fs_info['type'], 'path': fs_info['path'] } file_systems.append(instance) @@ -38,6 +78,9 @@ def get(self): self.finish() class FileSystemHandler(APIHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + @tornado.web.authenticated def get(self): """Retrieve list of files for directories or contents for files. @@ -63,20 +106,18 @@ def get(self): # if not item_path: # raise ValueError("Missing required parameter `item_path`") - fs = fs_manager.get_filesystem(key) + fs = self.fs_manager.get_filesystem(key) fs_type = fs['type'] - - # TODO: if fs_type == 'memory': print (f"accessed memory filesystem") - result = fs_manager.accessMemoryFS(key, item_path) + result = self.fs_manager.accessMemoryFS(key, item_path) self.set_status(result['status_code']) self.finish(result['response']) return if not item_path: if type != 'range': - item_path = fs_manager.filesystems[key]["path"] + item_path = self.fs_manager.filesystems[key]["path"] else: raise ValueError("Missing required parameter `item_path`") @@ -84,18 +125,18 @@ def get(self): raise ValueError(f"No filesystem found for key: {key}") if type == 'find': - result = fs_manager.read(key, item_path, find=True) + result = self.fs_manager.read(key, item_path, find=True) elif type == 'range': # add check for Range specific header range_header = self.request.headers.get('Range') start, end = parse_range(range_header) - result = fs_manager.open(key, item_path, start, end) + result = self.fs_manager.open(key, item_path, start, end) self.set_status(result["status_code"]) self.set_header('Content-Range', f'bytes {start}-{end}') self.finish(result['response']) return else: - result = fs_manager.read(key, item_path) + result = self.fs_manager.read(key, item_path) self.set_status(result["status_code"]) self.write(result['response']) @@ -107,7 +148,6 @@ def get(self): self.write({"status": "failed", "error": "ERROR_REQUESTING_READ", "description": f"Error occurred: {str(e)}"}) self.finish() - #TODO: add actions: write, move, copy (separate functions) @tornado.web.authenticated def post(self): """Create directories/files or perform other directory/file operations like move and copy @@ -126,7 +166,6 @@ def post(self): """ try: action = self.get_argument('action') - print(f"action is: {action}") request_data = json.loads(self.request.body.decode('utf-8')) key = request_data.get('key') @@ -137,24 +176,22 @@ def post(self): content = request_data.get('content').encode('utf-8') - fs = fs_manager.get_filesystem(key) + fs = self.fs_manager.get_filesystem(key) if fs is None: raise ValueError(f"No filesystem found for key: {key}") if action == 'move': - print(f"move") src_path = item_path dest_path = content.decode('utf-8') - print(f"dest_path is: {dest_path}") - if not fs_manager.exists(key, dest_path): + if not self.fs_manager.exists(key, dest_path): raise ValueError('Required parameter `content` is not a valid destination path for move action.') else: - fs_manager.move(key, src_path, content) + self.fs_manager.move(key, src_path, content) result = {"status_code": 200, "status": "success!"} elif action == 'copy': - print('copy') + result = {"status_code": 200, "status": "TBD"} else: # assume write - result = fs_manager.write(key, item_path, content) + result = self.fs_manager.write(key, item_path, content) self.set_status(result["status_code"]) self.write(result['response']) @@ -190,11 +227,11 @@ def put(self): content = request_data.get('content') - fs = fs_manager.get_filesystem(key) + fs = self.fs_manager.get_filesystem(key) if fs is None: raise ValueError(f"No filesystem found for key: {key}") - result = fs_manager.update(key, item_path, content) + result = self.fs_manager.update(key, item_path, content) self.set_status(result["status_code"]) self.write(result['response']) @@ -226,11 +263,11 @@ def delete(self): if not (key) or not (item_path): raise ValueError("Missing required parameter `key` or `item_path`") - fs = fs_manager.get_filesystem(key) + fs = self.fs_manager.get_filesystem(key) if fs is None: raise ValueError(f"No filesystem found for key: {key}") - result = fs_manager.delete(key, item_path) + result = self.fs_manager.delete(key, item_path) self.set_status(result["status_code"]) self.write(result['response']) self.finish() @@ -243,14 +280,273 @@ def delete(self): self.write({"status": "failed", "error": "ERROR_REQUESTING_DELETE" , "description": f"Error occurred: {str(e)}"}) self.finish() +class FileActionHandler(BaseFileSystemHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + + # POST /jupyter_fsspec/files/action?key=my-key&item_path=/some_directory/file.txt + def post(self): + """Move or copy the resource at the input path to destination path. + + :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved] + :param [action]: [Request body string move or copy] + :param [content]: [Request body property file or directory path] + + :return: dict with a status, description and (optionally) error + :rtype: dict + """ + key = self.get_argument('key') + # item_path = self.get_argument('item_path') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + action = request_data.get('action') + destination = request_data.get('content') + + fs, item_path = self.validate_fs('post') + + if action == 'move': + result = self.fs_manager.move(key, item_path, destination) + elif action == 'copy': + result = self.fs_manager.copy(key, item_path, destination) + else: + result = {"status_code": 400, "response": {"status": "failed", "error": "INVALID_ACTION", "description": f"Unsupported action: {action}"}} + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + +class FileActionCrossFSHandler(BaseFileSystemHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + + # POST /jupyter_fsspec/files/action?key=my-key&item_path=/some_directory/file.txt + def post(self): + """Move or copy the resource at the input path to destination path. + + :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved] + :param [action]: [Request body string move or copy] + :param [content]: [Request body property file or directory path] + + :return: dict with a status, description and (optionally) error + :rtype: dict + """ + key = self.get_argument('key') + # item_path = self.get_argument('item_path') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + action = request_data.get('action') + destination = request_data.get('content') + dest_fs_key = request_data.get('destination_key') + + fs, item_path = self.validate_fs('post') + + if action == 'move': + result = self.fs_manager.move_diff_fs(key, item_path, dest_fs_key, destination) + elif action == 'copy': + result = self.fs_manager.copy_diff_fs(key, item_path, dest_fs_key, destination) + else: + result = {"status_code": 400, "response": {"status": "failed", "error": "INVALID_ACTION", "description": f"Unsupported action: {action}"}} + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + +class RenameFileHandler(BaseFileSystemHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + + def post(self): + key = self.get_argument('key') + type = self.get_argument('type', default='default') + + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + content = request_data.get('content') + + fs, item_path = self.validate_fs('post') + result = self.fs_manager.rename(key, item_path, content) + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + + +class FileSysHandler(BaseFileSystemHandler): + def initialize(self, fs_manager): + self.fs_manager = fs_manager + + # GET + # /files + def get(self): + """Retrieve list of files for directories or contents for files. + + :param [key]: [Query arg string corresponding to the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved], defaults to [root path of the active filesystem] + :param [type]: [Optional query arg identifying the type of directory search or file content retrieval + if type is "find" recursive files/directories listed; + if type is "range", returns specified byte range content; + defaults to "default" for one level deep directory contents and single file entire contents] + + :return: dict with a status, description and content/error + content being a list of files, file information + :rtype: dict + """ + # GET /jupyter_fsspec/files?key=my-key&item_path=/some_directory/of_interest + # GET /jupyter_fsspec/files?key=my-key + # item_path: /some_directory/file.txt + # GET /jupyter_fsspec/files?key=my-key&item_path=/some_directory/file.txt&type=range + # content header specifying the byte range + try: + key = self.get_argument('key') + type = self.get_argument('type', default='default') + + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + fs, item_path = self.validate_fs('get') + if type == 'find': + result = self.fs_manager.read(key, item_path, find=True) + elif type == 'range': # add check for Range specific header + range_header = self.request.headers.get('Range') + start, end = parse_range(range_header) + + result = self.fs_manager.open(key, item_path, start, end) + self.set_status(result["status_code"]) + self.set_header('Content-Range', f'bytes {start}-{end}') + self.finish(result['response']) + return + else: + result = self.fs_manager.read(key, item_path) + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + return + except Exception as e: + print("Error requesting read: ", e) + self.set_status(500) + self.send_response({"status": "failed", "error": "ERROR_REQUESTING_READ", "description": f"Error occurred: {str(e)}"}) + + # POST /jupyter_fsspec/files?key=my-key + # JSON Payload + # item_path=/some_directory/file.txt + # content + def post(self): + """Create directories/files or perform other directory/file operations like move and copy + + :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved] + :param [content]: [Request body property file content, or directory name] + + :return: dict with a status, description and (optionally) error + :rtype: dict + """ + key = self.get_argument('key') + # item_path = self.get_argument('item_path') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + content = request_data.get('content') + + fs, item_path = self.validate_fs('post') + + result = self.fs_manager.write(key, item_path, content) + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + + # PUT /jupyter_fsspec/files?key=my-key&item_path=/some_directory/file.txt + # JSON Payload + # content + def put(self): + """Update ENTIRE content in file. + + :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved] + :param [content]: [Request body property file content] + + :return: dict with a status, description and (optionally) error + :rtype: dict + """ + key = self.get_argument('key') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + content = request_data.get('content') + + fs, item_path = self.validate_fs('put') + result = self.fs_manager.write(key, item_path, content, overwrite=True) + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + + def patch(self): + # Update PARTIAL file content + key = self.get_argument('key') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + content = request_data.get('content') + + fs, item_path = self.validate_fs('patch') + + #TODO: Properly Implement PATCH + result = self.fs_manager.update(key, item_path, content) + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + + + # DELETE /jupyter_fsspec/files?key=my-key&item_path=/some_directory/file.txt + async def delete(self): + """Delete the resource at the input path. + + :param [key]: [Query arg string used to retrieve the appropriate filesystem instance] + :param [item_path]: [Query arg string path to file or directory to be retrieved] + + :return: dict with a status, description and (optionally) error + :rtype: dict + """ + key = self.get_argument('key') + request_data = json.loads(self.request.body.decode('utf-8')) + item_path = request_data.get('item_path') + + fs, item_path = self.validate_fs('delete') + + result = self.fs_manager.delete(key, item_path) + + self.set_status(result["status_code"]) + self.write(result['response']) + self.finish() + #==================================================================================== # Update the handler in setup #==================================================================================== def setup_handlers(web_app): host_pattern = ".*$" + fs_manager = create_filesystem_manager() + base_url = web_app.settings["base_url"] route_fsspec_config = url_path_join(base_url, "jupyter_fsspec", "config") route_fsspec = url_path_join(base_url, "jupyter_fsspec", "fsspec") - handlers = [(route_fsspec_config, FsspecConfigHandler), (route_fsspec, FileSystemHandler)] + handlers = [ + (route_fsspec_config, FsspecConfigHandler, dict(fs_manager=fs_manager)), + (route_fsspec, FileSystemHandler, dict(fs_manager=fs_manager)) + ] + + route_files = url_path_join(base_url, "jupyter_fsspec", "files") + route_files_actions = url_path_join(base_url, "jupyter_fsspec", "files", "action") + route_rename_files = url_path_join(base_url, "jupyter_fsspec", "files", "rename") + route_fs_files_actions = url_path_join(base_url, "jupyter_fsspec", "files", "xaction") + + handlers_refactored = [ + (route_fsspec_config, FsspecConfigHandler, dict(fs_manager=fs_manager)), + (route_files, FileSysHandler, dict(fs_manager=fs_manager)), + (route_rename_files, RenameFileHandler, dict(fs_manager=fs_manager)), + (route_files_actions, FileActionHandler, dict(fs_manager=fs_manager)), + (route_fs_files_actions, FileActionCrossFSHandler, dict(fs_manager=fs_manager)), + ] + + web_app.add_handlers(host_pattern, handlers_refactored) web_app.add_handlers(host_pattern, handlers) diff --git a/jupyter_fsspec/tests/__init__.py b/jupyter_fsspec/tests/__init__.py new file mode 100644 index 0000000..369c9c4 --- /dev/null +++ b/jupyter_fsspec/tests/__init__.py @@ -0,0 +1 @@ +"""Python tests for jupyter-fsspec.""" diff --git a/jupyter_fsspec/tests/test_api.py b/jupyter_fsspec/tests/test_api.py new file mode 100644 index 0000000..f4e6914 --- /dev/null +++ b/jupyter_fsspec/tests/test_api.py @@ -0,0 +1,249 @@ +import json +import os +import pytest + +from tornado.httpclient import HTTPClientError +# TODO: Testing: different file types, received expected errors + +async def test_get_config(jp_fetch): + response = await jp_fetch("jupyter_fsspec", "config", method="GET") + assert response.code == 200 + + json_body = response.body.decode('utf-8') + body = json.loads(json_body) + assert body['status'] == 'success' + +async def test_get_files_memory(fs_manager_instance, jp_fetch): + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + mem_item_path = mem_fs_info['info']['path'] + assert mem_fs != None + + # Read directory + assert mem_fs.exists(mem_item_path) == True + dir_payload = {"item_path": mem_item_path} + dir_response = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key}, body=json.dumps(dir_payload), allow_nonstandard_methods=True) + + assert dir_response.code == 200 + json_body = dir_response.body.decode('utf-8') + body = json.loads(json_body) + assert body['status'] == 'success' + assert len(body['content']) == 3 + + # Read File + filepath = "/my_mem_dir/test_dir/file1.txt" + assert mem_fs.exists(filepath) == True + file_payload = {"item_path": filepath} + file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", params={"key": mem_key}, body=json.dumps(file_payload), allow_nonstandard_methods=True) + assert file_res.code == 200 + + file_json_body = file_res.body.decode('utf-8') + file_body = json.loads(file_json_body) + assert file_body['status'] == 'success' + assert file_body['content'] == "Test content" + + # GET file byte range + range_filepath = "/my_mem_dir/test_dir/file1.txt" + # previously checked file exists + range_file_payload = {"item_path": range_filepath} + range_file_res = await jp_fetch("jupyter_fsspec", "files", method="GET", headers={"Range": "0-8"}, params={"key": mem_key, "type": "range"}, body=json.dumps(range_file_payload), allow_nonstandard_methods=True) + assert range_file_res.code == 206 + + range_json_file_body = range_file_res.body.decode('utf-8') + range_file_body = json.loads(range_json_file_body) + assert range_file_body['status'] == 'success' + assert range_file_body['content'] == 'Test cont' + +async def test_post_files(fs_manager_instance, jp_fetch): + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # Post new file with content + filepath = "/my_mem_dir/test_dir/file2.txt" + # File does not already exist + assert mem_fs.exists(filepath) == False + file_payload = { "item_path": filepath, "content": "This is test file2 content"} + file_response = await jp_fetch("jupyter_fsspec", "files", method="POST", params={"key": mem_key}, body=json.dumps(file_payload)) + assert file_response.code == 200 + + file_json_body = file_response.body.decode('utf-8') + file_body = json.loads(file_json_body) + assert file_body['status'] == 'success' + assert file_body['description'] == 'Wrote /my_mem_dir/test_dir/file2.txt' + assert mem_fs.exists(filepath) == True + + # Post directory + newdirpath = "/my_mem_dir/test_dir/subdir" + # Directory does not already exist + assert mem_fs.exists(newdirpath) == False + dir_payload = {"item_path": "/my_mem_dir/test_dir", "content": "subdir"} + dir_response = await jp_fetch("jupyter_fsspec", "files", method="POST", params={"key": mem_key}, body=json.dumps(dir_payload)) + assert dir_response.code == 200 + dir_body_json = dir_response.body.decode('utf-8') + dir_body = json.loads(dir_body_json) + + assert dir_body['status'] == 'success' + assert dir_body['description'] == 'Wrote /my_mem_dir/test_dir/subdir/' + +async def test_delete_files(fs_manager_instance, jp_fetch): + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # Delete file + filepath = '/my_mem_dir/test_dir/file1.txt' + assert mem_fs.exists(filepath) == True + + file_payload = {"item_path": filepath} + response = await jp_fetch("jupyter_fsspec", "files", method="DELETE", params={"key": mem_key}, body=json.dumps(file_payload), allow_nonstandard_methods=True) + assert response.code == 200 + json_body = response.body.decode('utf-8') + body = json.loads(json_body) + + assert body['status'] == 'success' + assert mem_fs.exists(filepath) == False + + #delete directory + dirpath = "/my_mem_dir/test_dir" + assert mem_fs.exists(dirpath) == True + + dir_payload = {"item_path": dirpath} + dir_response = await jp_fetch("jupyter_fsspec", "files", method="DELETE", params={"key": mem_key}, body=json.dumps(dir_payload), allow_nonstandard_methods=True) + assert dir_response.code == 200 + dir_json_body = response.body.decode('utf-8') + dir_body = json.loads(dir_json_body) + + assert dir_body['status'] == 'success' + assert mem_fs.exists(dirpath) == False + +async def test_put_files(fs_manager_instance, jp_fetch): + # PUT replace entire resource + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # replace entire file content + filepath = '/my_mem_dir/test_dir/file1.txt' + file_payload = {"item_path": filepath, "content": "Replaced content"} + file_response = await jp_fetch("jupyter_fsspec", "files", method="PUT", params={"key": mem_key}, body=json.dumps(file_payload)) + assert file_response.code == 200 + + file_body_json = file_response.body.decode('utf-8') + file_body = json.loads(file_body_json) + assert file_body["status"] == 'success' + assert file_body['description'] == 'Wrote /my_mem_dir/test_dir/file1.txt' + + # replacing directory returns error + dirpath = '/my_mem_dir/test_dir' + dir_payload = {"item_path": dirpath, "content": "new_test_dir"} + with pytest.raises(HTTPClientError) as exc_info: + await jp_fetch("jupyter_fsspec", "files", method="PUT", params={"key": mem_key}, body=json.dumps(dir_payload)) + assert exc_info.value.code == 409 + +async def test_rename_files(fs_manager_instance, jp_fetch): + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # rename file + filepath = '/my_mem_dir/test_dir/file1.txt' + file_payload = {"item_path": filepath, "content": "new_file"} + file_response = await jp_fetch("jupyter_fsspec", "files", "rename", method="POST", params={"key": mem_key}, body=json.dumps(file_payload)) + assert file_response.code == 200 + + file_body_json = file_response.body.decode('utf-8') + file_body = json.loads(file_body_json) + assert file_body["status"] == 'success' + assert file_body['description'] == 'Renamed /my_mem_dir/test_dir/file1.txt to /my_mem_dir/test_dir/new_file.txt' + + + # rename directory + dirpath = '/my_mem_dir/second_dir' + dir_payload = {"item_path": dirpath, "content": "new_dir"} + dir_response = await jp_fetch("jupyter_fsspec", "files", "rename", method="POST", params={"key": mem_key}, body=json.dumps(dir_payload)) + assert dir_response.code == 200 + + dir_body_json = dir_response.body.decode('utf-8') + dir_body = json.loads(dir_body_json) + assert dir_body["status"] == 'success' + assert dir_body['description'] == "Renamed /my_mem_dir/second_dir to /my_mem_dir/new_dir" + +# TODO: Implement update functionality +# PATCH partial update without modifying entire data +async def test_patch_file(fs_manager_instance, jp_fetch): + #file only + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # replace partial file content + filepath = '/my_mem_dir/test_dir/file1.txt' + old_content = 'Test content' + file_payload = {"item_path": filepath, "content": " and new"} + file_res = await jp_fetch("jupyter_fsspec", "files", method="PATCH", params={"key": mem_key}, body=json.dumps(file_payload)) + assert file_res.code == 200 + +async def test_action_same_fs_files(fs_manager_instance, jp_fetch): + fs_manager = fs_manager_instance + mem_fs_info = fs_manager.get_filesystem_by_type('memory') + mem_key = mem_fs_info['key'] + mem_fs = mem_fs_info['info']['instance'] + assert mem_fs != None + + # Copy + copy_filepath = '/my_mem_dir/test_dir/file1.txt' + copy_file_payload = {"item_path": copy_filepath, "content": "/my_mem_dir/file_to_copy.txt", "action": "copy"} + copy_file_res = await jp_fetch("jupyter_fsspec", "files", "action", method="POST", params={"key": mem_key}, body=json.dumps(copy_file_payload)) + + cfile_body_json = copy_file_res.body.decode('utf-8') + cfile_body = json.loads(cfile_body_json) + assert cfile_body["status"] == 'success' + assert cfile_body['description'] == 'Copied /my_mem_dir/test_dir/file1.txt to /my_mem_dir/file_to_copy.txt' + + # Copy directory + copy_dirpath = '/my_mem_dir/test_dir' + copy_dir_payload = {"item_path": copy_dirpath, "content": "/my_mem_dir/second_dir", "action": "copy"} + copy_dir_res = await jp_fetch("jupyter_fsspec", "files", "action", method="POST", params={"key": mem_key}, body=json.dumps(copy_dir_payload)) + + cdir_body_json = copy_dir_res.body.decode('utf-8') + cdir_body = json.loads(cdir_body_json) + assert cdir_body["status"] == 'success' + assert cdir_body['description'] == 'Copied /my_mem_dir/test_dir to /my_mem_dir/second_dir' + + # Move file + move_filepath = '/my_mem_dir/test_dir/file1.txt' + move_file_payload = {"item_path": move_filepath, "content": "/my_mem_dir/new_file", "action": "move"} + move_file_res = await jp_fetch("jupyter_fsspec", "files", "action", method="POST", params={"key": mem_key}, body=json.dumps(move_file_payload)) + assert move_file_res.code == 200 + + mfile_body_json = move_file_res.body.decode('utf-8') + mfile_body = json.loads(mfile_body_json) + assert mfile_body["status"] == 'success' + assert mfile_body['description'] == 'Moved /my_mem_dir/test_dir/file1.txt to /my_mem_dir/new_file.txt' + + # Move directory + move_dirpath = '/my_mem_dir/test_dir' + move_dir_payload = {"item_path": move_dirpath, "content": "/my_mem_dir/second_dir", "action": "move"} + move_dir_res = await jp_fetch("jupyter_fsspec", "files", "action", method="POST", params={"key": mem_key}, body=json.dumps(move_dir_payload)) + + mdir_body_json = move_dir_res.body.decode('utf-8') + mdir_body = json.loads(mdir_body_json) + assert mdir_body["status"] == 'success' + assert mdir_body['description'] == 'Moved /my_mem_dir/test_dir to /my_mem_dir/second_dir' + +#TODO: Test xaction endpoint +async def xtest_xaction_diff_fs(fs_manager_instance, jp_fetch): + pass \ No newline at end of file diff --git a/jupyter_fsspec/tests/unit/test_filesystem_manager.py b/jupyter_fsspec/tests/unit/test_filesystem_manager.py new file mode 100644 index 0000000..43953aa --- /dev/null +++ b/jupyter_fsspec/tests/unit/test_filesystem_manager.py @@ -0,0 +1,331 @@ +import pytest +import fsspec +from pathlib import Path +import os +import yaml +from jupyter_fsspec.file_manager import FileSystemManager +from pathlib import PurePath +from unittest.mock import patch + +# Test FileSystemManager class and file operations + +# ============================================================ +# Test FileSystemManager config loading/read +# ============================================================ +@pytest.fixture +def config_file(tmp_path): + config = { + 'sources': [{ + 'name': 'inmem', + 'path': '/mem_dir', + 'type': 'memory' + }] + } + config_path = tmp_path / 'config.yaml' + with open(config_path, 'w') as file: + yaml.dump(config, file) + return config_path + +# @pytest.fixture +# def mock_filesystem(config_file): +# fs_test_manager = FileSystemManager(config_file) +# file_systems = fs_test_manager.filesystems +# return file_systems + + + +# test that the file systems are created +def test_filesystem_init(config_file): + fs_test_manager = FileSystemManager(config_file) + + assert fs_test_manager is not None + # assert 'memory' in fs_test_manager.filesystems + + # Validate in-memory filesystem + in_memory_fs = fs_test_manager.get_filesystem(fs_test_manager._encode_key({ + 'name': 'inmem', + 'path': '/mem_dir', + 'type': 'memory' + })) + + assert in_memory_fs is not None + # TODO: + # assert any('memory' in key for key in fs_test_manager.filesystems) + # assert in_memory_fs['type'] == 'memory' + # assert in_memory_fs['path'] == '/mem_dir' + +# test key encoding/decoding +def test_key_decode_encode(config_file): + fs_test_manager = FileSystemManager(config_file) + + fs_test_config = { + 'name': 'mylocal', + 'type': 'local', + 'path': str(config_file.parent) + } + + encoded_key = fs_test_manager._encode_key(fs_test_config) + decoded_type, decoded_path = fs_test_manager._decode_key(encoded_key) + + # Ensure both paths are absolute by adding leading slash if missing + decoded_path = '/' + decoded_path.lstrip('/') # Normalize decoded path + fs_test_config['path'] = '/' + fs_test_config['path'].lstrip('/') # Normalize original path + + assert decoded_path == fs_test_config['path'] # Compare as strings + assert decoded_type == fs_test_config['type'] + + +# ============================================================ +# Test FileSystemManager file operations +# ============================================================ +@pytest.fixture +def mock_config(): + mock_config = { + 'sources': [{ + 'type': 'memory', + 'name': 'test_memory_fs', + 'path': '/test_memory' + }] + } + return mock_config + +@pytest.fixture +def fs_manager_instance(mock_config): + fs_test_manager = FileSystemManager.__new__(FileSystemManager) + fs_test_manager.config = mock_config + fs_test_manager.filesystems = {} + fs_test_manager._initialize_filesystems() + return fs_test_manager + +@pytest.fixture +def populated_fs_manager(mock_config, fs_manager_instance): + key = fs_manager_instance._encode_key(mock_config['sources'][0]) + + dir_path = 'test_memory' + dest_path = 'test_dir' + file_path = f'{dir_path}/{dest_path}/test_file_pop.txt' + file_content = b"This is a test for a populated filesystem!" + + fs_manager_instance.write(key, dir_path, dest_path) + + fs_manager_instance.write(key, file_path, file_content) + + second_file_path = f'{dir_path}/second_test_file_pop.txt' + second_file_content = b"Second test for a populated filesystem!" + fs_manager_instance.write(key, second_file_path, second_file_content) + return fs_manager_instance, key + + +def test_file_read_write(mock_config, fs_manager_instance): + key = fs_manager_instance._encode_key(mock_config['sources'][0]) + + #write + item_path = '/test_memory/my_file.txt' + content = b"Hello, this is a test!" + + write_result = fs_manager_instance.write(key, item_path, content) + assert write_result['status_code'] == 200 + + #read + read_result = fs_manager_instance.read(key, item_path) + assert read_result['status_code'] == 200 + assert read_result['response']['content'] == content.decode('utf-8') + +def xtest_file_update_delete(populated_fs_manager): + key = fs_manager_instance._encode_key(mock_config['sources'][0]) + +def test_directory_read_write(mock_config, fs_manager_instance): + key = fs_manager_instance._encode_key(mock_config['sources'][0]) + + #write + item_path = 'test_memory' + dest_path = 'my_test_subdir' + + write_result = fs_manager_instance.write(key, item_path, dest_path) + assert write_result['status_code'] == 200 + + #read + read_result = fs_manager_instance.read(key, item_path) + content_list = read_result['response']['content'] + assert read_result['status_code'] == 200 + + dir_name_to_check = '/' + item_path + '/my_test_subdir' + subdir_exists = any(item['name'] == dir_name_to_check and item['type'] == 'directory' for item in content_list) + assert subdir_exists + +def xtest_directory_update_delete(populated_fs_manager): + key = fs_manager_instance._encode_key(mock_config['sources'][0]) + + #update + + #delete + + + + + + + +# ============================================================ +# OLD Test FileSystemManager file operations +# ============================================================ +# provide the file system with all needed information like key, path etc +# def generate_fs(): +# fs_test_config = { +# 'name': 'mylocal', +# 'type': 'local', +# 'path': str(config_file.parent) +# } + +# create file +#TODO: create fs_manager fixture to include for these tests? +# def test_create_file(config_file): +# fs_test_manager = FileSystemManager(config_file) +# filesystems = fs_test_manager.filesystems + +# for key in filesystems.keys(): +# fs_path = filesystems[key]['path'] + +# file_path = 'test_create_file.txt' +# complete_file_path = str(PurePath(fs_path) / file_path) +# content = b'testing file content' + +# fs_test_manager.write(key, complete_file_path, content) + +# fs_info = fs_test_manager.get_filesystem(key) +# fs = fs_info['instance'] +# assert fs.exists(complete_file_path), "File should exist" + +# file_content = fs.cat(complete_file_path) +# assert file_content == content, "File content should match expected content." + + +# create directory +# def test_create_dir(config_file): +# fs_test_manager = FileSystemManager(config_file) +# filesystems = fs_test_manager.filesystems + +# for key in filesystems.keys(): +# fs_path = filesystems[key]['path'] + +# dir_name = 'testing_dir_name' + +# fs_test_manager.write(key, fs_path, dir_name) +# complete_file_path = str(PurePath(fs_path) / dir_name) + '/' + +# fs_info = fs_test_manager.get_filesystem(key) +# fs = fs_info['instance'] +# assert fs.exists(complete_file_path), "Directory should exist" + + + +# read file +# TODO: use memory filesystem and mock_filesystem dict +# def test_read_file_success(memory_filesystem): +# mock_filesystem = { +# '/dir1': {}, +# '/dir1/file1.txt': 'Content of file1.', +# '/dir1/file2.txt': 'Content of file2.', +# '/dir2': {}, +# '/dir2/subdir': {}, +# '/dir2/subdir/file3.txt': 'Content of file3 in subdir of dir2.' +# } + +# with patch(fsspec.filesystem) +# def populate_filesystem(filesystem, structure, base_path='/'): +# for name, content in structure.items(): +# path = f"{base_path.rstrip('/')/{name}}" + +# if isinstance(content, dict): +# filesystem.mkdir(path) +# populate_filesystem(filesystem, content, base_path=path) +# else: +# if isinstance(content, bytes): +# filesystem.pipe(path, content) +# else: +# filesystem.pipe(path, content.encode()) + +# @pytest.fixture +# def populated_filesystem(mock_filesystem): +# directory_structure = { +# 'dir1': { +# 'file1.txt': 'Content of file1 in dir1.', +# 'file2.txt': 'Content of file2 in dir1.', +# }, +# 'dir2': { +# 'subdir': { +# 'file3.txt': 'Content of file3 in subdir of dir2.', +# 'file4.txt': 'Content of file4 in subdir of dir2.', +# }, +# }, +# 'fileOne.txt': 'This is content of fileOne in root dir.', +# 'fileTwo.txt': 'This is content of fileTwo in root dir.', +# 'binaryfile.bin': b'\x00\x01\x02' +# } + +# key, fs_info = next(iter(mock_filesystem.items())) +# fs_path = fs_info['path'] +# fs = fs_info['instance'] +# populate_filesystem(fs, directory_structure, fs_path) + +# return fs, key, fs_path + + +# def test_read_file(config_file): +# fs_test_manager = FileSystemManager(config_file) +# filesystems = fs_test_manager.filesystems + +# for key in filesystems.keys(): +# fs_path = filesystems[key]['path'] + +# fs_info = fs_test_manager.get_filesystem(key) +# fs = fs_info['instance'] +# def test_read_file(populated_filesystem): +# fs, key, item_path = populated_filesystem +# content = fs.read() + + +# # read directory +# def test_read_dir_success(config_file): +# fs_test_manager = FileSystemManager(config_file) + +# fs = fs_test_manager.filesystems +# +# +# +# +# +# +# +# + + + + + +# # write file +# def test_write_file_success(config_file): +# fs_test_manager = FileSystemManager(config_file) + +# fs = fs_test_manager.filesystems +# +# +# +# +# +# +# +# + +# # delete file +# def test_delete_file_success(config_file): +# fs_test_manager = FileSystemManager(config_file) + +# fs = fs_test_manager.filesystems + + +# # delete directory +# def test_delete_dir_success(config_file): + # fs_test_manager = FileSystemManager(config_file) + + # fs = fs_test_manager.filesystems \ No newline at end of file diff --git a/package.json b/package.json index 318528e..7ae291b 100644 --- a/package.json +++ b/package.json @@ -57,12 +57,12 @@ "watch:labextension": "jupyter labextension watch ." }, "dependencies": { - "@jupyter/react-components": "^0.17.0", - "@jupyter/web-components": "^0.17.0", + "@jupyter/react-components": "^0.15.0", + "@jupyter/web-components": "^0.15.0", "@jupyterlab/application": "^4.0.0", - "@jupyterlab/apputils": "^4.3.4", + "@jupyterlab/apputils": "^4.0.0", "@jupyterlab/settingregistry": "^4.0.0", - "@jupyterlab/ui-components": "^4.2.5", + "@jupyterlab/ui-components": "^4.0.0", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/src/FilesystemItem.ts b/src/FilesystemItem.ts deleted file mode 100644 index 307ef32..0000000 --- a/src/FilesystemItem.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Element for displaying a single fsspec filesystem - -class FilesystemItem { - element: HTMLElement; - filesysName: string; - filesysType: string; - fsInfo: any; - clickSlots: any; - nameField: any; - typeField: any; - - constructor(fsInfo: any, userClickSlots: any) { - this.filesysName = fsInfo.name; - this.filesysType = fsInfo.type; - this.fsInfo = fsInfo; - - this.clickSlots = []; - for (const slot of userClickSlots) { - this.clickSlots.push(slot); - } - - let fsItem = document.createElement('div'); - fsItem.classList.add('jfss-fsitem-root'); - this.element = fsItem; - - this.nameField = document.createElement('div'); - this.nameField.classList.add('jfss-fsitem'); - this.nameField.innerText = this.filesysName; - this.nameField.addEventListener('mouseenter', this.handleFsysHover.bind(this)); - this.nameField.addEventListener('mouseleave', this.handleFsysHover.bind(this)); - fsItem.appendChild(this.nameField); - - this.typeField = document.createElement('div'); - this.typeField.classList.add('jfss-fsitem'); - this.typeField.innerText = this.filesysType; - fsItem.appendChild(this.typeField); - - fsItem.addEventListener('click', this.handleClick.bind(this)); - } - - handleFsysHover(event: any) { - if (event.type == 'mouseenter') { - this.nameField.style.backgroundColor = '#bbb'; - this.typeField.style.backgroundColor = '#bbb'; - } - else { - this.nameField.style.backgroundColor = '#ddd'; - this.typeField.style.backgroundColor = '#ddd'; - } - } - - handleClick(_event: any) { - for (const slot of this.clickSlots) { - slot(this.fsInfo); - } - } - } - - export { FilesystemItem }; diff --git a/src/FssFilesysItem.ts b/src/FssFilesysItem.ts new file mode 100644 index 0000000..aacd56d --- /dev/null +++ b/src/FssFilesysItem.ts @@ -0,0 +1,112 @@ +// Element for displaying a single fsspec filesystem + +import { FssContextMenu } from './treeContext'; + +class FssFilesysItem { + root: HTMLElement; + filesysName: string; + filesysType: string; + fsInfo: any; + clickSlots: any; + nameField: any; + typeField: any; + + constructor(fsInfo: any, userClickSlots: any) { + this.filesysName = fsInfo.name; + this.filesysType = fsInfo.type; + this.fsInfo = fsInfo; + + this.clickSlots = []; + for (const slot of userClickSlots) { + this.clickSlots.push(slot); + } + + let fsItem = document.createElement('div'); + fsItem.classList.add('jfss-fsitem-root'); + fsItem.addEventListener('mouseenter', this.handleFsysHover.bind(this)); + fsItem.addEventListener('mouseleave', this.handleFsysHover.bind(this)); + this.root = fsItem; + + // Set the tooltip + this.root.title = `Root Path: ${fsInfo.path}`; + + this.nameField = document.createElement('div'); + this.nameField.classList.add('jfss-fsitem-name'); + this.nameField.innerText = this.filesysName; + fsItem.appendChild(this.nameField); + + this.typeField = document.createElement('div'); + this.typeField.classList.add('jfss-fsitem-type'); + this.typeField.innerText = 'Type: ' + this.filesysType; + fsItem.appendChild(this.typeField); + + fsItem.addEventListener('click', this.handleClick.bind(this)); + fsItem.addEventListener('contextmenu', this.handleContext.bind(this)); + } + + handleContext(event: any) { + // Prevent ancestors from adding extra context boxes + event.stopPropagation(); + + // Prevent default browser context menu (unless shift pressed + // as per usual JupyterLab conventions) + if (!event.shiftKey) { + event.preventDefault(); + } else { + return; + } + + // Make/add the context menu + let context = new FssContextMenu(); + context.root.dataset.fss = this.root.dataset.fss; + let body = document.getElementsByTagName('body')[0]; + body.appendChild(context.root); + + // Position it under the mouse (top left corner normally, + // or bottom right if that corner is out-of-viewport) + let parentRect = body.getBoundingClientRect(); + let contextRect = context.root.getBoundingClientRect(); + let xCoord = event.clientX - parentRect.x; + let yCoord = event.clientY - parentRect.y; + let spacing = 12; + if (xCoord + contextRect.width > window.innerWidth || yCoord + contextRect.height > window.innerHeight) { + // Context menu is cut off when positioned under mouse at top left corner, + // use the bottom right corner instead + xCoord -= contextRect.width; + yCoord -= contextRect.height; + // Shift the menu so the mouse is inside it, not at the corner/edge + xCoord += spacing; + yCoord += spacing; + } else { + // Shift the menu so the mouse is inside it, not at the corner/edge + xCoord -= spacing; + yCoord -= spacing; + } + + context.root.style.left = `${xCoord}` + 'px'; + context.root.style.top = `${yCoord}` + 'px'; + } + + setMetadata(value: string) { + this.root.dataset.fss = value; + } + + handleFsysHover(event: any) { + if (event.type == 'mouseenter') { + this.root.style.backgroundColor = 'var(--jp-layout-color3)'; + this.root.style.backgroundColor = 'var(--jp-layout-color3)'; + } + else { + this.root.style.backgroundColor = 'var(--jp-layout-color2)'; + this.root.style.backgroundColor = 'var(--jp-layout-color2)'; + } + } + + handleClick(_event: any) { + for (const slot of this.clickSlots) { + slot(this.fsInfo); + } + } + } + + export { FssFilesysItem }; diff --git a/src/FssTreeItem.ts b/src/FssTreeItem.ts index 2f71222..241463f 100644 --- a/src/FssTreeItem.ts +++ b/src/FssTreeItem.ts @@ -4,6 +4,7 @@ import { TreeItem } from '@jupyter/web-components'; import { fileIcon, folderIcon } from '@jupyterlab/ui-components'; import { FssContextMenu } from './treeContext'; +import { Logger } from "./logger" export class FssTreeItem { root: any; @@ -13,19 +14,37 @@ export class FssTreeItem { container: HTMLElement; clickSlots: any; isDir = false; + treeItemObserver: MutationObserver; + pendingExpandAction = false; + lazyLoadAutoExpand = true; + clickAnywhereDoesAutoExpand = true; - constructor(clickSlots: any) { + constructor(clickSlots: any, autoExpand: boolean, expandOnClickAnywhere: boolean) { // The TreeItem component is the root and handles // tree structure functionality in the UI let root = new TreeItem(); this.root = root; this.clickSlots = clickSlots; + this.lazyLoadAutoExpand = autoExpand; + this.clickAnywhereDoesAutoExpand = expandOnClickAnywhere; + + // Use a MutationObserver on the root TreeItem's shadow DOM, + // where the TreeItem's expand/collapse control will live once + // the item has children to show + let observeOptions = { + childList: true, + subtree: true, + attributes: true, + attributeFilter: ['class'], + attributeOldValue: true, + }; + this.treeItemObserver = new MutationObserver(this.handleDomMutation.bind(this)); // The main container holds custom fsspec UI/functionality let container = document.createElement('div'); container.classList.add('jfss-tree-item-container'); root.appendChild(container); - this.container = container + this.container = container; // Reserve space in the layout for the file/folder icon let dirSymbol = document.createElement('div'); @@ -42,6 +61,11 @@ export class FssTreeItem { // Add click and right click handlers to the tree component root.addEventListener('contextmenu', this.handleContext.bind(this)); root.addEventListener('click', this.handleClick.bind(this), true); + + // Start observing for changes to the TreeItem's shadow root + if (this.root.shadowRoot) { + this.treeItemObserver.observe(this.root.shadowRoot, observeOptions) + } } appendChild(elem: any) { @@ -70,14 +94,69 @@ export class FssTreeItem { } } + handleDomMutation(records: any, observer: any) { + // This is used to auto-expand directory-type TreeItem's to show children after + // a lazy-load. It checks the TreeItem's shadow dom for the addition of an + // "expand-collapse-button" child control which is used to expand and show + // children (in the tree) of this class's root TreeItem node. By auto expanding here, + // we save the user from having to click twice on a folder (once to lazy-load + // and another time to expand) when they want to expand it + if (this.lazyLoadAutoExpand && this.pendingExpandAction) { + for (const rec of records) { + let addedNodes = rec?.addedNodes; + if (addedNodes) { + for (let node of addedNodes) { + if (node?.classList && node.classList.contains('expand-collapse-button')) { + node.click(); + this.root.scrollTo(); + this.pendingExpandAction = false; + } + } + } + } + } + } + handleClick(event: any) { - if (this.isDir) { - for (let slot of this.clickSlots) { - slot(this.root.dataset.fss); + // Filter click events to handle this item's root+shadow and container + if (event.target === this.root || this.container.contains(event.target) || this.root.shadowRoot.contains(event.target)) { + // Handles normal click events on the TreeItem (unlike the MutationObserver system + // which is for handling folder auto-expand after lazy load) + if (this.clickAnywhereDoesAutoExpand) { + let expander = this.root.shadowRoot.querySelector('.expand-collapse-button'); + if (expander) { + let expRect = expander.getBoundingClientRect(); + if ((event.clientX < expRect.left + || event.clientX > expRect.right + || event.clientY < expRect.top + || event.clientY > expRect.bottom)) { + Logger.debug('--> Click outside expander, force expander click'); + expander.click(); + this.root.scrollTo(); + } + } + } + // Fire connected slots that were supplied to this item on init + if (this.isDir) { + for (let slot of this.clickSlots) { + slot(this.root.dataset.fss); + } + } + else { + this.root.click(); } } } + expandItem() { + // This method's purpose is to expand folder items to show children + // after a lazy load, but when this is called, the expand controls aren't + // ready...a flag is set here to indicate that an expand action is desired, + // which is used by the MutationObserver member var's handler to find the + // expand/collapse Element when it is added so that it can be click()'d + this.pendingExpandAction = true; + } + handleContext(event: any) { // Prevent ancestors from adding extra context boxes event.stopPropagation(); diff --git a/src/handler/fileOperations.ts b/src/handler/fileOperations.ts index 4bde18a..0757524 100644 --- a/src/handler/fileOperations.ts +++ b/src/handler/fileOperations.ts @@ -1,4 +1,5 @@ import { requestAPI } from './handler'; +import { Logger } from "../logger" /* interface IFilesystemConfig { @@ -10,26 +11,60 @@ interface IFilesystemConfig { } */ +declare global { + interface Window { + fsspecModel: FsspecModel; + } +} + export class FsspecModel { activeFilesystem: string = ''; userFilesystems: any = {}; + retry = 0; - constructor() { - } + async initialize(automatic: boolean=true, retry=3) { + this.retry = retry; + if (automatic) { + // Perform automatic setup: Fetch filesystems from config and store + // this model on the window as global application state + this.storeApplicationState(); - async initialize() { - try { - this.userFilesystems = await this.getStoredFilesystems(); - console.log('filesystem list is: ', JSON.stringify(this.userFilesystems)); - // Optional to set first filesystem as active. - if (Object.keys(this.userFilesystems).length > 0) { - this.activeFilesystem = Object.keys(this.userFilesystems)[0]; + // Attempt to read and store user config values + this.userFilesystems = {}; + try { + for (let i = 0; i < retry; i++) { + + Logger.info('[FSSpec] Attempting to read config file...'); + let result = await this.getStoredFilesystems(); + if (result?.status == 'success') { + // TODO report config entry errors + Logger.info(`[FSSpec] Successfully retrieved config:${JSON.stringify(result)}`); + this.userFilesystems = result.filesystems; + + // Set active filesystem to first + if (Object.keys(result).length > 0) { + this.activeFilesystem = Object.keys(this.userFilesystems)[0]; + } + break; + } else { + // TODO handle no config file + Logger.error('[FSSpec] Error fetching filesystems from user config'); + if (i + 1 < retry) { + Logger.info('[FSSpec] retrying...'); + } + } + } + } catch (error) { + Logger.error(`[FSSpec] Error: Unknown error initializing fsspec model:\n${error}`); } - } catch (error) { - console.error('Failed to initialize FsspecModel: ', error); } } + // Store model on the window as global app state + storeApplicationState() { + window.fsspecModel = this; + } + // ==================================================================== // FileSystem API calls // ==================================================================== @@ -45,32 +80,77 @@ export class FsspecModel { return this.userFilesystems[this.activeFilesystem]; } + async refreshConfig() { + // TODO fix/refactor + this.userFilesystems = {}; + try { + for (let i = 0; i < this.retry; i++) { + + Logger.info('[FSSpec] Attempting to read config file...'); + let result = await this.getStoredFilesystems(); + if (result?.status == 'success') { + // TODO report config entry errors + Logger.info(`[FSSpec] Successfully retrieved config:${JSON.stringify(result)}`); + this.userFilesystems = result.filesystems; + + // Set active filesystem to first + if (Object.keys(result).length > 0) { + this.activeFilesystem = Object.keys(this.userFilesystems)[0]; + } + break; + } else { + // TODO handle no config file + Logger.error('[FSSpec] Error fetching filesystems from user config'); + if (i + 1 < this.retry) { + Logger.info('[FSSpec] retrying...'); + } + } + } + } catch (error) { + Logger.error(`[FSSpec] Error: Unknown error initializing fsspec model:\n${error}`); + } + } + async getStoredFilesystems(): Promise { // Fetch list of filesystems stored in user's config file - const filesystems: any = {}; + let filesystems: any = {}; + let result = { + 'filesystems': filesystems, + 'status': 'success', + }; try { const response = await requestAPI('config'); - console.log('Fetch FSs'); - const fetchedFilesystems = response['content']; - console.log(fetchedFilesystems); - - // Map names to filesys metadata - for (const filesysInfo of fetchedFilesystems) { - if ('name' in filesysInfo) { - filesystems[filesysInfo.name] = filesysInfo; - } else { - console.error( - `Filesystem from config is missing a name: ${filesysInfo}` - ); + Logger.debug(`[FSSpec] Request config:\n${JSON.stringify(response)}`); + if (response?.status == 'success' && response?.content) { + for (const filesysInfo of response.content) { + if (filesysInfo?.name) { + Logger.debug(`[FSSpec] Found filesystem: ${JSON.stringify(filesysInfo)}`) + filesystems[filesysInfo.name] = filesysInfo; + } else { + // TODO better handling for partial errors + Logger.error( `[FSSpec] Error, filesystem from config is missing a name: ${filesysInfo}`); + } } + } else { + Logger.error(`[FSSpec] Error fetching config from server...`); } + // // const fetchedFilesystems = response['content']; + // // console.log(fetchedFilesystems); + // // Map names to filesys metadata + // for (const filesysInfo of fetchedFilesystems) { + // if ('name' in filesysInfo) { + // filesystems[filesysInfo.name] = filesysInfo; + // } else { + // console.error( + // `Filesystem from config is missing a name: ${filesysInfo}` + // ); + // } + // } } catch (error) { - console.error('Failed to fetch filesystems: ', error); + Logger.error(`[FSSpec] Error: Unknown error fetching config:\n${error}`); } - console.log( - `getStoredFilesystems Returns: \n${JSON.stringify(filesystems)}` - ); - return filesystems; + + return result; } async getContent( @@ -140,6 +220,25 @@ export class FsspecModel { } } + async delete_refactored(key: string, item_path: string): Promise { + try { + const query = new URLSearchParams({ + key, + item_path + }); + const response = await requestAPI(`files?${query.toString()}`, { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json' + } + }); + console.log('response is: ', response); + } catch (error) { + console.error('Failed to delete: ', error); + return null; + } + } + async deleteDir(key: string, item_path: string): Promise { try { const reqBody = JSON.stringify({ @@ -351,16 +450,35 @@ export class FsspecModel { } async listDirectory(key: string, item_path: string = '', type: string = ''): Promise { - const query = new URLSearchParams({ key, item_path, type}); + const query = new URLSearchParams({ key, item_path, type }).toString(); + let result = null; + Logger.debug(`[FSSpec] Fetching files -> ${query}`); try { - return await requestAPI(`fsspec?${query.toString()}`, { + result = await requestAPI(`fsspec?${query}`, { method: 'GET' }); } catch (error) { - console.error(`Failed to list filesystem ${key}: `, error); - return null; + Logger.error(`[FSSpec] Failed to list filesystem ${error}: `); + } + + return result; + } + + async listDirectory_refactored(key: string, item_path: string = '', type: string = 'default'): Promise { + const query = new URLSearchParams({ key, item_path, type }).toString(); + let result = null; + + Logger.debug(`[FSSpec] Fetching files -> ${query}`); + try { + result = await requestAPI(`files?${query}`, { + method: 'GET' + }); + } catch (error) { + Logger.error(`[FSSpec] Failed to list filesystem ${error}: `); } + + return result; } async updateFile( diff --git a/src/index.ts b/src/index.ts index 231fe2b..9d9607a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { ICommandPalette } from '@jupyterlab/apputils'; import { FileManagerWidget } from './FileManager'; import { FsspecModel } from './handler/fileOperations'; -import { FilesystemItem } from './FilesystemItem'; +import { FssFilesysItem } from './FssFilesysItem'; import { FssTreeItem } from './FssTreeItem'; import { Widget } from '@lumino/widgets'; @@ -39,6 +39,9 @@ class FsspecWidget extends Widget { upperArea: any; model: any; selectedFsLabel: any; + fsDetails: any; + detailName: any; + detailPath: any; treeView: any; elementHeap: any = {}; filesysContainer: any; @@ -62,6 +65,27 @@ class FsspecWidget extends Widget { mainLabel.innerText = 'Jupyter FSSpec' this.upperArea.appendChild(mainLabel); + let sourcesControls = document.createElement('div'); + sourcesControls.classList.add('jfss-sourcescontrols'); + this.upperArea.appendChild(sourcesControls); + + let sourcesLabel = document.createElement('div'); + sourcesLabel.classList.add('jfss-sourceslabel'); + sourcesLabel.innerText = 'Configured Filesystems' + sourcesLabel.title = 'A list of filesystems stored in the Jupyter FSSpec yaml'; + sourcesControls.appendChild(sourcesLabel); + + let sourcesDivider = document.createElement('div'); + sourcesLabel.classList.add('jfss-sourcesdivider'); + sourcesControls.appendChild(sourcesDivider); + + let refreshConfig = document.createElement('div'); + refreshConfig.title = 'Re-read and refresh sources from config'; + refreshConfig.classList.add('jfss-refreshconfig'); + refreshConfig.innerText = '\u{21bb}' + refreshConfig.addEventListener('click', this.fetchConfig.bind(this)) + sourcesControls.appendChild(refreshConfig); + this.filesysContainer = document.createElement('div'); this.filesysContainer.classList.add('jfss-userfilesystems'); this.upperArea.appendChild(this.filesysContainer); @@ -72,15 +96,19 @@ class FsspecWidget extends Widget { let lowerArea = document.createElement('div'); lowerArea.classList.add('jfss-lowerarea'); - let resultArea = document.createElement('div'); - resultArea.classList.add('jfss-resultarea'); - lowerArea.appendChild(resultArea); + let browserAreaLabel = document.createElement('div'); + browserAreaLabel.classList.add('jfss-browseAreaLabel'); + browserAreaLabel.innerText = 'Browse Filesystem'; + lowerArea.appendChild(browserAreaLabel); this.selectedFsLabel = document.createElement('div'); this.selectedFsLabel.classList.add('jfss-selectedFsLabel'); - this.selectedFsLabel.classList.add('jfss-mainlabel'); - this.selectedFsLabel.innerText = 'Select a filesystem to display'; - resultArea.appendChild(this.selectedFsLabel); + this.selectedFsLabel.innerText = '