[WIP] Add sync handler base

jupyter_fsspec/handlers.py

@@ -251,6 +251,77 @@ def post(self):
         self.finish()
 
 
+# ====================================================================================
+# Handle Syncing Local and Remote filesystems
+# ====================================================================================
+class FilesystemSyncHandler(BaseFileSystemHandler):
+    def initialize(self, fs_manager):
+        self.fs_manager = fs_manager
+
+    async def get(self):
+        # remote to local (fetching latest changes)
+        request_data = json.loads(self.request.body.decode("utf-8"))
+        local_destination_path = request_data.get("local_path")
+        remote_source_path = request_data.get("remote_path")
+        dest_fs_key = request_data.get("destination_key")
+        dest_fs_info = self.fs_manager.get_filesystem(dest_fs_key)
+        dest_path = dest_fs_info["path"]
+
+        response = {"content": None}
+
+        fs, dest_path = self.validate_fs("post", dest_fs_key, dest_path)
+        remote_fs_instance = fs["instance"]  # noqa: F841
+
+        try:
+            # rsync
+            # (remote_source_path, local_destination_path)
+            # await remote_fs_instance....
+            self.set_status(200)
+            response["status"] = "success"
+            response["description"] = (
+                f"Synced {local_destination_path} to {remote_source_path}."
+            )
+        except Exception as e:
+            print(f"Error with sync handler: {e}")
+            self.set_status(500)
+            response["status"] = "failed"
+            response["description"] = str(e)
+        self.write(response)
+        self.finish()
+
+    async def post(self):
+        # local to remote (pushing latest changes)
+        key = self.get_argument("key")  # noqa: F841
+        request_data = json.loads(self.request.body.decode("utf-8"))
+        local_source_path = request_data.get("local_path")
+        remote_destination_path = request_data.get("remote_path")
+        dest_fs_key = request_data.get("destination_key")
+        dest_fs_info = self.fs_manager.get_filesystem(dest_fs_key)
+        dest_path = dest_fs_info["path"]
+
+        response = {"content": None}
+
+        fs, dest_path = self.validate_fs("post", dest_fs_key, dest_path)
+        remote_fs_instance = fs["instance"]  # noqa: F841
+
+        try:
+            # rsync
+            # (local_source_path, remote_destination_path)
+            # await remote_fs_instance....
+            self.set_status(200)
+            response["status"] = "success"
+            response["description"] = (
+                f"Synced {remote_destination_path} to {local_source_path}."
+            )
+        except Exception as e:
+            logger.debug(f"Error with sync handler: {e}")
+            self.set_status(500)
+            response["status"] = "failed"
+            response["description"] = str(e)
+        self.write(response)
+        self.finish()
+
+
 # ====================================================================================
 # CRUD for FileSystem
 # ====================================================================================
@@ -556,13 +627,15 @@ def setup_handlers(web_app):
     route_fs_file_transfer = url_path_join(
         base_url, "jupyter_fsspec", "files", "transfer"
     )
+    route_fs_sync = url_path_join(base_url, "jupyter_fsspec", "sync")
 
     handlers = [
         (route_fsspec_config, FsspecConfigHandler, dict(fs_manager=fs_manager)),
         (route_files, FileSystemHandler, dict(fs_manager=fs_manager)),
         (route_rename_files, RenameFileHandler, dict(fs_manager=fs_manager)),
         (route_file_actions, FileActionHandler, dict(fs_manager=fs_manager)),
         (route_fs_file_transfer, FileTransferHandler, dict(fs_manager=fs_manager)),
+        (route_fs_sync, FilesystemSyncHandler, dict(fs_manager=fs_manager)),
     ]
 
     web_app.add_handlers(host_pattern, handlers)

jupyter_fsspec/tests/test_api.py

@@ -539,3 +539,70 @@ async def test_upload_download(fs_manager_instance, jp_fetch):
     # downloaded_dirpath = local_root_path + '/some'
     # new_local_items = local_fs.ls(downloaded_dirpath)
     # assert downloaded_dirpath in new_local_items
+
+
+async def xtest_sync_push(fs_manager_instance, jp_fetch):
+    # WIP
+    fs_manager = fs_manager_instance
+    remote_fs_info = fs_manager.get_filesystem_by_protocol("s3")
+    remote_key = remote_fs_info["key"]
+    remote_fs = remote_fs_info["info"]["instance"]
+    remote_root_path = remote_fs_info["info"]["path"]
+    assert remote_fs is not None
+
+    local_fs_info = fs_manager.get_filesystem_by_protocol("local")
+    local_key = local_fs_info["key"]  # noqa: F841
+    local_fs = local_fs_info["info"]["instance"]
+    local_root_path = local_fs_info["info"]["path"]
+    assert local_fs is not None
+
+    push_sync_payload = {
+        "local_path": local_root_path,
+        "remote_path": remote_root_path,
+        "destination_key": remote_key,
+    }
+
+    # calling sync on the remote
+    sync_local_to_remote_response = await jp_fetch(
+        "jupyter_fsspec",
+        "sync",
+        method="POST",
+        params={"key": remote_key},
+        body=json.dumps(push_sync_payload),
+    )
+
+    assert sync_local_to_remote_response.code == 200
+
+
+async def test_sync_pull(fs_manager_instance, jp_fetch):
+    fs_manager = fs_manager_instance
+    remote_fs_info = fs_manager.get_filesystem_by_protocol("s3")
+    remote_key = remote_fs_info["key"]
+    remote_fs = remote_fs_info["info"]["instance"]
+    remote_root_path = remote_fs_info["info"]["path"]
+    assert remote_fs is not None
+
+    local_fs_info = fs_manager.get_filesystem_by_protocol("local")
+    local_key = local_fs_info["key"]  # noqa: F841
+    local_fs = local_fs_info["info"]["instance"]
+    local_root_path = local_fs_info["info"]["path"]
+    assert local_fs is not None
+
+    pull_sync_payload = {
+        "local_path": local_root_path,
+        "remote_path": remote_root_path,
+        "destination_key": remote_key,
+    }
+
+    # calling sync on the remote
+    sync_remote_to_local_response = await jp_fetch(
+        "jupyter_fsspec",
+        "sync",
+        method="GET",
+        params={"key": remote_key},
+        body=json.dumps(pull_sync_payload),
+        allow_nonstandard_methods=True,
+    )
+
+    assert sync_remote_to_local_response.code == 200
+    # assert body["status"] == "success"

src/handler/fileOperations.ts

@@ -366,6 +366,52 @@ export class FsspecModel {
     }
   }
 
+  async sync_push(
+    key: string,
+    remote_path: string,
+    local_path: string
+  ): Promise<any> {
+    try {
+      const reqBody = JSON.stringify({
+        key: key,
+        remote_path,
+        local_path
+      });
+      await requestAPI<any>('sync', {
+        method: 'POST',
+        body: reqBody,
+        headers: {
+          'Content-Type': 'application/json'
+        }
+      });
+    } catch (error) {
+      console.error('Failed to sync local to remote: ', error);
+    }
+  }
+
+  async sync_pull(
+    key: string,
+    remote_path: string,
+    local_path: string
+  ): Promise<any> {
+    try {
+      const reqBody = JSON.stringify({
+        key: key,
+        remote_path,
+        local_path
+      });
+      await requestAPI<any>('sync', {
+        method: 'GET',
+        body: reqBody,
+        headers: {
+          'Content-Type': 'application/json'
+        }
+      });
+    } catch (error) {
+      console.error('Failed to sync remote to local: ', error);
+    }
+  }
+
   // async update(
   //   key: any = 'local%7CSourceDisk%7C.',
   //   item_path = '',