✨ Recursively upload via context if file path is directory (#725)

* Create sync command client for WSFS * Formatting * Update `CHANGELOG.md` --------- Co-authored-by: Ivan Trusov <[email protected]>
databrickslabs · Apr 11, 2023 · cdb1fe7 · cdb1fe7
1 parent 1fd8ca9
commit cdb1fe7
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add `sync workspace` subcommand for syncing local files to Databricks and watching for changes.
 
+### Fixed
+
+- Recursively upload required files when a directory is passed as a task parameter with `dbx execute` and `--upload-via-context`.
+
 ## [0.8.10] - 2023-03-21
 
 

diff --git a/dbx/api/context.py b/dbx/api/context.py
@@ -169,16 +169,22 @@ def remove_dir(self, _dir: str):
         self._client.execute_command(command, verbose=False)
 
     def upload_file(self, file: Path, prefix_dir: str) -> str:
-        _contents = file.read_bytes()
-        contents = b64encode(_contents)
-        command = f"""
-        from pathlib import Path
-        from base64 import b64decode
-        DBX_UPLOAD_CONTENTS = b64decode({contents})
-        file_path = Path("{prefix_dir}") / "{file.as_posix()}"
-        if not file_path.parent.exists():
-            file_path.parent.mkdir(parents=True)
-        file_path.write_bytes(DBX_UPLOAD_CONTENTS)
-        print(file_path)
-        """
-        return self._client.execute_command(command, verbose=False)
+        files = list(file.rglob("*")) if file.is_dir() else [file]
+        result_data = []
+
+        for _file in files:
+            _contents = _file.read_bytes()
+            contents = b64encode(_contents)
+            command = f"""
+            from pathlib import Path
+            from base64 import b64decode
+            DBX_UPLOAD_CONTENTS = b64decode({contents})
+            file_path = Path("{prefix_dir}") / "{_file.as_posix()}"
+            if not file_path.parent.exists():
+                file_path.parent.mkdir(parents=True)
+            file_path.write_bytes(DBX_UPLOAD_CONTENTS)
+            print(file_path)
+            """
+            result_data.append(self._client.execute_command(command, verbose=False))
+
+        return "\n".join(result_data)