Merge branch 'main' into join-where

pola-rs · Oct 3, 2024 · bf6f820 · bf6f820
2 parents 265de82 + 64c9076
commit bf6f820
Show file tree

Hide file tree

Showing 13 changed files with 142 additions and 40 deletions.
diff --git a/.github/workflows/flake.yml b/.github/workflows/flake.yml
@@ -5,6 +5,7 @@ on:
     - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00
 permissions:
   contents: write
+  pull-requests: write
 
 jobs:
   lockfile:
@@ -17,5 +18,4 @@ jobs:
       - name: Update flake.lock
         uses: DeterminateSystems/update-flake-lock@v24
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
           pr-title: "chore: update flake.lock"
diff --git a/NEWS.md b/NEWS.md
@@ -17,6 +17,7 @@
 - New argument `strict` in `$drop()` to determine whether unknown column names
   should trigger an error (#1220).
 - New method `$to_dummies()` for `DataFrame` (#1225).
+- New argument `include_file_paths` in `pl_scan_csv()` and `pl_read_csv()` (#1235).
 - New method `$join_where()` for `DataFrame` and `LazyFrame` to perform
   inequality joins (#1237).
 

diff --git a/R/extendr-wrappers.R b/R/extendr-wrappers.R
@@ -92,7 +92,7 @@ concat_df_horizontal <- function(l) .Call(wrap__concat_df_horizontal, l)
 
 concat_series <- function(l, rechunk, to_supertypes) .Call(wrap__concat_series, l, rechunk, to_supertypes)
 
-new_from_csv <- function(path, has_header, separator, comment_prefix, quote_char, skip_rows, dtypes, null_values, ignore_errors, cache, infer_schema_length, n_rows, encoding, low_memory, rechunk, skip_rows_after_header, row_index_name, row_index_offset, try_parse_dates, eol_char, raise_if_empty, truncate_ragged_lines) .Call(wrap__new_from_csv, path, has_header, separator, comment_prefix, quote_char, skip_rows, dtypes, null_values, ignore_errors, cache, infer_schema_length, n_rows, encoding, low_memory, rechunk, skip_rows_after_header, row_index_name, row_index_offset, try_parse_dates, eol_char, raise_if_empty, truncate_ragged_lines)
+new_from_csv <- function(path, has_header, separator, comment_prefix, quote_char, skip_rows, dtypes, null_values, ignore_errors, cache, infer_schema_length, n_rows, encoding, low_memory, rechunk, skip_rows_after_header, row_index_name, row_index_offset, try_parse_dates, eol_char, raise_if_empty, truncate_ragged_lines, include_file_paths) .Call(wrap__new_from_csv, path, has_header, separator, comment_prefix, quote_char, skip_rows, dtypes, null_values, ignore_errors, cache, infer_schema_length, n_rows, encoding, low_memory, rechunk, skip_rows_after_header, row_index_name, row_index_offset, try_parse_dates, eol_char, raise_if_empty, truncate_ragged_lines, include_file_paths)
 
 import_arrow_ipc <- function(path, n_rows, cache, rechunk, row_name, row_index, hive_partitioning, hive_schema, try_parse_hive_dates, include_file_paths) .Call(wrap__import_arrow_ipc, path, n_rows, cache, rechunk, row_name, row_index, hive_partitioning, hive_schema, try_parse_hive_dates, include_file_paths)
 

diff --git a/R/io_csv.R b/R/io_csv.R
@@ -65,6 +65,8 @@
 #' @param truncate_ragged_lines Truncate lines that are longer than the schema.
 #' @param reuse_downloaded If `TRUE`(default) and a URL was provided, cache the
 #' downloaded files in session for an easy reuse.
+#' @param include_file_paths Include the path of the source file(s) as a column
+#' with this name.
 #' @return [LazyFrame][LazyFrame_class]
 #' @examples
 #' my_file = tempfile()
@@ -97,7 +99,8 @@ pl_scan_csv = function(
     eol_char = "\n",
     raise_if_empty = TRUE,
     truncate_ragged_lines = FALSE,
-    reuse_downloaded = TRUE) {
+    reuse_downloaded = TRUE,
+    include_file_paths = NULL) {
   # capture all args and modify some to match lower level function
   args = as.list(environment())
 
@@ -181,7 +184,8 @@ pl_read_csv = function(
     eol_char = "\n",
     raise_if_empty = TRUE,
     truncate_ragged_lines = FALSE,
-    reuse_downloaded = TRUE) {
+    reuse_downloaded = TRUE,
+    include_file_paths = NULL) {
   .args = as.list(environment())
   result({
     do.call(pl$scan_csv, .args)$collect()

diff --git a/flake.lock b/flake.lock
diff --git a/man/IO_read_csv.Rd b/man/IO_read_csv.Rd
diff --git a/man/IO_read_parquet.Rd b/man/IO_read_parquet.Rd
diff --git a/man/IO_scan_csv.Rd b/man/IO_scan_csv.Rd
diff --git a/man/IO_scan_parquet.Rd b/man/IO_scan_parquet.Rd
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
@@ -42,14 +42,14 @@ extendr-api = { git = "https://github.com/extendr/extendr", rev = "1895bfc8ee223
 ] }
 flume = "0.11.0"
 indenter = "0.3.3"
-ipc-channel = "0.18.1"
-once_cell = "1.20.0"
+ipc-channel = "0.18.3"
+once_cell = "1.20.1"
 rayon = "1.10.0"
 serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "*"
 smartstring = "1.0.1"
 state = "0.6.0"
-thiserror = "1.0.63"
+thiserror = "1.0.64"
 polars-core = { git = "https://github.com/pola-rs/polars.git", rev = "54218e7e35e3defd4b0801e820c56eea6b91e525", default-features = false }
 polars-lazy = { git = "https://github.com/pola-rs/polars.git", rev = "54218e7e35e3defd4b0801e820c56eea6b91e525", default-features = false }
 either = "1"