WIP: Upgrade DataFusion to arrow-rs/parquet 58.0.0 / object_store 13.0.0

Cargo.toml

@@ -164,7 +164,7 @@ itertools = "0.14"
 liblzma = { version = "0.4.4", features = ["static"] }
 log = "^0.4"
 num-traits = { version = "0.2" }
-object_store = { version = "0.12.4", default-features = false }
+object_store = { version = "0.13.1", default-features = false }
 parking_lot = "0.12"
 parquet = { version = "57.2.0", default-features = false, features = [
     "arrow",
@@ -275,3 +275,32 @@ incremental = false
 inherits = "release"
 debug = true
 strip = false
+
+## Temporary arrow-rs patch until 58 is released
+
+[patch.crates-io]
+arrow = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-array = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-cast = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-data = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-ipc = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-select = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-string = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-ord = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+parquet = { git = "https://github.com/apache/arrow-rs.git", branch = "main" }
+
+#arrow = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow" }
+#arrow-array = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-array" }
+#arrow-buffer = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-buffer" }
+#arrow-cast = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-cast" }
+#arrow-data = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-data" }
+#arrow-ipc = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-ipc" }
+#arrow-schema = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-schema" }
+#arrow-select = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-select" }
+#arrow-string = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-string" }
+#arrow-ord = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-ord" }
+#arrow-flight = { path= "/Users/andrewlamb/Software/arrow-rs2/arrow-flight" }
+#parquet = { path= "/Users/andrewlamb/Software/arrow-rs2/parquet" }

datafusion-cli/src/exec.rs

@@ -521,6 +521,7 @@ mod tests {
     use datafusion::common::plan_err;
 
     use datafusion::prelude::SessionContext;
+    use datafusion_common::assert_contains;
     use url::Url;
 
     async fn create_external_table_test(location: &str, sql: &str) -> Result<()> {
@@ -714,25 +715,24 @@ mod tests {
         let err = create_external_table_test(location, &sql)
             .await
             .unwrap_err();
-        assert!(err.to_string().contains("os error 2"));
+        assert_contains!(err.to_string(), "os error 2");
 
         // for service_account_key
         let sql = format!(
             "CREATE EXTERNAL TABLE test STORED AS PARQUET OPTIONS('gcp.service_account_key' '{service_account_key}') LOCATION '{location}'"
         );
         let err = create_external_table_test(location, &sql)
             .await
-            .unwrap_err()
-            .to_string();
-        assert!(err.contains("No RSA key found in pem file"), "{err}");
+            .unwrap_err();
+        assert_contains!(err.to_string(), "Error reading pem file: no items found");
 
         // for application_credentials_path
         let sql = format!("CREATE EXTERNAL TABLE test STORED AS PARQUET
             OPTIONS('gcp.application_credentials_path' '{application_credentials_path}') LOCATION '{location}'");
         let err = create_external_table_test(location, &sql)
             .await
             .unwrap_err();
-        assert!(err.to_string().contains("os error 2"));
+        assert_contains!(err.to_string(), "os error 2");
 
         Ok(())
     }

datafusion-cli/src/main.rs

@@ -617,8 +617,8 @@ mod tests {
         | filename                          | file_size_bytes | metadata_size_bytes | hits | extra            |
         +-----------------------------------+-----------------+---------------------+------+------------------+
         | alltypes_plain.parquet            | 1851            | 8882                | 2    | page_index=false |
-        | alltypes_tiny_pages.parquet       | 454233          | 269266              | 2    | page_index=true  |
-        | lz4_raw_compressed_larger.parquet | 380836          | 1347                | 2    | page_index=false |
+        | alltypes_tiny_pages.parquet       | 454233          | 269074              | 2    | page_index=true  |
+        | lz4_raw_compressed_larger.parquet | 380836          | 1339                | 2    | page_index=false |
         +-----------------------------------+-----------------+---------------------+------+------------------+
         ");
 
@@ -648,8 +648,8 @@ mod tests {
         | filename                          | file_size_bytes | metadata_size_bytes | hits | extra            |
         +-----------------------------------+-----------------+---------------------+------+------------------+
         | alltypes_plain.parquet            | 1851            | 8882                | 5    | page_index=false |
-        | alltypes_tiny_pages.parquet       | 454233          | 269266              | 2    | page_index=true  |
-        | lz4_raw_compressed_larger.parquet | 380836          | 1347                | 3    | page_index=false |
+        | alltypes_tiny_pages.parquet       | 454233          | 269074              | 2    | page_index=true  |
+        | lz4_raw_compressed_larger.parquet | 380836          | 1339                | 3    | page_index=false |
         +-----------------------------------+-----------------+---------------------+------+------------------+
         ");