You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/01/11 07:01:39 UTC
[arrow-rs] branch master updated: Upgrade base64 to 0.21 (#3500)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new eecd991d0 Upgrade base64 to 0.21 (#3500)
eecd991d0 is described below
commit eecd991d044b0e5331867bc611bee42cb752f5c8
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Tue Jan 10 23:01:34 2023 -0800
Upgrade base64 to 0.21 (#3500)
* Upgrade base64 to 0.21
* Move to function
* Use prelude
---
arrow-flight/Cargo.toml | 2 +-
arrow-flight/examples/flight_sql_server.rs | 5 ++++-
arrow-flight/src/lib.rs | 4 +++-
arrow-flight/src/sql/client.rs | 4 +++-
object_store/Cargo.toml | 2 +-
object_store/src/azure/client.rs | 4 +++-
object_store/src/azure/credential.rs | 6 ++++--
object_store/src/azure/mod.rs | 7 ++++++-
object_store/src/gcp/credential.rs | 12 ++++--------
parquet/Cargo.toml | 4 ++--
parquet/src/arrow/schema/mod.rs | 6 ++++--
parquet/src/record/api.rs | 5 ++++-
12 files changed, 39 insertions(+), 22 deletions(-)
diff --git a/arrow-flight/Cargo.toml b/arrow-flight/Cargo.toml
index 1664004bd..d357e747c 100644
--- a/arrow-flight/Cargo.toml
+++ b/arrow-flight/Cargo.toml
@@ -33,7 +33,7 @@ arrow-buffer = { version = "30.0.0", path = "../arrow-buffer" }
arrow-cast = { version = "30.0.0", path = "../arrow-cast" }
arrow-ipc = { version = "30.0.0", path = "../arrow-ipc" }
arrow-schema = { version = "30.0.0", path = "../arrow-schema" }
-base64 = { version = "0.20", default-features = false, features = ["std"] }
+base64 = { version = "0.21", default-features = false, features = ["std"] }
tonic = { version = "0.8", default-features = false, features = ["transport", "codegen", "prost"] }
bytes = { version = "1", default-features = false }
prost = { version = "0.11", default-features = false }
diff --git a/arrow-flight/examples/flight_sql_server.rs b/arrow-flight/examples/flight_sql_server.rs
index 54e19a8cc..5aff347e4 100644
--- a/arrow-flight/examples/flight_sql_server.rs
+++ b/arrow-flight/examples/flight_sql_server.rs
@@ -24,6 +24,8 @@ use arrow_flight::{
Action, FlightData, FlightEndpoint, HandshakeRequest, HandshakeResponse, IpcMessage,
Location, SchemaAsIpc, Ticket,
};
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use futures::{stream, Stream};
use prost::Message;
use std::pin::Pin;
@@ -96,7 +98,8 @@ impl FlightSqlService for FlightSqlServiceImpl {
)))?;
}
let base64 = &authorization[basic.len()..];
- let bytes = base64::decode(base64)
+ let bytes = BASE64_STANDARD
+ .decode(base64)
.map_err(|e| status!("authorization not decodable", e))?;
let str = String::from_utf8(bytes)
.map_err(|e| status!("authorization not parsable", e))?;
diff --git a/arrow-flight/src/lib.rs b/arrow-flight/src/lib.rs
index 3057735a6..a44b4b06e 100644
--- a/arrow-flight/src/lib.rs
+++ b/arrow-flight/src/lib.rs
@@ -36,6 +36,8 @@ use arrow_ipc::{convert, writer, writer::EncodedData, writer::IpcWriteOptions};
use arrow_schema::{ArrowError, Schema};
use arrow_ipc::convert::try_schema_from_ipc_buffer;
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use bytes::Bytes;
use std::{
convert::{TryFrom, TryInto},
@@ -265,7 +267,7 @@ impl fmt::Display for Ticket {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Ticket {{")?;
write!(f, " ticket: ")?;
- write!(f, "{}", base64::encode(&self.ticket))
+ write!(f, "{}", BASE64_STANDARD.encode(&self.ticket))
}
}
diff --git a/arrow-flight/src/sql/client.rs b/arrow-flight/src/sql/client.rs
index 679213af0..ecc121d98 100644
--- a/arrow-flight/src/sql/client.rs
+++ b/arrow-flight/src/sql/client.rs
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use bytes::Bytes;
use std::collections::HashMap;
use std::sync::Arc;
@@ -166,7 +168,7 @@ impl FlightSqlServiceClient {
payload: Default::default(),
};
let mut req = tonic::Request::new(stream::iter(vec![cmd]));
- let val = base64::encode(format!("{}:{}", username, password));
+ let val = BASE64_STANDARD.encode(format!("{}:{}", username, password));
let val = format!("Basic {}", val)
.parse()
.map_err(|_| ArrowError::ParseError("Cannot parse header".to_string()))?;
diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml
index e61a127c9..4be6d63fc 100644
--- a/object_store/Cargo.toml
+++ b/object_store/Cargo.toml
@@ -43,7 +43,7 @@ url = "2.2"
walkdir = "2"
# Cloud storage support
-base64 = { version = "0.20", default-features = false, features = ["std"], optional = true }
+base64 = { version = "0.21", default-features = false, features = ["std"], optional = true }
quick-xml = { version = "0.27.0", features = ["serialize"], optional = true }
serde = { version = "1.0", default-features = false, features = ["derive"], optional = true }
serde_json = { version = "1.0", default-features = false, optional = true }
diff --git a/object_store/src/azure/client.rs b/object_store/src/azure/client.rs
index 556a2ad2b..426b3b164 100644
--- a/object_store/src/azure/client.rs
+++ b/object_store/src/azure/client.rs
@@ -25,6 +25,8 @@ use crate::{
BoxStream, ClientOptions, ListResult, ObjectMeta, Path, Result, RetryConfig,
StreamExt,
};
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use bytes::{Buf, Bytes};
use chrono::{DateTime, Utc};
use itertools::Itertools;
@@ -528,7 +530,7 @@ impl BlockList {
for block_id in &self.blocks {
let node = format!(
"\t<Uncommitted>{}</Uncommitted>\n",
- base64::encode(block_id)
+ BASE64_STANDARD.encode(block_id)
);
s.push_str(&node);
}
diff --git a/object_store/src/azure/credential.rs b/object_store/src/azure/credential.rs
index 38e6e64f1..96ff8ce15 100644
--- a/object_store/src/azure/credential.rs
+++ b/object_store/src/azure/credential.rs
@@ -19,6 +19,8 @@ use crate::client::retry::RetryExt;
use crate::client::token::{TemporaryToken, TokenCache};
use crate::util::hmac_sha256;
use crate::RetryConfig;
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use chrono::Utc;
use reqwest::header::ACCEPT;
use reqwest::{
@@ -153,8 +155,8 @@ fn generate_authorization(
key: &str,
) -> String {
let str_to_sign = string_to_sign(h, u, method, account);
- let auth = hmac_sha256(base64::decode(key).unwrap(), str_to_sign);
- format!("SharedKey {}:{}", account, base64::encode(auth))
+ let auth = hmac_sha256(BASE64_STANDARD.decode(key).unwrap(), str_to_sign);
+ format!("SharedKey {}:{}", account, BASE64_STANDARD.encode(auth))
}
fn add_if_exists<'a>(h: &'a HeaderMap, key: &HeaderName) -> &'a str {
diff --git a/object_store/src/azure/mod.rs b/object_store/src/azure/mod.rs
index cbd5a35dc..3bce8e598 100644
--- a/object_store/src/azure/mod.rs
+++ b/object_store/src/azure/mod.rs
@@ -34,6 +34,8 @@ use crate::{
RetryConfig,
};
use async_trait::async_trait;
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use bytes::Bytes;
use chrono::{TimeZone, Utc};
use futures::{stream::BoxStream, StreamExt, TryStreamExt};
@@ -330,7 +332,10 @@ impl CloudMultiPartUploadImpl for AzureMultiPartUpload {
&self.location,
Some(buf.into()),
true,
- &[("comp", "block"), ("blockid", &base64::encode(block_id))],
+ &[
+ ("comp", "block"),
+ ("blockid", &BASE64_STANDARD.encode(block_id)),
+ ],
)
.await?;
diff --git a/object_store/src/gcp/credential.rs b/object_store/src/gcp/credential.rs
index a2a98a39b..cc157dd41 100644
--- a/object_store/src/gcp/credential.rs
+++ b/object_store/src/gcp/credential.rs
@@ -18,17 +18,13 @@
use crate::client::retry::RetryExt;
use crate::client::token::TemporaryToken;
use crate::RetryConfig;
-use base64::engine::fast_portable::FastPortable;
+use base64::prelude::BASE64_URL_SAFE_NO_PAD;
+use base64::Engine;
use reqwest::{Client, Method};
use ring::signature::RsaKeyPair;
use snafu::{ResultExt, Snafu};
use std::time::{Duration, Instant};
-const URL_SAFE_NO_PAD: FastPortable = FastPortable::from(
- &base64::alphabet::URL_SAFE,
- base64::engine::fast_portable::NO_PAD,
-);
-
#[derive(Debug, Snafu)]
pub enum Error {
#[snafu(display("No RSA key found in pem file"))]
@@ -172,7 +168,7 @@ impl OAuthProvider {
)
.context(SignSnafu)?;
- let signature = base64::encode_engine(&sig_bytes, &URL_SAFE_NO_PAD);
+ let signature = BASE64_URL_SAFE_NO_PAD.encode(sig_bytes);
let jwt = [message, signature].join(".");
let body = [
@@ -224,5 +220,5 @@ fn decode_first_rsa_key(private_key_pem: String) -> Result<RsaKeyPair> {
fn b64_encode_obj<T: serde::Serialize>(obj: &T) -> Result<String> {
let string = serde_json::to_string(obj).context(EncodeSnafu)?;
- Ok(base64::encode_engine(string, &URL_SAFE_NO_PAD))
+ Ok(BASE64_URL_SAFE_NO_PAD.encode(string))
}
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
index 2aa744978..ade8d9521 100644
--- a/parquet/Cargo.toml
+++ b/parquet/Cargo.toml
@@ -55,7 +55,7 @@ zstd = { version = "0.12.0", optional = true, default-features = false }
chrono = { version = "0.4.23", default-features = false, features = ["alloc"] }
num = { version = "0.4", default-features = false }
num-bigint = { version = "0.4", default-features = false }
-base64 = { version = "0.20", default-features = false, features = ["std", ], optional = true }
+base64 = { version = "0.21", default-features = false, features = ["std", ], optional = true }
clap = { version = "4", default-features = false, features = ["std", "derive", "env", "help", "error-context", "usage"], optional = true }
serde = { version = "1.0", default-features = false, features = ["derive"], optional = true }
serde_json = { version = "1.0", default-features = false, features = ["std"], optional = true }
@@ -67,7 +67,7 @@ twox-hash = { version = "1.6", default-features = false }
paste = { version = "1.0" }
[dev-dependencies]
-base64 = { version = "0.20", default-features = false, features = ["std"] }
+base64 = { version = "0.21", default-features = false, features = ["std"] }
criterion = { version = "0.4", default-features = false }
snap = { version = "1.0", default-features = false }
tempfile = { version = "3.0", default-features = false }
diff --git a/parquet/src/arrow/schema/mod.rs b/parquet/src/arrow/schema/mod.rs
index f03a6c695..2ca4b7ef8 100644
--- a/parquet/src/arrow/schema/mod.rs
+++ b/parquet/src/arrow/schema/mod.rs
@@ -23,6 +23,8 @@
//!
//! The interfaces for converting arrow schema to parquet schema is coming.
+use base64::prelude::BASE64_STANDARD;
+use base64::Engine;
use std::collections::HashMap;
use std::sync::Arc;
@@ -100,7 +102,7 @@ pub(crate) fn parquet_to_array_schema_and_fields(
/// Try to convert Arrow schema metadata into a schema
fn get_arrow_schema_from_metadata(encoded_meta: &str) -> Result<Schema> {
- let decoded = base64::decode(encoded_meta);
+ let decoded = BASE64_STANDARD.decode(encoded_meta);
match decoded {
Ok(bytes) => {
let slice = if bytes.len() > 8 && bytes[0..4] == [255u8; 4] {
@@ -148,7 +150,7 @@ fn encode_arrow_schema(schema: &Schema) -> String {
len_prefix_schema.append((schema_len as u32).to_le_bytes().to_vec().as_mut());
len_prefix_schema.append(&mut serialized_schema.ipc_message);
- base64::encode(&len_prefix_schema)
+ BASE64_STANDARD.encode(&len_prefix_schema)
}
/// Mutates writer metadata by storing the encoded Arrow schema.
diff --git a/parquet/src/record/api.rs b/parquet/src/record/api.rs
index 2d15e126f..0880e7179 100644
--- a/parquet/src/record/api.rs
+++ b/parquet/src/record/api.rs
@@ -669,6 +669,9 @@ impl Field {
#[cfg(any(feature = "json", test))]
pub fn to_json_value(&self) -> Value {
+ use base64::prelude::BASE64_STANDARD;
+ use base64::Engine;
+
match &self {
Field::Null => Value::Null,
Field::Bool(b) => Value::Bool(*b),
@@ -688,7 +691,7 @@ impl Field {
.unwrap_or(Value::Null),
Field::Decimal(n) => Value::String(convert_decimal_to_string(n)),
Field::Str(s) => Value::String(s.to_owned()),
- Field::Bytes(b) => Value::String(base64::encode(b.data())),
+ Field::Bytes(b) => Value::String(BASE64_STANDARD.encode(b.data())),
Field::Date(d) => Value::String(convert_date_to_string(*d)),
Field::TimestampMillis(ts) => {
Value::String(convert_timestamp_millis_to_string(*ts))