diff --git a/Cargo.lock b/Cargo.lock index f9c763d1..55a7233b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3520,6 +3520,20 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.1.0", + "opendal-core", + "reqwest", +] + [[package]] name = "opendal-service-fs" version = "0.59.1" @@ -5989,6 +6003,7 @@ dependencies = [ "num-derive", "num-traits", "opendal", + "opendal-http-transport-reqwest", "openidconnect", "openssl", "pastey 0.2.3", diff --git a/Cargo.toml b/Cargo.toml index cc6dff02..d3a3d5e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ vendored_openssl = ["openssl/vendored"] enable_mimalloc = ["dep:mimalloc"] s3 = [ "opendal/services-s3", + "dep:opendal-http-transport-reqwest", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", @@ -257,6 +258,7 @@ grass_compiler = { version = "0.13.4", default-features = false } # File are accessed through Apache OpenDAL opendal = { version = "0.59.1", default-features = false, features = ["services-fs"] } +opendal-http-transport-reqwest = { version = "0.59.1", default-features = false, features = ["rustls-no-provider"], optional = true } # For retrieving AWS credentials, including temporary SSO credentials aws-config = { version = "1.12.0", optional = true, default-features = false, features = [ diff --git a/src/storage.rs b/src/storage.rs index 689be302..855a2619 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -77,10 +77,18 @@ pub(crate) fn operator_for_path(path: &str) -> Result = LazyLock::new(|| { + // Storage endpoints are administrator-configured and may be private. + crate::http_client::get_reqwest_client_builder(false).build().expect("Failed to build OpenDAL HTTP client") + }); + pub(super) fn is_uri(path: &str) -> bool { path.starts_with("s3://") } @@ -236,7 +244,9 @@ mod s3 { builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider)); } - Ok(opendal::Operator::new(builder)?) + let http_transport = opendal::HttpTransporter::new(ReqwestTransport::new(HTTP_CLIENT.clone())); + let context = opendal::OperationContext::new().with_http_transport(http_transport); + Ok(opendal::Operator::new(builder)?.with_context(context)) } fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool {