Browse Source

storage: route OpenDAL through HTTP client

OpenDAL 0.58 requires applications to provide an HTTP transport. Its
default installer creates a standalone client, bypassing Vaultwarden
DNS, redirect, proxy, timeout, and request configuration.

Build the client through the internal HTTP interface and inject it into
OpenDAL's public reqwest transport.
pull/7639/head
Chase Douglas 1 month ago
committed by BlackDex
parent
commit
6ee632d323
No known key found for this signature in database GPG Key ID: 58C80A2AA6C765E1
  1. 15
      Cargo.lock
  2. 2
      Cargo.toml
  3. 12
      src/storage.rs

15
Cargo.lock

@ -3520,6 +3520,20 @@ dependencies = [
"web-time", "web-time",
] ]
[[package]]
name = "opendal-http-transport-reqwest"
version = "0.59.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06"
dependencies = [
"bytes",
"futures",
"http 1.5.0",
"http-body 1.1.0",
"opendal-core",
"reqwest",
]
[[package]] [[package]]
name = "opendal-service-fs" name = "opendal-service-fs"
version = "0.59.1" version = "0.59.1"
@ -5989,6 +6003,7 @@ dependencies = [
"num-derive", "num-derive",
"num-traits", "num-traits",
"opendal", "opendal",
"opendal-http-transport-reqwest",
"openidconnect", "openidconnect",
"openssl", "openssl",
"pastey 0.2.3", "pastey 0.2.3",

2
Cargo.toml

@ -40,6 +40,7 @@ vendored_openssl = ["openssl/vendored"]
enable_mimalloc = ["dep:mimalloc"] enable_mimalloc = ["dep:mimalloc"]
s3 = [ s3 = [
"opendal/services-s3", "opendal/services-s3",
"dep:opendal-http-transport-reqwest",
"dep:aws-config", "dep:aws-config",
"dep:aws-credential-types", "dep:aws-credential-types",
"dep:aws-smithy-runtime-api", "dep:aws-smithy-runtime-api",
@ -257,6 +258,7 @@ grass_compiler = { version = "0.13.4", default-features = false }
# File are accessed through Apache OpenDAL # File are accessed through Apache OpenDAL
opendal = { version = "0.59.1", default-features = false, features = ["services-fs"] } opendal = { version = "0.59.1", default-features = false, features = ["services-fs"] }
opendal-http-transport-reqwest = { version = "0.59.1", default-features = false, features = ["rustls-no-provider"], optional = true }
# For retrieving AWS credentials, including temporary SSO credentials # For retrieving AWS credentials, including temporary SSO credentials
aws-config = { version = "1.12.0", optional = true, default-features = false, features = [ aws-config = { version = "1.12.0", optional = true, default-features = false, features = [

12
src/storage.rs

@ -77,10 +77,18 @@ pub(crate) fn operator_for_path(path: &str) -> Result<opendal::Operator, crate::
#[cfg(s3)] #[cfg(s3)]
mod s3 { mod s3 {
use std::sync::LazyLock;
use opendal_http_transport_reqwest::ReqwestTransport;
use reqwest::Url; use reqwest::Url;
use crate::error::Error; use crate::error::Error;
static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
// Storage endpoints are administrator-configured and may be private.
crate::http_client::get_reqwest_client_builder(false).build().expect("Failed to build OpenDAL HTTP client")
});
pub(super) fn is_uri(path: &str) -> bool { pub(super) fn is_uri(path: &str) -> bool {
path.starts_with("s3://") path.starts_with("s3://")
} }
@ -236,7 +244,9 @@ mod s3 {
builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider)); builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider));
} }
Ok(opendal::Operator::new(builder)?) let http_transport = opendal::HttpTransporter::new(ReqwestTransport::new(HTTP_CLIENT.clone()));
let context = opendal::OperationContext::new().with_http_transport(http_transport);
Ok(opendal::Operator::new(builder)?.with_context(context))
} }
fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool { fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool {

Loading…
Cancel
Save