diff --git a/Cargo.lock b/Cargo.lock index b0a36edf..4e63d41b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,6 +320,16 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "atomic" version = "0.5.3" @@ -3100,15 +3110,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "mea" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" -dependencies = [ - "slab", -] - [[package]] name = "memchr" version = "2.8.3" @@ -3437,11 +3438,12 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.58.1" +version = "0.58.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" dependencies = [ "anyhow", + "asyncband", "base64 0.23.1", "bytes", "futures", @@ -3449,7 +3451,6 @@ dependencies = [ "jiff", "log", "md-5", - "mea", "percent-encoding", "quick-xml", "reqsign-core", @@ -3461,6 +3462,20 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.1.0", + "opendal-core", + "reqwest", +] + [[package]] name = "opendal-service-fs" version = "0.58.1" @@ -5889,6 +5904,7 @@ dependencies = [ "num-derive", "num-traits", "opendal", + "opendal-http-transport-reqwest", "openidconnect", "openssl", "pastey 0.2.3", diff --git a/Cargo.toml b/Cargo.toml index 3e187ff3..bfd7317f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ vendored_openssl = ["openssl/vendored"] enable_mimalloc = ["dep:mimalloc"] s3 = [ "opendal/services-s3", + "dep:opendal-http-transport-reqwest", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", @@ -258,6 +259,7 @@ grass_compiler = { version = "0.13.4", default-features = false } # File are accessed through Apache OpenDAL opendal = { version = "0.58.1", default-features = false, features = ["services-fs"] } +opendal-http-transport-reqwest = { version = "0.58.1", default-features = false, features = ["rustls-no-provider"], optional = true } # For retrieving AWS credentials, including temporary SSO credentials aws-config = { version = "1.10.1", optional = true, default-features = false, features = [ diff --git a/src/http_client.rs b/src/http_client.rs index 0831d990..a04c4336 100644 --- a/src/http_client.rs +++ b/src/http_client.rs @@ -14,7 +14,10 @@ use reqwest::{ }; use url::Host; -use crate::{CONFIG, util::is_global}; +use crate::{ + CONFIG, + util::{get_env_bool, is_global}, +}; pub fn make_http_request(method: reqwest::Method, url: &str) -> Result { static INSTANCE: LazyLock = @@ -36,7 +39,7 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder { let mut headers = header::HeaderMap::new(); headers.insert(header::USER_AGENT, header::HeaderValue::from_static("Vaultwarden")); - let redirect_policy = reqwest::redirect::Policy::custom(|attempt| { + let redirect_policy = reqwest::redirect::Policy::custom(move |attempt| { if attempt.previous().len() >= 5 { return attempt.error("Too many redirects"); } @@ -45,7 +48,7 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder { return attempt.error("Invalid host"); }; - if let Err(e) = should_block_host(&host) { + if enforce_block && let Err(e) = should_block_host(&host) { return attempt.error(e); } @@ -59,6 +62,14 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder { .timeout(Duration::from_secs(10)) } +fn dns_prefer_ipv6() -> bool { + // CONFIG may require DNS to initialize, so avoid forcing it during bootstrap. + match LazyLock::get(&CONFIG) { + Some(config) => config.dns_prefer_ipv6(), + None => get_env_bool("DNS_PREFER_IPV6").unwrap_or(false), + } +} + fn should_block_ip(ip: IpAddr) -> bool { if !CONFIG.http_request_block_non_global_ips() { return false; @@ -258,12 +269,8 @@ impl CustomDnsResolver { fn new() -> Arc { TokioResolver::builder(TokioRuntimeProvider::default()) .and_then(|mut builder| { - // Hickory's default since v0.26 is `Ipv6AndIpv4`, which sorts IPv6 first - // This might cause issues on IPv4 only systems or containers - // Unless someone enabled DNS_PREFER_IPV6, use Ipv4AndIpv6, which returns IPv4 first which was our previous default - if !CONFIG.dns_prefer_ipv6() { - builder.options_mut().ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6; - } + // Query both families; the preferred order is applied per lookup below. + builder.options_mut().ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6; builder.build() }) .inspect_err(|e| warn!("Error creating Hickory resolver, falling back to default: {e:?}")) @@ -289,6 +296,14 @@ impl CustomDnsResolver { } } +fn sort_addresses(addresses: &mut [SocketAddr], prefer_ipv6: bool) { + if prefer_ipv6 { + addresses.sort_by_key(SocketAddr::is_ipv4); + } else { + addresses.sort_by_key(SocketAddr::is_ipv6); + } +} + fn pre_resolve(name: &str, enforce_block: bool) -> Result<(), CustomHttpClientError> { let Ok(host) = get_valid_host(name) else { return Err(CustomHttpClientError::Invalid { @@ -320,7 +335,9 @@ impl Resolve for CustomDns { let this = Arc::clone(&self.resolver); Box::pin(async move { let name = name.as_str(); - let results = this.resolve_domain(name, enforce_block).await?; + let mut results = this.resolve_domain(name, enforce_block).await?; + // Recheck after bootstrap so long-lived clients adopt the loaded config. + sort_addresses(&mut results, dns_prefer_ipv6()); if results.is_empty() { warn!("Unable to resolve {name} to any valid IP address"); } @@ -339,10 +356,29 @@ pub(crate) mod aws { }; use reqwest::Client; + use super::get_reqwest_client_builder; + // Adapter that wraps reqwest to be compatible with the AWS SDK #[derive(Debug)] pub(crate) struct AwsReqwestConnector { - pub(crate) client: Client, + client: Client, + } + + impl AwsReqwestConnector { + pub(crate) fn new() -> Self { + let client = get_reqwest_client_builder(false).build().expect("Failed to build AWS HTTP client"); + Self { + client, + } + } + } + + fn connector_error(error: reqwest::Error) -> ConnectorError { + if error.is_timeout() { + ConnectorError::timeout(Box::new(error)) + } else { + ConnectorError::io(Box::new(error)) + } } impl HttpConnector for AwsReqwestConnector { @@ -362,10 +398,10 @@ pub(crate) mod aws { req_builder = req_builder.body(body_bytes.to_vec()); } - let response = req_builder.send().await.map_err(|e| ConnectorError::io(Box::new(e)))?; + let response = req_builder.send().await.map_err(connector_error)?; let status = response.status().into(); - let bytes = response.bytes().await.map_err(|e| ConnectorError::io(Box::new(e)))?; + let bytes = response.bytes().await.map_err(connector_error)?; Ok(HttpResponse::new(status, bytes.into())) }; @@ -391,7 +427,7 @@ pub(crate) mod aws { mod tests { use super::*; use crate::util::is_global_hardcoded; - use std::net::Ipv4Addr; + use std::net::{Ipv4Addr, Ipv6Addr}; use url::Host; // === @@ -404,6 +440,26 @@ mod tests { } } + #[test] + fn dns_setup_does_not_initialize_config() { + assert!(LazyLock::get(&CONFIG).is_none()); + drop(CustomDns::instance(false)); + assert!(LazyLock::get(&CONFIG).is_none()); + } + + #[test] + fn dns_preference_orders_addresses() { + let ipv4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); + let ipv6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0); + let mut addresses = [ipv6, ipv4]; + + sort_addresses(&mut addresses, false); + assert_eq!(addresses, [ipv4, ipv6]); + + sort_addresses(&mut addresses, true); + assert_eq!(addresses, [ipv6, ipv4]); + } + #[test] fn dotted_decimal_loopback_normalizes() { let ip = parse_to_ip("127.0.0.1").unwrap(); diff --git a/src/storage.rs b/src/storage.rs index 689be302..32562a0d 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -77,10 +77,18 @@ pub(crate) fn operator_for_path(path: &str) -> Result = LazyLock::new(|| { + // Storage endpoints are administrator-configured and may be private. + crate::http_client::get_reqwest_client_builder(false).build().expect("Failed to build OpenDAL HTTP client") + }); + pub(super) fn is_uri(path: &str) -> bool { path.starts_with("s3://") } @@ -177,12 +185,7 @@ mod s3 { let chain = DEFAULT_CREDENTIAL_CHAIN .get_or_init(|| { - let reqwest_client = reqwest::Client::builder().build().unwrap(); - let connector = AwsReqwestConnector { - client: reqwest_client, - }; - - let conf = ProviderConfig::default().with_http_client(connector); + let conf = ProviderConfig::default().with_http_client(AwsReqwestConnector::new()); DefaultCredentialsChain::builder().configure(conf).build() }) @@ -236,7 +239,9 @@ mod s3 { builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider)); } - Ok(opendal::Operator::new(builder)?) + let http_transport = opendal::HttpTransporter::new(ReqwestTransport::new(HTTP_CLIENT.clone())); + let context = opendal::OperationContext::new().with_http_transport(http_transport); + Ok(opendal::Operator::new(builder)?.with_context(context)) } fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool {