Browse Source

Merge e51126c823 into fdc156b247

pull/7639/merge
Chase Douglas 6 days ago
committed by GitHub
parent
commit
3e7a1c1287
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 40
      Cargo.lock
  2. 2
      Cargo.toml
  3. 84
      src/http_client.rs
  4. 19
      src/storage.rs

40
Cargo.lock

@ -320,6 +320,16 @@ dependencies = [
"syn 3.0.3",
]
[[package]]
name = "asyncband"
version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0"
dependencies = [
"hashbrown 0.17.1",
"slab",
]
[[package]]
name = "atomic"
version = "0.5.3"
@ -3100,15 +3110,6 @@ dependencies = [
"digest 0.11.3",
]
[[package]]
name = "mea"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0"
dependencies = [
"slab",
]
[[package]]
name = "memchr"
version = "2.8.3"
@ -3437,11 +3438,12 @@ dependencies = [
[[package]]
name = "opendal-core"
version = "0.58.1"
version = "0.58.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed"
checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7"
dependencies = [
"anyhow",
"asyncband",
"base64 0.23.1",
"bytes",
"futures",
@ -3449,7 +3451,6 @@ dependencies = [
"jiff",
"log",
"md-5",
"mea",
"percent-encoding",
"quick-xml",
"reqsign-core",
@ -3461,6 +3462,20 @@ dependencies = [
"web-time",
]
[[package]]
name = "opendal-http-transport-reqwest"
version = "0.58.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c"
dependencies = [
"bytes",
"futures",
"http 1.5.0",
"http-body 1.1.0",
"opendal-core",
"reqwest",
]
[[package]]
name = "opendal-service-fs"
version = "0.58.1"
@ -5889,6 +5904,7 @@ dependencies = [
"num-derive",
"num-traits",
"opendal",
"opendal-http-transport-reqwest",
"openidconnect",
"openssl",
"pastey 0.2.3",

2
Cargo.toml

@ -41,6 +41,7 @@ vendored_openssl = ["openssl/vendored"]
enable_mimalloc = ["dep:mimalloc"]
s3 = [
"opendal/services-s3",
"dep:opendal-http-transport-reqwest",
"dep:aws-config",
"dep:aws-credential-types",
"dep:aws-smithy-runtime-api",
@ -258,6 +259,7 @@ grass_compiler = { version = "0.13.4", default-features = false }
# File are accessed through Apache OpenDAL
opendal = { version = "0.58.1", default-features = false, features = ["services-fs"] }
opendal-http-transport-reqwest = { version = "0.58.1", default-features = false, features = ["rustls-no-provider"], optional = true }
# For retrieving AWS credentials, including temporary SSO credentials
aws-config = { version = "1.10.1", optional = true, default-features = false, features = [

84
src/http_client.rs

@ -14,7 +14,10 @@ use reqwest::{
};
use url::Host;
use crate::{CONFIG, util::is_global};
use crate::{
CONFIG,
util::{get_env_bool, is_global},
};
pub fn make_http_request(method: reqwest::Method, url: &str) -> Result<reqwest::RequestBuilder, crate::Error> {
static INSTANCE: LazyLock<Client> =
@ -36,7 +39,7 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder {
let mut headers = header::HeaderMap::new();
headers.insert(header::USER_AGENT, header::HeaderValue::from_static("Vaultwarden"));
let redirect_policy = reqwest::redirect::Policy::custom(|attempt| {
let redirect_policy = reqwest::redirect::Policy::custom(move |attempt| {
if attempt.previous().len() >= 5 {
return attempt.error("Too many redirects");
}
@ -45,7 +48,7 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder {
return attempt.error("Invalid host");
};
if let Err(e) = should_block_host(&host) {
if enforce_block && let Err(e) = should_block_host(&host) {
return attempt.error(e);
}
@ -59,6 +62,14 @@ pub fn get_reqwest_client_builder(enforce_block: bool) -> ClientBuilder {
.timeout(Duration::from_secs(10))
}
fn dns_prefer_ipv6() -> bool {
// CONFIG may require DNS to initialize, so avoid forcing it during bootstrap.
match LazyLock::get(&CONFIG) {
Some(config) => config.dns_prefer_ipv6(),
None => get_env_bool("DNS_PREFER_IPV6").unwrap_or(false),
}
}
fn should_block_ip(ip: IpAddr) -> bool {
if !CONFIG.http_request_block_non_global_ips() {
return false;
@ -258,12 +269,8 @@ impl CustomDnsResolver {
fn new() -> Arc<Self> {
TokioResolver::builder(TokioRuntimeProvider::default())
.and_then(|mut builder| {
// Hickory's default since v0.26 is `Ipv6AndIpv4`, which sorts IPv6 first
// This might cause issues on IPv4 only systems or containers
// Unless someone enabled DNS_PREFER_IPV6, use Ipv4AndIpv6, which returns IPv4 first which was our previous default
if !CONFIG.dns_prefer_ipv6() {
builder.options_mut().ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6;
}
// Query both families; the preferred order is applied per lookup below.
builder.options_mut().ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6;
builder.build()
})
.inspect_err(|e| warn!("Error creating Hickory resolver, falling back to default: {e:?}"))
@ -289,6 +296,14 @@ impl CustomDnsResolver {
}
}
fn sort_addresses(addresses: &mut [SocketAddr], prefer_ipv6: bool) {
if prefer_ipv6 {
addresses.sort_by_key(SocketAddr::is_ipv4);
} else {
addresses.sort_by_key(SocketAddr::is_ipv6);
}
}
fn pre_resolve(name: &str, enforce_block: bool) -> Result<(), CustomHttpClientError> {
let Ok(host) = get_valid_host(name) else {
return Err(CustomHttpClientError::Invalid {
@ -320,7 +335,9 @@ impl Resolve for CustomDns {
let this = Arc::clone(&self.resolver);
Box::pin(async move {
let name = name.as_str();
let results = this.resolve_domain(name, enforce_block).await?;
let mut results = this.resolve_domain(name, enforce_block).await?;
// Recheck after bootstrap so long-lived clients adopt the loaded config.
sort_addresses(&mut results, dns_prefer_ipv6());
if results.is_empty() {
warn!("Unable to resolve {name} to any valid IP address");
}
@ -339,10 +356,29 @@ pub(crate) mod aws {
};
use reqwest::Client;
use super::get_reqwest_client_builder;
// Adapter that wraps reqwest to be compatible with the AWS SDK
#[derive(Debug)]
pub(crate) struct AwsReqwestConnector {
pub(crate) client: Client,
client: Client,
}
impl AwsReqwestConnector {
pub(crate) fn new() -> Self {
let client = get_reqwest_client_builder(false).build().expect("Failed to build AWS HTTP client");
Self {
client,
}
}
}
fn connector_error(error: reqwest::Error) -> ConnectorError {
if error.is_timeout() {
ConnectorError::timeout(Box::new(error))
} else {
ConnectorError::io(Box::new(error))
}
}
impl HttpConnector for AwsReqwestConnector {
@ -362,10 +398,10 @@ pub(crate) mod aws {
req_builder = req_builder.body(body_bytes.to_vec());
}
let response = req_builder.send().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
let response = req_builder.send().await.map_err(connector_error)?;
let status = response.status().into();
let bytes = response.bytes().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
let bytes = response.bytes().await.map_err(connector_error)?;
Ok(HttpResponse::new(status, bytes.into()))
};
@ -391,7 +427,7 @@ pub(crate) mod aws {
mod tests {
use super::*;
use crate::util::is_global_hardcoded;
use std::net::Ipv4Addr;
use std::net::{Ipv4Addr, Ipv6Addr};
use url::Host;
// ===
@ -404,6 +440,26 @@ mod tests {
}
}
#[test]
fn dns_setup_does_not_initialize_config() {
assert!(LazyLock::get(&CONFIG).is_none());
drop(CustomDns::instance(false));
assert!(LazyLock::get(&CONFIG).is_none());
}
#[test]
fn dns_preference_orders_addresses() {
let ipv4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0);
let ipv6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0);
let mut addresses = [ipv6, ipv4];
sort_addresses(&mut addresses, false);
assert_eq!(addresses, [ipv4, ipv6]);
sort_addresses(&mut addresses, true);
assert_eq!(addresses, [ipv6, ipv4]);
}
#[test]
fn dotted_decimal_loopback_normalizes() {
let ip = parse_to_ip("127.0.0.1").unwrap();

19
src/storage.rs

@ -77,10 +77,18 @@ pub(crate) fn operator_for_path(path: &str) -> Result<opendal::Operator, crate::
#[cfg(s3)]
mod s3 {
use std::sync::LazyLock;
use opendal_http_transport_reqwest::ReqwestTransport;
use reqwest::Url;
use crate::error::Error;
static HTTP_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
// Storage endpoints are administrator-configured and may be private.
crate::http_client::get_reqwest_client_builder(false).build().expect("Failed to build OpenDAL HTTP client")
});
pub(super) fn is_uri(path: &str) -> bool {
path.starts_with("s3://")
}
@ -177,12 +185,7 @@ mod s3 {
let chain = DEFAULT_CREDENTIAL_CHAIN
.get_or_init(|| {
let reqwest_client = reqwest::Client::builder().build().unwrap();
let connector = AwsReqwestConnector {
client: reqwest_client,
};
let conf = ProviderConfig::default().with_http_client(connector);
let conf = ProviderConfig::default().with_http_client(AwsReqwestConnector::new());
DefaultCredentialsChain::builder().configure(conf).build()
})
@ -236,7 +239,9 @@ mod s3 {
builder.credential_provider_chain(ProvideCredentialChain::new().push(OpenDALS3CredentialProvider));
}
Ok(opendal::Operator::new(builder)?)
let http_transport = opendal::HttpTransporter::new(ReqwestTransport::new(HTTP_CLIENT.clone()));
let context = opendal::OperationContext::new().with_http_transport(http_transport);
Ok(opendal::Operator::new(builder)?.with_context(context))
}
fn uri_has_option(uri: &opendal::OperatorUri, names: &[&str]) -> bool {

Loading…
Cancel
Save