topaLE
/
vaultwarden
mirror of https://github.com/dani-garcia/vaultwarden
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Use existing reqwest client for AWS S3 requests (#5917) 

This removes a lot of duplicate client dependency bloat for roughly
equivalent functionality.

Co-authored-by: Mathijs van Veluw <black.dex@gmail.com>
pull/6004/head
Chase Douglas 4 months ago
committed by GitHub
parent
a545636ee5
commit
6b9333b33e
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
4 changed files with 89 additions and 202 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 207
      Cargo.lock
  2. 8
      Cargo.toml
  3. 18
      src/config.rs
  4. 58
      src/http_client.rs

207
Cargo.lock
View File

@ -184,7 +184,7 @@ dependencies = [
"futures-lite",
"parking",
"polling",
"rustix 1.0.7",
"rustix",
"slab",
"tracing",
"windows-sys 0.59.0",
@ -216,7 +216,7 @@ dependencies = [
"cfg-if",
"event-listener 5.4.0",
"futures-lite",
"rustix 1.0.7",
"rustix",
"tracing",
]
@ -232,7 +232,7 @@ dependencies = [
"cfg-if",
"futures-core",
"futures-io",
"rustix 1.0.7",
"rustix",
"signal-hook-registry",
"slab",
"windows-sys 0.59.0",
@ -373,29 +373,6 @@ dependencies = [
"zeroize",
]
[[package]]
name = "aws-lc-rs"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7"
dependencies = [
"aws-lc-sys",
"zeroize",
]
[[package]]
name = "aws-lc-sys"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079"
dependencies = [
"bindgen",
"cc",
"cmake",
"dunce",
"fs_extra",
]
[[package]]
name = "aws-runtime"
version = "1.5.8"
@ -540,29 +517,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws-smithy-http-client"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
"aws-smithy-types",
"h2",
"http 1.3.1",
"hyper 1.6.0",
"hyper-rustls",
"hyper-util",
"pin-project-lite",
"rustls 0.23.28",
"rustls-native-certs",
"rustls-pki-types",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-json"
version = "0.61.4"
@ -599,7 +553,6 @@ checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-client",
"aws-smithy-observability",
"aws-smithy-runtime-api",
"aws-smithy-types",
@ -757,29 +710,6 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
[[package]]
name = "bindgen"
version = "0.69.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"itertools",
"lazy_static",
"lazycell",
"log",
"prettyplease",
"proc-macro2",
"quote",
"regex",
"rustc-hash 1.1.0",
"shlex",
"syn",
"which 4.4.2",
]
[[package]]
name = "bitflags"
version = "2.9.1"
@ -937,15 +867,6 @@ dependencies = [
"shlex",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom 7.1.3",
]
[[package]]
name = "cfg-if"
version = "1.0.1"
@ -1014,26 +935,6 @@ dependencies = [
"inout",
]
[[package]]
name = "clang-sys"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "cmake"
version = "0.1.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
dependencies = [
"cc",
]
[[package]]
name = "codemap"
version = "0.1.3"
@ -1519,12 +1420,6 @@ dependencies = [
"syn",
]
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "either"
version = "1.15.0"
@ -1698,12 +1593,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs_extra"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
version = "0.3.31"
@ -2459,15 +2348,6 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "itertools"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.15"
@ -2553,12 +2433,6 @@ dependencies = [
"spin",
]
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lettre"
version = "0.11.17"
@ -2597,16 +2471,6 @@ version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "libloading"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
"windows-targets 0.53.2",
]
[[package]]
name = "libm"
version = "0.2.15"
@ -2634,12 +2498,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
@ -3420,7 +3278,7 @@ dependencies = [
"concurrent-queue",
"hermit-abi",
"pin-project-lite",
"rustix 1.0.7",
"rustix",
"tracing",
"windows-sys 0.59.0",
]
@ -3465,16 +3323,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "prettyplease"
version = "0.2.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.95"
@ -3564,7 +3412,7 @@ dependencies = [
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustc-hash",
"rustls 0.23.28",
"socket2",
"thiserror 2.0.12",
@ -3584,7 +3432,7 @@ dependencies = [
"lru-slab",
"rand 0.9.1",
"ring",
"rustc-hash 2.1.1",
"rustc-hash",
"rustls 0.23.28",
"rustls-pki-types",
"slab",
@ -4075,12 +3923,6 @@ version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc-hash"
version = "2.1.1"
@ -4096,19 +3938,6 @@ dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]]
name = "rustix"
version = "1.0.7"
@ -4118,7 +3947,7 @@ dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.9.4",
"linux-raw-sys",
"windows-sys 0.59.0",
]
@ -4140,7 +3969,6 @@ version = "0.23.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643"
dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
@ -4197,7 +4025,6 @@ version = "0.103.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@ -4649,7 +4476,7 @@ dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
"rustix 1.0.7",
"rustix",
"windows-sys 0.59.0",
]
@ -5166,6 +4993,7 @@ dependencies = [
"argon2",
"aws-config",
"aws-credential-types",
"aws-smithy-runtime-api",
"bigdecimal",
"bytes",
"cached",
@ -5190,6 +5018,7 @@ dependencies = [
"handlebars",
"hickory-resolver",
"html5gum",
"http 1.3.1",
"job_scheduler_ng",
"jsonwebtoken",
"lettre",
@ -5228,7 +5057,7 @@ dependencies = [
"url",
"uuid",
"webauthn-rs",
"which 8.0.0",
"which",
"yubico_ng",
]
@ -5416,18 +5245,6 @@ dependencies = [
"rustls-pki-types",
]
[[package]]
name = "which"
version = "4.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
dependencies = [
"either",
"home",
"once_cell",
"rustix 0.38.44",
]
[[package]]
name = "which"
version = "8.0.0"
@ -5435,7 +5252,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d"
dependencies = [
"env_home",
"rustix 1.0.7",
"rustix",
"winsafe",
]

8
Cargo.toml
View File

@ -32,7 +32,7 @@ enable_mimalloc = ["dep:mimalloc"]
# You also need to set an env variable `QUERY_LOGGER=1` to fully activate this so you do not have to re-compile
# if you want to turn off the logging for a specific run.
query_logger = ["dep:diesel_logger"]
s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:anyhow", "dep:reqsign"]
s3 = ["opendal/services-s3", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-runtime-api", "dep:anyhow", "dep:http", "dep:reqsign"]
# Enable unstable features, requires nightly
# Currently only used to enable rusts official ip support
@ -180,12 +180,14 @@ rpassword = "7.4.0"
grass_compiler = { version = "0.13.4", default-features = false }
# File are accessed through Apache OpenDAL
opendal = { version = "0.53.3", features = ["services-fs"] }
opendal = { version = "0.53.3", features = ["services-fs"], default-features = false }
# For retrieving AWS credentials, including temporary SSO credentials
anyhow = { version = "1.0.98", optional = true }
aws-config = { version = "1.8.0", features = ["behavior-version-latest"], optional = true }
aws-config = { version = "1.8.0", features = ["behavior-version-latest", "rt-tokio", "credentials-process", "sso"], default-features = false, optional = true }
aws-credential-types = { version = "1.2.3", optional = true }
aws-smithy-runtime-api = { version = "1.8.1", optional = true }
http = { version = "1.3.1", optional = true }
reqsign = { version = "0.16.4", optional = true }
# Strip debuginfo from the release builds

18
src/config.rs
View File

@ -1188,6 +1188,9 @@ fn opendal_operator_for_path(path: &str) -> Result<opendal::Operator, Error> {
#[cfg(s3)]
fn opendal_s3_operator_for_path(path: &str) -> Result<opendal::Operator, Error> {
use crate::http_client::aws::AwsReqwestConnector;
use aws_config::{default_provider::credentials::DefaultCredentialsChain, provider_config::ProviderConfig};
// This is a custom AWS credential loader that uses the official AWS Rust
// SDK config crate to load credentials. This ensures maximum compatibility
// with AWS credential configurations. For example, OpenDAL doesn't support
@ -1200,12 +1203,19 @@ fn opendal_s3_operator_for_path(path: &str) -> Result<opendal::Operator, Error>
use aws_credential_types::provider::ProvideCredentials as _;
use tokio::sync::OnceCell;
static DEFAULT_CREDENTIAL_CHAIN: OnceCell<
aws_config::default_provider::credentials::DefaultCredentialsChain,
> = OnceCell::const_new();
static DEFAULT_CREDENTIAL_CHAIN: OnceCell<DefaultCredentialsChain> = OnceCell::const_new();
let chain = DEFAULT_CREDENTIAL_CHAIN
.get_or_init(|| aws_config::default_provider::credentials::DefaultCredentialsChain::builder().build())
.get_or_init(|| {
let reqwest_client = reqwest::Client::builder().build().unwrap();
let connector = AwsReqwestConnector {
client: reqwest_client,
};
let conf = ProviderConfig::default().with_http_client(connector);
DefaultCredentialsChain::builder().configure(conf).build()
})
.await;
let creds = chain.provide_credentials().await?;

58
src/http_client.rs
View File

@ -244,3 +244,61 @@ impl Resolve for CustomDnsResolver {
})
}
}
#[cfg(s3)]
pub(crate) mod aws {
use aws_smithy_runtime_api::client::{
http::{HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector},
orchestrator::HttpResponse,
result::ConnectorError,
runtime_components::RuntimeComponents,
};
use reqwest::Client;
// Adapter that wraps reqwest to be compatible with the AWS SDK
#[derive(Debug)]
pub(crate) struct AwsReqwestConnector {
pub(crate) client: Client,
}
impl HttpConnector for AwsReqwestConnector {
fn call(&self, request: aws_smithy_runtime_api::client::orchestrator::HttpRequest) -> HttpConnectorFuture {
// Convert the AWS-style request to a reqwest request
let client = self.client.clone();
let future = async move {
let method = reqwest::Method::from_bytes(request.method().as_bytes())
.map_err(|e| ConnectorError::user(Box::new(e)))?;
let mut req_builder = client.request(method, request.uri().to_string());
for (name, value) in request.headers() {
req_builder = req_builder.header(name, value);
}
if let Some(body_bytes) = request.body().bytes() {
req_builder = req_builder.body(body_bytes.to_vec());
}
let response = req_builder.send().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
let status = response.status().into();
let bytes = response.bytes().await.map_err(|e| ConnectorError::io(Box::new(e)))?;
Ok(HttpResponse::new(status, bytes.into()))
};
HttpConnectorFuture::new(Box::pin(future))
}
}
impl HttpClient for AwsReqwestConnector {
fn http_connector(
&self,
_settings: &HttpConnectorSettings,
_components: &RuntimeComponents,
) -> SharedHttpConnector {
SharedHttpConnector::new(AwsReqwestConnector {
client: self.client.clone(),
})
}
}
}

Write Preview
Loading…
Cancel
Save