diff --git a/Cargo.lock b/Cargo.lock index 72728273..ab394f70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + [[package]] name = "argon2" version = "0.5.3" @@ -311,6 +317,17 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "backon" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fef586913a57ff189f25c9b3d034356a5bf6b3fa9a7f067588fe1698ba1f5d" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -502,6 +519,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.39" @@ -1603,6 +1626,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.1", "tower-service", + "webpki-roots", ] [[package]] @@ -2080,6 +2104,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2329,6 +2363,33 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "opendal" +version = "0.51.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b1063ea459fa9e94584115743b06330f437902dd1d9f692b863ef1875a20548" +dependencies = [ + "anyhow", + "async-trait", + "backon", + "base64 0.22.1", + "bytes", + "chrono", + "futures", + "getrandom 0.2.15", + "http 1.2.0", + "log", + "md-5", + "once_cell", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tokio", + "uuid", +] + [[package]] name = "openssl" version = "0.10.71" @@ -2729,6 +2790,68 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.23", + "socket2", + "thiserror 2.0.11", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +dependencies = [ + "bytes", + "getrandom 0.2.15", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls 0.23.23", + "rustls-pki-types", + "slab", + "thiserror 2.0.11", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.38" @@ -2940,7 +3063,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls 0.23.23", "rustls-pemfile 2.2.0", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2948,6 +3074,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls 0.26.1", "tokio-socks", "tokio-util", "tower", @@ -2957,6 +3084,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", "windows-registry", ] @@ -3127,6 +3255,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustix" version = "0.38.44" @@ -3159,6 +3293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.102.8", "subtle", @@ -3188,6 +3323,9 @@ name = "rustls-pki-types" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -4074,6 +4212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1" dependencies = [ "getrandom 0.3.1", + "serde", ] [[package]] @@ -4127,6 +4266,7 @@ dependencies = [ "num-derive", "num-traits", "once_cell", + "opendal", "openssl", "paste", "percent-encoding", @@ -4324,6 +4464,15 @@ dependencies = [ "url", ] +[[package]] +name = "webpki-roots" +version = "0.26.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "7.0.2" diff --git a/Cargo.toml b/Cargo.toml index 8fdd6866..caf04c7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,9 @@ rpassword = "7.3.1" # Loading a dynamic CSS Stylesheet grass_compiler = { version = "0.13.4", default-features = false } +# File are accessed through Apache OpenDAL +opendal = { version = "0.51.2", features = ["services-fs"] } + [patch.crates-io] # Patch yubico to remove duplicate crates of older versions yubico = { git = "https://github.com/BlackDex/yubico-rs", rev = "00df14811f58155c0f02e3ab10f1570ed3e115c6" } diff --git a/src/api/admin.rs b/src/api/admin.rs index b3e703d9..6a0585ca 100644 --- a/src/api/admin.rs +++ b/src/api/admin.rs @@ -745,17 +745,17 @@ fn get_diagnostics_http(code: u16, _token: AdminToken) -> EmptyResult { } #[post("/config", format = "application/json", data = "")] -fn post_config(data: Json, _token: AdminToken) -> EmptyResult { +async fn post_config(data: Json, _token: AdminToken) -> EmptyResult { let data: ConfigBuilder = data.into_inner(); - if let Err(e) = CONFIG.update_config(data, true) { + if let Err(e) = CONFIG.update_config(data, true).await { err!(format!("Unable to save config: {e:?}")) } Ok(()) } #[post("/config/delete", format = "application/json")] -fn delete_config(_token: AdminToken) -> EmptyResult { - if let Err(e) = CONFIG.delete_user_config() { +async fn delete_config(_token: AdminToken) -> EmptyResult { + if let Err(e) = CONFIG.delete_user_config().await { err!(format!("Unable to delete config: {e:?}")) } Ok(()) diff --git a/src/api/core/ciphers.rs b/src/api/core/ciphers.rs index 6c75d246..24304439 100644 --- a/src/api/core/ciphers.rs +++ b/src/api/core/ciphers.rs @@ -11,10 +11,11 @@ use rocket::{ use serde_json::Value; use crate::auth::ClientVersion; -use crate::util::NumberOrString; +use crate::util::{save_temp_file, NumberOrString}; use crate::{ api::{self, core::log_event, EmptyResult, JsonResult, Notify, PasswordOrOtpData, UpdateType}, auth::Headers, + config::PathType, crypto, db::{models::*, DbConn, DbPool}, CONFIG, @@ -105,12 +106,7 @@ struct SyncData { } #[get("/sync?")] -async fn sync( - data: SyncData, - headers: Headers, - client_version: Option, - mut conn: DbConn, -) -> Json { +async fn sync(data: SyncData, headers: Headers, client_version: Option, mut conn: DbConn) -> JsonResult { let user_json = headers.user.to_json(&mut conn).await; // Get all ciphers which are visible by the user @@ -134,7 +130,7 @@ async fn sync( for c in ciphers { ciphers_json.push( c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), CipherSyncType::User, &mut conn) - .await, + .await?, ); } @@ -159,7 +155,7 @@ async fn sync( api::core::_get_eq_domains(headers, true).into_inner() }; - Json(json!({ + Ok(Json(json!({ "profile": user_json, "folders": folders_json, "collections": collections_json, @@ -168,11 +164,11 @@ async fn sync( "domains": domains_json, "sends": sends_json, "object": "sync" - })) + }))) } #[get("/ciphers")] -async fn get_ciphers(headers: Headers, mut conn: DbConn) -> Json { +async fn get_ciphers(headers: Headers, mut conn: DbConn) -> JsonResult { let ciphers = Cipher::find_by_user_visible(&headers.user.uuid, &mut conn).await; let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, CipherSyncType::User, &mut conn).await; @@ -180,15 +176,15 @@ async fn get_ciphers(headers: Headers, mut conn: DbConn) -> Json { for c in ciphers { ciphers_json.push( c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), CipherSyncType::User, &mut conn) - .await, + .await?, ); } - Json(json!({ + Ok(Json(json!({ "data": ciphers_json, "object": "list", "continuationToken": null - })) + }))) } #[get("/ciphers/")] @@ -201,7 +197,7 @@ async fn get_cipher(cipher_id: CipherId, headers: Headers, mut conn: DbConn) -> err!("Cipher is not owned by user") } - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } #[get("/ciphers//admin")] @@ -339,7 +335,7 @@ async fn post_ciphers(data: Json, headers: Headers, mut conn: DbConn let mut cipher = Cipher::new(data.r#type, data.name.clone()); update_cipher_from_data(&mut cipher, data, &headers, None, &mut conn, &nt, UpdateType::SyncCipherCreate).await?; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } /// Enforces the personal ownership policy on user-owned ciphers, if applicable. @@ -676,7 +672,7 @@ async fn put_cipher( update_cipher_from_data(&mut cipher, data, &headers, None, &mut conn, &nt, UpdateType::SyncCipherUpdate).await?; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } #[post("/ciphers//partial", data = "")] @@ -714,7 +710,7 @@ async fn put_cipher_partial( // Update favorite cipher.set_favorite(Some(data.favorite), &headers.user.uuid, &mut conn).await?; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } #[derive(Deserialize)] @@ -825,7 +821,7 @@ async fn post_collections_update( ) .await; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } #[put("/ciphers//collections-admin", data = "")] @@ -1030,7 +1026,7 @@ async fn share_cipher_by_uuid( update_cipher_from_data(&mut cipher, data.cipher, headers, Some(shared_to_collections), conn, nt, ut).await?; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, conn).await?)) } /// v2 API for downloading an attachment. This just redirects the client to @@ -1055,7 +1051,7 @@ async fn get_attachment( } match Attachment::find_by_id(&attachment_id, &mut conn).await { - Some(attachment) if cipher_id == attachment.cipher_uuid => Ok(Json(attachment.to_json(&headers.host))), + Some(attachment) if cipher_id == attachment.cipher_uuid => Ok(Json(attachment.to_json(&headers.host).await?)), Some(_) => err!("Attachment doesn't belong to cipher"), None => err!("Attachment doesn't exist"), } @@ -1116,7 +1112,7 @@ async fn post_attachment_v2( "attachmentId": attachment_id, "url": url, "fileUploadType": FileUploadType::Direct as i32, - response_key: cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await, + response_key: cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?, }))) } @@ -1142,7 +1138,7 @@ async fn save_attachment( mut conn: DbConn, nt: Notify<'_>, ) -> Result<(Cipher, DbConn), crate::error::Error> { - let mut data = data.into_inner(); + let data = data.into_inner(); let Some(size) = data.data.len().to_i64() else { err!("Attachment data size overflow"); @@ -1269,13 +1265,7 @@ async fn save_attachment( attachment.save(&mut conn).await.expect("Error saving attachment"); } - let folder_path = tokio::fs::canonicalize(&CONFIG.attachments_folder()).await?.join(cipher_id.as_ref()); - let file_path = folder_path.join(file_id.as_ref()); - tokio::fs::create_dir_all(&folder_path).await?; - - if let Err(_err) = data.data.persist_to(&file_path).await { - data.data.move_copy_to(file_path).await? - } + save_temp_file(PathType::Attachments, &format!("{cipher_id}/{file_id}"), data.data).await?; nt.send_cipher_update( UpdateType::SyncCipherUpdate, @@ -1342,7 +1332,7 @@ async fn post_attachment( let (cipher, mut conn) = save_attachment(attachment, cipher_id, data, &headers, conn, nt).await?; - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, &mut conn).await?)) } #[post("/ciphers//attachment-admin", format = "multipart/form-data", data = "")] @@ -1786,7 +1776,7 @@ async fn _restore_cipher_by_uuid( .await; } - Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, conn).await)) + Ok(Json(cipher.to_json(&headers.host, &headers.user.uuid, None, CipherSyncType::User, conn).await?)) } async fn _restore_multiple_ciphers( diff --git a/src/api/core/emergency_access.rs b/src/api/core/emergency_access.rs index 8c6fcb65..39f7490a 100644 --- a/src/api/core/emergency_access.rs +++ b/src/api/core/emergency_access.rs @@ -582,7 +582,7 @@ async fn view_emergency_access(emer_id: EmergencyAccessId, headers: Headers, mut CipherSyncType::User, &mut conn, ) - .await, + .await?, ); } diff --git a/src/api/core/organizations.rs b/src/api/core/organizations.rs index aabcc5e2..c5a84895 100644 --- a/src/api/core/organizations.rs +++ b/src/api/core/organizations.rs @@ -901,21 +901,26 @@ async fn get_org_details(data: OrgIdData, headers: OrgMemberHeaders, mut conn: D } Ok(Json(json!({ - "data": _get_org_details(&data.organization_id, &headers.host, &headers.user.uuid, &mut conn).await, + "data": _get_org_details(&data.organization_id, &headers.host, &headers.user.uuid, &mut conn).await?, "object": "list", "continuationToken": null, }))) } -async fn _get_org_details(org_id: &OrganizationId, host: &str, user_id: &UserId, conn: &mut DbConn) -> Value { +async fn _get_org_details( + org_id: &OrganizationId, + host: &str, + user_id: &UserId, + conn: &mut DbConn, +) -> Result { let ciphers = Cipher::find_by_org(org_id, conn).await; let cipher_sync_data = CipherSyncData::new(user_id, CipherSyncType::Organization, conn).await; let mut ciphers_json = Vec::with_capacity(ciphers.len()); for c in ciphers { - ciphers_json.push(c.to_json(host, user_id, Some(&cipher_sync_data), CipherSyncType::Organization, conn).await); + ciphers_json.push(c.to_json(host, user_id, Some(&cipher_sync_data), CipherSyncType::Organization, conn).await?); } - json!(ciphers_json) + Ok(json!(ciphers_json)) } #[derive(FromForm)] @@ -3317,7 +3322,7 @@ async fn get_org_export( "continuationToken": null, }, "ciphers": { - "data": convert_json_key_lcase_first(_get_org_details(&org_id, &headers.host, &headers.user.uuid, &mut conn).await), + "data": convert_json_key_lcase_first(_get_org_details(&org_id, &headers.host, &headers.user.uuid, &mut conn).await?), "object": "list", "continuationToken": null, } @@ -3326,7 +3331,7 @@ async fn get_org_export( // v2023.1.0 and newer response Ok(Json(json!({ "collections": convert_json_key_lcase_first(_get_org_collections(&org_id, &mut conn).await), - "ciphers": convert_json_key_lcase_first(_get_org_details(&org_id, &headers.host, &headers.user.uuid, &mut conn).await), + "ciphers": convert_json_key_lcase_first(_get_org_details(&org_id, &headers.host, &headers.user.uuid, &mut conn).await?), }))) } } diff --git a/src/api/core/sends.rs b/src/api/core/sends.rs index bf7a5ec8..c55acd6b 100644 --- a/src/api/core/sends.rs +++ b/src/api/core/sends.rs @@ -11,8 +11,9 @@ use serde_json::Value; use crate::{ api::{ApiResult, EmptyResult, JsonResult, Notify, UpdateType}, auth::{ClientIp, Headers, Host}, + config::PathType, db::{models::*, DbConn, DbPool}, - util::NumberOrString, + util::{save_temp_file, NumberOrString}, CONFIG, }; @@ -210,7 +211,7 @@ async fn post_send_file(data: Form>, headers: Headers, mut conn: let UploadData { model, - mut data, + data, } = data.into_inner(); let model = model.into_inner(); @@ -250,13 +251,8 @@ async fn post_send_file(data: Form>, headers: Headers, mut conn: } let file_id = crate::crypto::generate_send_file_id(); - let folder_path = tokio::fs::canonicalize(&CONFIG.sends_folder()).await?.join(&send.uuid); - let file_path = folder_path.join(&file_id); - tokio::fs::create_dir_all(&folder_path).await?; - if let Err(_err) = data.persist_to(&file_path).await { - data.move_copy_to(file_path).await? - } + save_temp_file(PathType::Sends, &format!("{}/{file_id}", send.uuid), data).await?; let mut data_value: Value = serde_json::from_str(&send.data)?; if let Some(o) = data_value.as_object_mut() { @@ -363,7 +359,7 @@ async fn post_send_file_v2_data( ) -> EmptyResult { enforce_disable_send_policy(&headers, &mut conn).await?; - let mut data = data.into_inner(); + let data = data.into_inner(); let Some(send) = Send::find_by_uuid_and_user(&send_id, &headers.user.uuid, &mut conn).await else { err!("Send not found. Unable to save the file.", "Invalid send uuid or does not belong to user.") @@ -406,19 +402,20 @@ async fn post_send_file_v2_data( err!("Send file size does not match.", format!("Expected a file size of {} got {size}", send_data.size)); } - let folder_path = tokio::fs::canonicalize(&CONFIG.sends_folder()).await?.join(send_id); - let file_path = folder_path.join(file_id); + let operator = CONFIG.opendal_operator_for_path_type(PathType::Sends)?; + let file_path = format!("{send_id}/{file_id}"); // Check if the file already exists, if that is the case do not overwrite it - if tokio::fs::metadata(&file_path).await.is_ok() { + if operator.exists(&file_path).await.map_err(|e| { + crate::Error::new( + "Unexpected error while creating send file", + format!("Error while checking existence of send file at path {file_path}: {e:?}"), + ) + })? { err!("Send file has already been uploaded.", format!("File {file_path:?} already exists")) } - tokio::fs::create_dir_all(&folder_path).await?; - - if let Err(_err) = data.data.persist_to(&file_path).await { - data.data.move_copy_to(file_path).await? - } + save_temp_file(PathType::Sends, &file_path, data.data).await?; nt.send_send_update( UpdateType::SyncSendCreate, @@ -551,15 +548,20 @@ async fn post_access_file( ) .await; - let token_claims = crate::auth::generate_send_claims(&send_id, &file_id); - let token = crate::auth::encode_jwt(&token_claims); Ok(Json(json!({ "object": "send-fileDownload", "id": file_id, - "url": format!("{}/api/sends/{}/{}?t={}", &host.host, send_id, file_id, token) + "url": download_url(&host, &send_id, &file_id).await?, }))) } +async fn download_url(host: &Host, send_id: &SendId, file_id: &SendFileId) -> Result { + let token_claims = crate::auth::generate_send_claims(send_id, file_id); + let token = crate::auth::encode_jwt(&token_claims); + + Ok(format!("{}/api/sends/{}/{}?t={}", &host.host, send_id, file_id, token)) +} + #[get("/sends//?")] async fn download_send(send_id: SendId, file_id: SendFileId, t: &str) -> Option { if let Ok(claims) = crate::auth::decode_send(t) { diff --git a/src/api/core/two_factor/duo.rs b/src/api/core/two_factor/duo.rs index aa281ae7..f46c1e90 100644 --- a/src/api/core/two_factor/duo.rs +++ b/src/api/core/two_factor/duo.rs @@ -258,7 +258,7 @@ pub(crate) async fn get_duo_keys_email(email: &str, conn: &mut DbConn) -> ApiRes } .map_res("Can't fetch Duo Keys")?; - Ok((data.ik, data.sk, CONFIG.get_duo_akey(), data.host)) + Ok((data.ik, data.sk, CONFIG.get_duo_akey().await, data.host)) } pub async fn generate_duo_signature(email: &str, conn: &mut DbConn) -> ApiResult<(String, String)> { diff --git a/src/api/icons.rs b/src/api/icons.rs index 0b437d53..850ebadb 100644 --- a/src/api/icons.rs +++ b/src/api/icons.rs @@ -14,14 +14,11 @@ use reqwest::{ Client, Response, }; use rocket::{http::ContentType, response::Redirect, Route}; -use tokio::{ - fs::{create_dir_all, remove_file, symlink_metadata, File}, - io::{AsyncReadExt, AsyncWriteExt}, -}; use html5gum::{Emitter, HtmlString, Readable, StringReader, Tokenizer}; use crate::{ + config::PathType, error::Error, http_client::{get_reqwest_client_builder, should_block_address, CustomHttpClientError}, util::Cached, @@ -159,7 +156,7 @@ fn is_valid_domain(domain: &str) -> bool { } async fn get_icon(domain: &str) -> Option<(Vec, String)> { - let path = format!("{}/{}.png", CONFIG.icon_cache_folder(), domain); + let path = format!("{domain}.png"); // Check for expiration of negatively cached copy if icon_is_negcached(&path).await { @@ -181,7 +178,7 @@ async fn get_icon(domain: &str) -> Option<(Vec, String)> { // Get the icon, or None in case of error match download_icon(domain).await { Ok((icon, icon_type)) => { - save_icon(&path, &icon).await; + save_icon(&path, icon.to_vec()).await; Some((icon.to_vec(), icon_type.unwrap_or("x-icon").to_string())) } Err(e) => { @@ -194,7 +191,7 @@ async fn get_icon(domain: &str) -> Option<(Vec, String)> { warn!("Unable to download icon: {:?}", e); let miss_indicator = path + ".miss"; - save_icon(&miss_indicator, &[]).await; + save_icon(&miss_indicator, vec![]).await; None } } @@ -207,11 +204,9 @@ async fn get_cached_icon(path: &str) -> Option> { } // Try to read the cached icon, and return it if it exists - if let Ok(mut f) = File::open(path).await { - let mut buffer = Vec::new(); - - if f.read_to_end(&mut buffer).await.is_ok() { - return Some(buffer); + if let Ok(operator) = CONFIG.opendal_operator_for_path_type(PathType::IconCache) { + if let Ok(buf) = operator.read(path).await { + return Some(buf.to_vec()); } } @@ -219,9 +214,11 @@ async fn get_cached_icon(path: &str) -> Option> { } async fn file_is_expired(path: &str, ttl: u64) -> Result { - let meta = symlink_metadata(path).await?; - let modified = meta.modified()?; - let age = SystemTime::now().duration_since(modified)?; + let operator = CONFIG.opendal_operator_for_path_type(PathType::IconCache)?; + let meta = operator.stat(path).await?; + let modified = + meta.last_modified().ok_or_else(|| std::io::Error::other(format!("No last modified time for `{path}`")))?; + let age = SystemTime::now().duration_since(modified.into())?; Ok(ttl > 0 && ttl <= age.as_secs()) } @@ -233,8 +230,13 @@ async fn icon_is_negcached(path: &str) -> bool { match expired { // No longer negatively cached, drop the marker Ok(true) => { - if let Err(e) = remove_file(&miss_indicator).await { - error!("Could not remove negative cache indicator for icon {:?}: {:?}", path, e); + match CONFIG.opendal_operator_for_path_type(PathType::IconCache) { + Ok(operator) => { + if let Err(e) = operator.delete_iter([miss_indicator]).await { + error!("Could not remove negative cache indicator for icon {:?}: {:?}", path, e); + } + } + Err(e) => error!("Could not remove negative cache indicator for icon {:?}: {:?}", path, e), } false } @@ -568,17 +570,17 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> { Ok((buffer, icon_type)) } -async fn save_icon(path: &str, icon: &[u8]) { - match File::create(path).await { - Ok(mut f) => { - f.write_all(icon).await.expect("Error writing icon file"); - } - Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => { - create_dir_all(&CONFIG.icon_cache_folder()).await.expect("Error creating icon cache folder"); - } +async fn save_icon(path: &str, icon: Vec) { + let operator = match CONFIG.opendal_operator_for_path_type(PathType::IconCache) { + Ok(operator) => operator, Err(e) => { - warn!("Unable to save icon: {:?}", e); + warn!("Failed to get OpenDAL operator while saving icon: {e}"); + return; } + }; + + if let Err(e) = operator.write(path, icon).await { + warn!("Unable to save icon: {e:?}"); } } diff --git a/src/auth.rs b/src/auth.rs index 0fabd6a4..7194801f 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -7,16 +7,14 @@ use once_cell::sync::{Lazy, OnceCell}; use openssl::rsa::Rsa; use serde::de::DeserializeOwned; use serde::ser::Serialize; -use std::{ - env, - fs::File, - io::{Read, Write}, - net::IpAddr, -}; - -use crate::db::models::{ - AttachmentId, CipherId, CollectionId, DeviceId, EmergencyAccessId, MembershipId, OrgApiKeyId, OrganizationId, - SendFileId, SendId, UserId, +use std::{env, net::IpAddr}; + +use crate::{ + config::PathType, + db::models::{ + AttachmentId, CipherId, CollectionId, DeviceId, EmergencyAccessId, MembershipId, OrgApiKeyId, OrganizationId, + SendFileId, SendId, UserId, + }, }; use crate::{error::Error, CONFIG}; @@ -40,37 +38,44 @@ static JWT_REGISTER_VERIFY_ISSUER: Lazy = Lazy::new(|| format!("{}|regis static PRIVATE_RSA_KEY: OnceCell = OnceCell::new(); static PUBLIC_RSA_KEY: OnceCell = OnceCell::new(); -pub fn initialize_keys() -> Result<(), Error> { - fn read_key(create_if_missing: bool) -> Result<(Rsa, Vec), Error> { - let mut priv_key_buffer = Vec::with_capacity(2048); +pub async fn initialize_keys() -> Result<(), Error> { + async fn read_key(create_if_missing: bool) -> Result<(Rsa, Vec), std::io::Error> { + use std::io::{Error, ErrorKind}; - let mut priv_key_file = File::options() - .create(create_if_missing) - .truncate(false) - .read(true) - .write(create_if_missing) - .open(CONFIG.private_rsa_key())?; + let rsa_key_filename = std::path::PathBuf::from(CONFIG.private_rsa_key()) + .file_name() + .ok_or_else(|| Error::other("Private RSA key path missing filename"))? + .to_str() + .ok_or_else(|| Error::other("Private RSA key path filename is not valid UTF-8"))? + .to_string(); - #[allow(clippy::verbose_file_reads)] - let bytes_read = priv_key_file.read_to_end(&mut priv_key_buffer)?; + let operator = CONFIG.opendal_operator_for_path_type(PathType::RsaKey).map_err(Error::other)?; - let rsa_key = if bytes_read > 0 { - Rsa::private_key_from_pem(&priv_key_buffer[..bytes_read])? - } else if create_if_missing { - // Only create the key if the file doesn't exist or is empty - let rsa_key = Rsa::generate(2048)?; - priv_key_buffer = rsa_key.private_key_to_pem()?; - priv_key_file.write_all(&priv_key_buffer)?; - info!("Private key '{}' created correctly", CONFIG.private_rsa_key()); - rsa_key - } else { - err!("Private key does not exist or invalid format", CONFIG.private_rsa_key()); + let priv_key_buffer = match operator.read(&rsa_key_filename).await { + Ok(buffer) => Some(buffer), + Err(e) if e.kind() == opendal::ErrorKind::NotFound && create_if_missing => None, + Err(e) if e.kind() == opendal::ErrorKind::NotFound => { + return Err(Error::new(ErrorKind::NotFound, "Private key not found")) + } + Err(e) => return Err(Error::new(ErrorKind::InvalidData, format!("Error reading private key: {e}"))), }; - Ok((rsa_key, priv_key_buffer)) + if let Some(priv_key_buffer) = priv_key_buffer { + Ok((Rsa::private_key_from_pem(priv_key_buffer.to_vec().as_slice())?, priv_key_buffer.to_vec())) + } else { + let rsa_key = Rsa::generate(2048)?; + let priv_key_buffer = rsa_key.private_key_to_pem()?; + operator.write(&rsa_key_filename, priv_key_buffer).await?; + info!("Private key '{}' created correctly", CONFIG.private_rsa_key()); + Err(Error::new(ErrorKind::NotFound, "Private key created, forcing attempt to read it again")) + } } - let (priv_key, priv_key_buffer) = read_key(true).or_else(|_| read_key(false))?; + let (priv_key, priv_key_buffer) = match read_key(true).await { + Ok(key) => key, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => read_key(false).await?, + Err(e) => return Err(e.into()), + }; let pub_key_buffer = priv_key.public_key_to_pem()?; let enc = EncodingKey::from_rsa_pem(&priv_key_buffer)?; diff --git a/src/config.rs b/src/config.rs index 6a06cac6..3d569b8a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,9 +1,10 @@ use std::{ + collections::HashMap, env::consts::EXE_SUFFIX, process::exit, sync::{ atomic::{AtomicBool, Ordering}, - RwLock, + LazyLock, Mutex, RwLock, }, }; @@ -22,10 +23,32 @@ static CONFIG_FILE: Lazy = Lazy::new(|| { get_env("CONFIG_FILE").unwrap_or_else(|| format!("{data_folder}/config.json")) }); +static CONFIG_FILE_PARENT_DIR: LazyLock = LazyLock::new(|| { + let path = std::path::PathBuf::from(&*CONFIG_FILE); + path.parent().unwrap_or(std::path::Path::new("data")).to_str().unwrap_or("data").to_string() +}); + +static CONFIG_FILENAME: LazyLock = LazyLock::new(|| { + let path = std::path::PathBuf::from(&*CONFIG_FILE); + path.file_name().unwrap_or(std::ffi::OsStr::new("config.json")).to_str().unwrap_or("config.json").to_string() +}); + pub static SKIP_CONFIG_VALIDATION: AtomicBool = AtomicBool::new(false); pub static CONFIG: Lazy = Lazy::new(|| { - Config::load().unwrap_or_else(|e| { + std::thread::spawn(|| { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap_or_else(|e| { + println!("Error loading config:\n {e:?}\n"); + exit(12) + }); + + rt.block_on(Config::load()).unwrap_or_else(|e| { + println!("Error loading config:\n {e:?}\n"); + exit(12) + }) + }) + .join() + .unwrap_or_else(|e| { println!("Error loading config:\n {e:?}\n"); exit(12) }) @@ -110,9 +133,12 @@ macro_rules! make_config { builder } - fn from_file(path: &str) -> Result { - let config_str = std::fs::read_to_string(path)?; - println!("[INFO] Using saved config from `{path}` for configuration.\n"); + async fn from_file() -> Result { + let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + let config_bytes = operator.read(&CONFIG_FILENAME).await?; + let config_str = String::from_utf8(config_bytes.to_vec()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + println!("[INFO] Using saved config from `{}` for configuration.\n", *CONFIG_FILE); serde_json::from_str(&config_str).map_err(Into::into) } @@ -1132,11 +1158,39 @@ fn smtp_convert_deprecated_ssl_options(smtp_ssl: Option, smtp_explicit_tls "starttls".to_string() } +fn opendal_operator_for_path(path: &str) -> Result { + // Cache of previously built operators by path + static OPERATORS_BY_PATH: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + + let mut operators_by_path = + OPERATORS_BY_PATH.lock().map_err(|e| format!("Failed to lock OpenDAL operators cache: {e}"))?; + + if let Some(operator) = operators_by_path.get(path) { + return Ok(operator.clone()); + } + + let builder = opendal::services::Fs::default().root(path); + let operator = opendal::Operator::new(builder).map_err(Into::::into)?.finish(); + + operators_by_path.insert(path.to_string(), operator.clone()); + + Ok(operator) +} + +pub enum PathType { + Data, + IconCache, + Attachments, + Sends, + RsaKey, +} + impl Config { - pub fn load() -> Result { + pub async fn load() -> Result { // Loading from env and file let _env = ConfigBuilder::from_env(); - let _usr = ConfigBuilder::from_file(&CONFIG_FILE).unwrap_or_default(); + let _usr = ConfigBuilder::from_file().await.unwrap_or_default(); // Create merged config, config file overwrites env let mut _overrides = Vec::new(); @@ -1160,7 +1214,7 @@ impl Config { }) } - pub fn update_config(&self, other: ConfigBuilder, ignore_non_editable: bool) -> Result<(), Error> { + pub async fn update_config(&self, other: ConfigBuilder, ignore_non_editable: bool) -> Result<(), Error> { // Remove default values //let builder = other.remove(&self.inner.read().unwrap()._env); @@ -1192,20 +1246,19 @@ impl Config { } //Save to file - use std::{fs::File, io::Write}; - let mut file = File::create(&*CONFIG_FILE)?; - file.write_all(config_str.as_bytes())?; + let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + operator.write(&CONFIG_FILENAME, config_str).await?; Ok(()) } - fn update_config_partial(&self, other: ConfigBuilder) -> Result<(), Error> { + async fn update_config_partial(&self, other: ConfigBuilder) -> Result<(), Error> { let builder = { let usr = &self.inner.read().unwrap()._usr; let mut _overrides = Vec::new(); usr.merge(&other, false, &mut _overrides) }; - self.update_config(builder, false) + self.update_config(builder, false).await } /// Tests whether an email's domain is allowed. A domain is allowed if it @@ -1247,8 +1300,9 @@ impl Config { } } - pub fn delete_user_config(&self) -> Result<(), Error> { - std::fs::remove_file(&*CONFIG_FILE)?; + pub async fn delete_user_config(&self) -> Result<(), Error> { + let operator = opendal_operator_for_path(&CONFIG_FILE_PARENT_DIR)?; + operator.delete(&CONFIG_FILENAME).await?; // Empty user config let usr = ConfigBuilder::default(); @@ -1278,7 +1332,7 @@ impl Config { inner._enable_smtp && (inner.smtp_host.is_some() || inner.use_sendmail) } - pub fn get_duo_akey(&self) -> String { + pub async fn get_duo_akey(&self) -> String { if let Some(akey) = self._duo_akey() { akey } else { @@ -1289,7 +1343,7 @@ impl Config { _duo_akey: Some(akey_s.clone()), ..Default::default() }; - self.update_config_partial(builder).ok(); + self.update_config_partial(builder).await.ok(); akey_s } @@ -1302,6 +1356,23 @@ impl Config { token.is_some() && !token.unwrap().trim().is_empty() } + pub fn opendal_operator_for_path_type(&self, path_type: PathType) -> Result { + let path = match path_type { + PathType::Data => self.data_folder(), + PathType::IconCache => self.icon_cache_folder(), + PathType::Attachments => self.attachments_folder(), + PathType::Sends => self.sends_folder(), + PathType::RsaKey => std::path::Path::new(&self.rsa_key_filename()) + .parent() + .ok_or_else(|| std::io::Error::other("Failed to get directory of RSA key file"))? + .to_str() + .ok_or_else(|| std::io::Error::other("Failed to convert RSA key file directory to UTF-8 string"))? + .to_string(), + }; + + opendal_operator_for_path(&path) + } + pub fn render_template(&self, name: &str, data: &T) -> Result { if self.reload_templates() { warn!("RELOADING TEMPLATES"); diff --git a/src/db/models/attachment.rs b/src/db/models/attachment.rs index 09348f78..1a8d1c0a 100644 --- a/src/db/models/attachment.rs +++ b/src/db/models/attachment.rs @@ -1,11 +1,9 @@ -use std::io::ErrorKind; - use bigdecimal::{BigDecimal, ToPrimitive}; use derive_more::{AsRef, Deref, Display}; use serde_json::Value; use super::{CipherId, OrganizationId, UserId}; -use crate::CONFIG; +use crate::{config::PathType, CONFIG}; use macros::IdFromParam; db_object! { @@ -41,24 +39,24 @@ impl Attachment { } pub fn get_file_path(&self) -> String { - format!("{}/{}/{}", CONFIG.attachments_folder(), self.cipher_uuid, self.id) + format!("{}/{}", self.cipher_uuid, self.id) } - pub fn get_url(&self, host: &str) -> String { + pub async fn get_url(&self, host: &str) -> Result { let token = encode_jwt(&generate_file_download_claims(self.cipher_uuid.clone(), self.id.clone())); - format!("{}/attachments/{}/{}?token={}", host, self.cipher_uuid, self.id, token) + Ok(format!("{}/attachments/{}/{}?token={}", host, self.cipher_uuid, self.id, token)) } - pub fn to_json(&self, host: &str) -> Value { - json!({ + pub async fn to_json(&self, host: &str) -> Result { + Ok(json!({ "id": self.id, - "url": self.get_url(host), + "url": self.get_url(host).await?, "fileName": self.file_name, "size": self.file_size.to_string(), "sizeName": crate::util::get_display_size(self.file_size), "key": self.akey, "object": "attachment" - }) + })) } } @@ -104,26 +102,26 @@ impl Attachment { pub async fn delete(&self, conn: &mut DbConn) -> EmptyResult { db_run! { conn: { - let _: () = crate::util::retry( + crate::util::retry( || diesel::delete(attachments::table.filter(attachments::id.eq(&self.id))).execute(conn), 10, ) - .map_res("Error deleting attachment")?; - - let file_path = &self.get_file_path(); - - match std::fs::remove_file(file_path) { - // Ignore "file not found" errors. This can happen when the - // upstream caller has already cleaned up the file as part of - // its own error handling. - Err(e) if e.kind() == ErrorKind::NotFound => { - debug!("File '{}' already deleted.", file_path); - Ok(()) - } - Err(e) => Err(e.into()), - _ => Ok(()), + .map(|_| ()) + .map_res("Error deleting attachment") + }}?; + + let operator = CONFIG.opendal_operator_for_path_type(PathType::Attachments)?; + let file_path = self.get_file_path(); + + if let Err(e) = operator.delete_iter([file_path.clone()]).await { + if e.kind() == opendal::ErrorKind::NotFound { + debug!("File '{file_path}' already deleted."); + } else { + return Err(e.into()); } - }} + } + + Ok(()) } pub async fn delete_all_by_cipher(cipher_uuid: &CipherId, conn: &mut DbConn) -> EmptyResult { diff --git a/src/db/models/cipher.rs b/src/db/models/cipher.rs index d9dbd28d..7c3785fa 100644 --- a/src/db/models/cipher.rs +++ b/src/db/models/cipher.rs @@ -141,18 +141,28 @@ impl Cipher { cipher_sync_data: Option<&CipherSyncData>, sync_type: CipherSyncType, conn: &mut DbConn, - ) -> Value { + ) -> Result { use crate::util::{format_date, validate_and_format_date}; let mut attachments_json: Value = Value::Null; if let Some(cipher_sync_data) = cipher_sync_data { if let Some(attachments) = cipher_sync_data.cipher_attachments.get(&self.uuid) { - attachments_json = attachments.iter().map(|c| c.to_json(host)).collect(); + if !attachments.is_empty() { + let mut attachments_json_vec = vec![]; + for attachment in attachments { + attachments_json_vec.push(attachment.to_json(host).await?); + } + attachments_json = Value::Array(attachments_json_vec); + } } } else { let attachments = Attachment::find_by_cipher(&self.uuid, conn).await; if !attachments.is_empty() { - attachments_json = attachments.iter().map(|c| c.to_json(host)).collect() + let mut attachments_json_vec = vec![]; + for attachment in attachments { + attachments_json_vec.push(attachment.to_json(host).await?); + } + attachments_json = Value::Array(attachments_json_vec); } } @@ -384,7 +394,7 @@ impl Cipher { }; json_object[key] = type_data_json; - json_object + Ok(json_object) } pub async fn update_users_revision(&self, conn: &mut DbConn) -> Vec { diff --git a/src/db/models/send.rs b/src/db/models/send.rs index c0bb0b33..bf82c181 100644 --- a/src/db/models/send.rs +++ b/src/db/models/send.rs @@ -1,7 +1,7 @@ use chrono::{NaiveDateTime, Utc}; use serde_json::Value; -use crate::util::LowerCase; +use crate::{config::PathType, util::LowerCase, CONFIG}; use super::{OrganizationId, User, UserId}; use id::SendId; @@ -226,7 +226,8 @@ impl Send { self.update_users_revision(conn).await; if self.atype == SendType::File as i32 { - std::fs::remove_dir_all(std::path::Path::new(&crate::CONFIG.sends_folder()).join(&self.uuid)).ok(); + let operator = CONFIG.opendal_operator_for_path_type(PathType::Sends)?; + operator.remove_all(&self.uuid).await.ok(); } db_run! { conn: { diff --git a/src/error.rs b/src/error.rs index 1061a08d..c6c77275 100644 --- a/src/error.rs +++ b/src/error.rs @@ -46,6 +46,7 @@ use jsonwebtoken::errors::Error as JwtErr; use lettre::address::AddressError as AddrErr; use lettre::error::Error as LettreErr; use lettre::transport::smtp::Error as SmtpErr; +use opendal::Error as OpenDALErr; use openssl::error::ErrorStack as SSLErr; use regex::Error as RegexErr; use reqwest::Error as ReqErr; @@ -95,6 +96,8 @@ make_error! { DieselCon(DieselConErr): _has_source, _api_error, Webauthn(WebauthnErr): _has_source, _api_error, + + OpenDAL(OpenDALErr): _has_source, _api_error, } impl std::fmt::Debug for Error { diff --git a/src/main.rs b/src/main.rs index 530c7b2c..6920be40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -75,16 +75,13 @@ async fn main() -> Result<(), Error> { let level = init_logging()?; check_data_folder().await; - auth::initialize_keys().unwrap_or_else(|e| { + auth::initialize_keys().await.unwrap_or_else(|e| { error!("Error creating private key '{}'\n{e:?}\nExiting Vaultwarden!", CONFIG.private_rsa_key()); exit(1); }); check_web_vault(); - create_dir(&CONFIG.icon_cache_folder(), "icon cache"); create_dir(&CONFIG.tmp_folder(), "tmp folder"); - create_dir(&CONFIG.sends_folder(), "sends folder"); - create_dir(&CONFIG.attachments_folder(), "attachments folder"); let pool = create_db_pool().await; schedule_jobs(pool.clone()); diff --git a/src/util.rs b/src/util.rs index 1f8d1c27..491490af 100644 --- a/src/util.rs +++ b/src/util.rs @@ -16,7 +16,7 @@ use tokio::{ time::{sleep, Duration}, }; -use crate::CONFIG; +use crate::{config::PathType, CONFIG}; pub struct AppHeaders(); @@ -816,6 +816,28 @@ pub fn is_global(ip: std::net::IpAddr) -> bool { ip.is_global() } +/// Saves a Rocket temporary file to the OpenDAL Operator at the given path. +/// +/// Ideally we would stream the Rocket TempFile directly to the OpenDAL +/// Operator, but Tempfile exposes a tokio ASyncBufRead trait, which OpenDAL +/// does not support. This could be reworked in the future to read and write +/// chunks to reduce copy overhead. +pub async fn save_temp_file( + path_type: PathType, + path: &str, + temp_file: rocket::fs::TempFile<'_>, +) -> Result<(), crate::Error> { + use tokio::io::AsyncReadExt as _; + + let operator = CONFIG.opendal_operator_for_path_type(path_type)?; + + let mut read_stream = temp_file.open().await?; + let mut buf = Vec::with_capacity(temp_file.len() as usize); + read_stream.read_to_end(&mut buf).await?; + operator.write(path, buf).await?; + Ok(()) +} + /// These are some tests to check that the implementations match /// The IPv4 can be all checked in 30 seconds or so and they are correct as of nightly 2023-07-17 /// The IPV6 can't be checked in a reasonable time, so we check over a hundred billion random ones, so far correct