@ -1,6 +1,6 @@
use std ::{ use std ::{
net ::IpAddr , net ::IpAddr ,
sync ::Arc , sync ::{ Arc , Mutex } ,
time ::{ Duration , SystemTime } , time ::{ Duration , SystemTime } ,
} ; } ;
@ -16,14 +16,13 @@ use rocket::{http::ContentType, response::Redirect, Route};
use tokio ::{ use tokio ::{
fs ::{ create_dir_all , remove_file , symlink_metadata , File } , fs ::{ create_dir_all , remove_file , symlink_metadata , File } ,
io ::{ AsyncReadExt , AsyncWriteExt } , io ::{ AsyncReadExt , AsyncWriteExt } ,
net ::lookup_host ,
} ; } ;
use html5gum ::{ Emitter , HtmlString , InfallibleTokenizer , Readable , StringReader , Tokenizer } ; use html5gum ::{ Emitter , HtmlString , InfallibleTokenizer , Readable , StringReader , Tokenizer } ;
use crate ::{ use crate ::{
error ::Error , error ::Error ,
util ::{ get_reqwest_client_builder , Cached } , util ::{ get_reqwest_client_builder , Cached , CustomDnsResolver , CustomResolverError } ,
CONFIG , CONFIG ,
} ; } ;
@ -49,48 +48,32 @@ static CLIENT: Lazy<Client> = Lazy::new(|| {
let icon_download_timeout = Duration ::from_secs ( CONFIG . icon_download_timeout ( ) ) ; let icon_download_timeout = Duration ::from_secs ( CONFIG . icon_download_timeout ( ) ) ;
let pool_idle_timeout = Duration ::from_secs ( 10 ) ; let pool_idle_timeout = Duration ::from_secs ( 10 ) ;
// Reuse the client between requests
// Reuse the client between requests
let client = get_reqwest_client_builder ( )
. cookie_provider ( Arc ::clone ( & cookie_store ) )
. timeout ( icon_download_timeout )
. pool_max_idle_per_host ( 5 ) // Configure the Hyper Pool to only have max 5 idle connections
. pool_idle_timeout ( pool_idle_timeout ) // Configure the Hyper Pool to timeout after 10 seconds
. trust_dns ( true )
. default_headers ( default_headers . clone ( ) ) ;
match client . build ( ) {
Ok ( client ) = > client ,
Err ( e ) = > {
error ! ( "Possible trust-dns error, trying with trust-dns disabled: '{e}'" ) ;
get_reqwest_client_builder ( ) get_reqwest_client_builder ( )
. cookie_provider ( cookie_store ) . cookie_provider ( Arc ::clone ( & cookie_store ) )
. timeout ( icon_download_timeout ) . timeout ( icon_download_timeout )
. pool_max_idle_per_host ( 5 ) // Configure the Hyper Pool to only have max 5 idle connections
. pool_max_idle_per_host ( 5 ) // Configure the Hyper Pool to only have max 5 idle connections
. pool_idle_timeout ( pool_idle_timeout ) // Configure the Hyper Pool to timeout after 10 seconds
. pool_idle_timeout ( pool_idle_timeout ) // Configure the Hyper Pool to timeout after 10 seconds
. trust_dns ( false ) . dns_resolver ( CustomDnsResolver ::instance ( ) )
. default_headers ( default_headers ) . default_headers ( default_headers . clone ( ) )
. build ( ) . build ( )
. expect ( "Failed to build client" ) . expect ( "Failed to build client" )
}
}
} ) ; } ) ;
// Build Regex only once since this takes a lot of time.
// Build Regex only once since this takes a lot of time.
static ICON_SIZE_REGEX : Lazy < Regex > = Lazy ::new ( | | Regex ::new ( r"(?x)(\d+)\D*(\d+)" ) . unwrap ( ) ) ; static ICON_SIZE_REGEX : Lazy < Regex > = Lazy ::new ( | | Regex ::new ( r"(?x)(\d+)\D*(\d+)" ) . unwrap ( ) ) ;
// Special HashMap which holds the user defined Regex to speedup matching the regex.
#[ get( " /<domain>/icon.png " ) ]
static ICON_BLACKLIST_REGEX : Lazy < dashmap ::DashMap < String , Regex > > = Lazy ::new ( dashmap ::DashMap ::new ) ; fn icon_external ( domain : & str ) -> Option < Redirect > {
async fn icon_redirect ( domain : & str , template : & str ) -> Option < Redirect > {
if ! is_valid_domain ( domain ) { if ! is_valid_domain ( domain ) {
warn ! ( "Invalid domain: {}" , domain ) ; warn ! ( "Invalid domain: {}" , domain ) ;
return None ; return None ;
} }
if check_domain_blacklist_reason ( domain ) . await . is_some ( ) { if is_domain_blacklisted ( domain ) {
return None ; return None ;
} }
let url = template . replace ( "{}" , domain ) ; let url = CONFIG . _icon_service_url ( ) . replace ( "{}" , domain ) ;
match CONFIG . icon_redirect_code ( ) { match CONFIG . icon_redirect_code ( ) {
301 = > Some ( Redirect ::moved ( url ) ) , // legacy permanent redirect
301 = > Some ( Redirect ::moved ( url ) ) , // legacy permanent redirect
302 = > Some ( Redirect ::found ( url ) ) , // legacy temporary redirect
302 = > Some ( Redirect ::found ( url ) ) , // legacy temporary redirect
@ -103,11 +86,6 @@ async fn icon_redirect(domain: &str, template: &str) -> Option<Redirect> {
} }
} }
#[ get( " /<domain>/icon.png " ) ]
async fn icon_external ( domain : & str ) -> Option < Redirect > {
icon_redirect ( domain , & CONFIG . _icon_service_url ( ) ) . await
}
#[ get( " /<domain>/icon.png " ) ] #[ get( " /<domain>/icon.png " ) ]
async fn icon_internal ( domain : & str ) -> Cached < ( ContentType , Vec < u8 > ) > { async fn icon_internal ( domain : & str ) -> Cached < ( ContentType , Vec < u8 > ) > {
const FALLBACK_ICON : & [ u8 ] = include_bytes ! ( "../static/images/fallback-icon.png" ) ; const FALLBACK_ICON : & [ u8 ] = include_bytes ! ( "../static/images/fallback-icon.png" ) ;
@ -166,153 +144,28 @@ fn is_valid_domain(domain: &str) -> bool {
true true
} }
/// TODO: This is extracted from IpAddr::is_global, which is unstable:
pub fn is_domain_blacklisted ( domain : & str ) -> bool {
/// https://doc.rust-lang.org/nightly/std/net/enum.IpAddr.html#method.is_global
let Some ( config_blacklist ) = CONFIG . icon_blacklist_regex ( ) else {
/// Remove once https://github.com/rust-lang/rust/issues/27709 is merged
return false ;
#[ allow(clippy::nonminimal_bool) ] } ;
#[ cfg(not(feature = " unstable " )) ]
fn is_global ( ip : IpAddr ) -> bool {
match ip {
IpAddr ::V4 ( ip ) = > {
// check if this address is 192.0.0.9 or 192.0.0.10. These addresses are the only two
// globally routable addresses in the 192.0.0.0/24 range.
if u32 ::from ( ip ) = = 0xc0000009 | | u32 ::from ( ip ) = = 0xc000000a {
return true ;
}
! ip . is_private ( )
& & ! ip . is_loopback ( )
& & ! ip . is_link_local ( )
& & ! ip . is_broadcast ( )
& & ! ip . is_documentation ( )
& & ! ( ip . octets ( ) [ 0 ] = = 100 & & ( ip . octets ( ) [ 1 ] & 0b1100_0000 = = 0b0100_0000 ) )
& & ! ( ip . octets ( ) [ 0 ] = = 192 & & ip . octets ( ) [ 1 ] = = 0 & & ip . octets ( ) [ 2 ] = = 0 )
& & ! ( ip . octets ( ) [ 0 ] & 240 = = 240 & & ! ip . is_broadcast ( ) )
& & ! ( ip . octets ( ) [ 0 ] = = 198 & & ( ip . octets ( ) [ 1 ] & 0xfe ) = = 18 )
// Make sure the address is not in 0.0.0.0/8
& & ip . octets ( ) [ 0 ] ! = 0
}
IpAddr ::V6 ( ip ) = > {
if ip . is_multicast ( ) & & ip . segments ( ) [ 0 ] & 0x000f = = 14 {
true
} else {
! ip . is_multicast ( )
& & ! ip . is_loopback ( )
& & ! ( ( ip . segments ( ) [ 0 ] & 0xffc0 ) = = 0xfe80 )
& & ! ( ( ip . segments ( ) [ 0 ] & 0xfe00 ) = = 0xfc00 )
& & ! ip . is_unspecified ( )
& & ! ( ( ip . segments ( ) [ 0 ] = = 0x2001 ) & & ( ip . segments ( ) [ 1 ] = = 0xdb8 ) )
}
}
}
}
#[ cfg(feature = " unstable " ) ] // Compiled domain blacklist
fn is_global ( ip : IpAddr ) -> bool { static COMPILED_BLACKLIST : Mutex < Option < ( String , Regex ) > > = Mutex ::new ( None ) ;
ip . is_global ( ) let mut guard = COMPILED_BLACKLIST . lock ( ) . unwrap ( ) ;
}
/// These are some tests to check that the implementations match
// If the stored regex is up to date, use it
/// The IPv4 can be all checked in 5 mins or so and they are correct as of nightly 2020-07-11
if let Some ( ( value , regex ) ) = & * guard {
/// The IPV6 can't be checked in a reasonable time, so we check about ten billion random ones, so far correct
if value = = & config_blacklist {
/// Note that the is_global implementation is subject to change as new IP RFCs are created
return regex . is_match ( domain ) ;
///
/// To run while showing progress output:
/// cargo test --features sqlite,unstable -- --nocapture --ignored
#[ cfg(test) ]
#[ cfg(feature = " unstable " ) ]
mod tests {
use super ::* ;
#[ test ]
#[ ignore ]
fn test_ipv4_global ( ) {
for a in 0 . . u8 ::MAX {
println ! ( "Iter: {}/255" , a ) ;
for b in 0 . . u8 ::MAX {
for c in 0 . . u8 ::MAX {
for d in 0 . . u8 ::MAX {
let ip = IpAddr ::V4 ( std ::net ::Ipv4Addr ::new ( a , b , c , d ) ) ;
assert_eq ! ( ip . is_global ( ) , is_global ( ip ) )
}
}
}
}
}
#[ test ]
#[ ignore ]
fn test_ipv6_global ( ) {
use ring ::rand ::{ SecureRandom , SystemRandom } ;
let mut v = [ 0 u8 ; 16 ] ;
let rand = SystemRandom ::new ( ) ;
for i in 0 . . 1_000 {
println ! ( "Iter: {}/1_000" , i ) ;
for _ in 0 . . 10_000_000 {
rand . fill ( & mut v ) . expect ( "Error generating random values" ) ;
let ip = IpAddr ::V6 ( std ::net ::Ipv6Addr ::new (
( v [ 14 ] as u16 ) < < 8 | v [ 15 ] as u16 ,
( v [ 12 ] as u16 ) < < 8 | v [ 13 ] as u16 ,
( v [ 10 ] as u16 ) < < 8 | v [ 11 ] as u16 ,
( v [ 8 ] as u16 ) < < 8 | v [ 9 ] as u16 ,
( v [ 6 ] as u16 ) < < 8 | v [ 7 ] as u16 ,
( v [ 4 ] as u16 ) < < 8 | v [ 5 ] as u16 ,
( v [ 2 ] as u16 ) < < 8 | v [ 3 ] as u16 ,
( v [ 0 ] as u16 ) < < 8 | v [ 1 ] as u16 ,
) ) ;
assert_eq ! ( ip . is_global ( ) , is_global ( ip ) )
}
}
} }
}
#[ derive(Clone) ]
enum DomainBlacklistReason {
Regex ,
IP ,
}
use cached ::proc_macro ::cached ;
#[ cached(key = " String " , convert = r# " { domain.to_string() } " #, size = 16, time = 60) ]
async fn check_domain_blacklist_reason ( domain : & str ) -> Option < DomainBlacklistReason > {
// First check the blacklist regex if there is a match.
// This prevents the blocked domain(s) from being leaked via a DNS lookup.
if let Some ( blacklist ) = CONFIG . icon_blacklist_regex ( ) {
// Use the pre-generate Regex stored in a Lazy HashMap if there's one, else generate it.
let is_match = if let Some ( regex ) = ICON_BLACKLIST_REGEX . get ( & blacklist ) {
regex . is_match ( domain )
} else {
// Clear the current list if the previous key doesn't exists.
// To prevent growing of the HashMap after someone has changed it via the admin interface.
if ICON_BLACKLIST_REGEX . len ( ) > = 1 {
ICON_BLACKLIST_REGEX . clear ( ) ;
} }
// Generate the regex to store in too the Lazy Static HashMap.
// If we don't have a regex stored, or it's not up to date, recreate it
let blacklist_ regex = Regex ::new ( & blacklist ) . unwrap ( ) ; let regex = Regex ::new ( & config_blacklist ) . unwrap ( ) ;
let is_match = blacklist_ regex. is_match ( domain ) ; let is_match = regex . is_match ( domain ) ;
ICON_BLACKLIST_REGEX . insert ( blacklist . clone ( ) , blacklist_ regex) ; * guard = Some ( ( config_blacklist , regex ) ) ;
is_match is_match
} ;
if is_match {
debug ! ( "Blacklisted domain: {} matched ICON_BLACKLIST_REGEX" , domain ) ;
return Some ( DomainBlacklistReason ::Regex ) ;
}
}
if CONFIG . icon_blacklist_non_global_ips ( ) {
if let Ok ( s ) = lookup_host ( ( domain , 0 ) ) . await {
for addr in s {
if ! is_global ( addr . ip ( ) ) {
debug ! ( "IP {} for domain '{}' is not a global IP!" , addr . ip ( ) , domain ) ;
return Some ( DomainBlacklistReason ::IP ) ;
}
}
}
}
None
} }
async fn get_icon ( domain : & str ) -> Option < ( Vec < u8 > , String ) > { async fn get_icon ( domain : & str ) -> Option < ( Vec < u8 > , String ) > {
@ -342,6 +195,13 @@ async fn get_icon(domain: &str) -> Option<(Vec<u8>, String)> {
Some ( ( icon . to_vec ( ) , icon_type . unwrap_or ( "x-icon" ) . to_string ( ) ) ) Some ( ( icon . to_vec ( ) , icon_type . unwrap_or ( "x-icon" ) . to_string ( ) ) )
} }
Err ( e ) = > { Err ( e ) = > {
// If this error comes from the custom resolver, this means this is a blacklisted domain
// or non global IP, don't save the miss file in this case to avoid leaking it
if let Some ( error ) = CustomResolverError ::downcast_ref ( & e ) {
warn ! ( "{error}" ) ;
return None ;
}
warn ! ( "Unable to download icon: {:?}" , e ) ; warn ! ( "Unable to download icon: {:?}" , e ) ;
let miss_indicator = path + ".miss" ; let miss_indicator = path + ".miss" ;
save_icon ( & miss_indicator , & [ ] ) . await ; save_icon ( & miss_indicator , & [ ] ) . await ;
@ -491,12 +351,12 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
let ssldomain = format ! ( "https://{domain}" ) ; let ssldomain = format ! ( "https://{domain}" ) ;
let httpdomain = format ! ( "http://{domain}" ) ; let httpdomain = format ! ( "http://{domain}" ) ;
// First check the domain as given during the request for both HTTPS and HTTP .
// First check the domain as given during the request for HTTPS.
let resp = match get_page ( & ssldomain ) . or_else ( | _ | get_page ( & httpdomain ) ) . await { let resp = match get_page ( & ssldomain ) . await {
Ok ( c ) = > Ok ( c ) , Err ( e ) if CustomResolverError ::downcast_ref ( & e ) . is_none ( ) = > {
Err ( e ) = > { // If we get an error that is not caused by the blacklist, we retry with HTTP
let mut sub_resp = Err ( e ) ; match get_page ( & httpdomain ) . await {
mut sub_resp @ Err ( _ ) = > {
// When the domain is not an IP, and has more then one dot, remove all subdomains.
// When the domain is not an IP, and has more then one dot, remove all subdomains.
let is_ip = domain . parse ::< IpAddr > ( ) ; let is_ip = domain . parse ::< IpAddr > ( ) ;
if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) > 1 { if is_ip . is_err ( ) & & domain . matches ( '.' ) . count ( ) > 1 {
@ -527,6 +387,12 @@ async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
} }
sub_resp sub_resp
} }
res = > res ,
}
}
// If we get a result or a blacklist error, just continue
res = > res ,
} ; } ;
// Create the iconlist
// Create the iconlist
@ -573,21 +439,12 @@ async fn get_page(url: &str) -> Result<Response, Error> {
} }
async fn get_page_with_referer ( url : & str , referer : & str ) -> Result < Response , Error > { async fn get_page_with_referer ( url : & str , referer : & str ) -> Result < Response , Error > {
match check_domain_blacklist_reason ( url ::Url ::parse ( url ) . unwrap ( ) . host_str ( ) . unwrap_or_default ( ) ) . await {
Some ( DomainBlacklistReason ::Regex ) = > warn ! ( "Favicon '{}' is from a blacklisted domain!" , url ) ,
Some ( DomainBlacklistReason ::IP ) = > warn ! ( "Favicon '{}' is hosted on a non-global IP!" , url ) ,
None = > ( ) ,
}
let mut client = CLIENT . get ( url ) ; let mut client = CLIENT . get ( url ) ;
if ! referer . is_empty ( ) { if ! referer . is_empty ( ) {
client = client . header ( "Referer" , referer ) client = client . header ( "Referer" , referer )
} }
match client . send ( ) . await { Ok ( client . send ( ) . await ? . error_for_status ( ) ? )
Ok ( c ) = > c . error_for_status ( ) . map_err ( Into ::into ) ,
Err ( e ) = > err_silent ! ( format ! ( "{e}" ) ) ,
}
} }
/// Returns a Integer with the priority of the type of the icon which to prefer.
/// Returns a Integer with the priority of the type of the icon which to prefer.
@ -670,12 +527,6 @@ fn parse_sizes(sizes: &str) -> (u16, u16) {
} }
async fn download_icon ( domain : & str ) -> Result < ( Bytes , Option < & str > ) , Error > { async fn download_icon ( domain : & str ) -> Result < ( Bytes , Option < & str > ) , Error > {
match check_domain_blacklist_reason ( domain ) . await {
Some ( DomainBlacklistReason ::Regex ) = > err_silent ! ( "Domain is blacklisted" , domain ) ,
Some ( DomainBlacklistReason ::IP ) = > err_silent ! ( "Host resolves to a non-global IP" , domain ) ,
None = > ( ) ,
}
let icon_result = get_icon_url ( domain ) . await ? ; let icon_result = get_icon_url ( domain ) . await ? ;
let mut buffer = Bytes ::new ( ) ; let mut buffer = Bytes ::new ( ) ;
@ -711,8 +562,8 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
_ = > debug ! ( "Extracted icon from data:image uri is invalid" ) , _ = > debug ! ( "Extracted icon from data:image uri is invalid" ) ,
} ; } ;
} else { } else {
match get_page_with_referer ( & icon . href , & icon_result . referer ) . await { let res = get_page_with_referer ( & icon . href , & icon_result . referer ) . await ? ;
Ok ( res ) = > {
buffer = stream_to_bytes_limit ( res , 5120 * 1024 ) . await ? ; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
buffer = stream_to_bytes_limit ( res , 5120 * 1024 ) . await ? ; // 5120KB/5MB for each icon max (Same as icons.bitwarden.net)
// Check if the icon type is allowed, else try an icon from the list.
// Check if the icon type is allowed, else try an icon from the list.
@ -725,9 +576,6 @@ async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
info ! ( "Downloaded icon from {}" , icon . href ) ; info ! ( "Downloaded icon from {}" , icon . href ) ;
break ; break ;
} }
Err ( e ) = > debug ! ( "{:?}" , e ) ,
} ;
}
} }
if buffer . is_empty ( ) { if buffer . is_empty ( ) {
Mathijs van Veluw
11 months ago
GitHub