You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
843 lines
31 KiB
843 lines
31 KiB
use std::{
|
|
collections::HashMap,
|
|
net::IpAddr,
|
|
sync::{Arc, LazyLock},
|
|
time::{Duration, SystemTime},
|
|
};
|
|
|
|
use bytes::{Bytes, BytesMut};
|
|
use futures::{TryFutureExt, stream::StreamExt};
|
|
use html5gum::{Emitter, HtmlString, Readable, StringReader, Tokenizer};
|
|
use regex::Regex;
|
|
use reqwest::{
|
|
Client, Response,
|
|
header::{self, HeaderMap, HeaderValue},
|
|
};
|
|
use rocket::{Route, http::ContentType, response::Redirect};
|
|
use svg_hush::{Filter, data_url_filter};
|
|
|
|
use crate::{
|
|
CONFIG,
|
|
config::PathType,
|
|
error::Error,
|
|
http_client::{CustomHttpClientError, get_reqwest_client_builder, get_valid_host, should_block_host},
|
|
util::Cached,
|
|
};
|
|
|
|
pub fn routes() -> Vec<Route> {
|
|
if CONFIG.icon_service().as_str() == "internal" {
|
|
routes![icon_internal]
|
|
} else {
|
|
routes![icon_external]
|
|
}
|
|
}
|
|
|
|
static CLIENT: LazyLock<Client> = LazyLock::new(|| {
|
|
// Generate the default headers
|
|
let mut default_headers = HeaderMap::new();
|
|
default_headers.insert(
|
|
header::USER_AGENT,
|
|
HeaderValue::from_static(
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
|
),
|
|
);
|
|
default_headers.insert(header::ACCEPT, HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"));
|
|
default_headers.insert(header::ACCEPT_LANGUAGE, HeaderValue::from_static("en-US,en;q=0.9"));
|
|
default_headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-cache"));
|
|
default_headers.insert(header::PRAGMA, HeaderValue::from_static("no-cache"));
|
|
default_headers.insert(header::UPGRADE_INSECURE_REQUESTS, HeaderValue::from_static("1"));
|
|
|
|
default_headers.insert("Sec-Ch-Ua-Mobile", HeaderValue::from_static("?0"));
|
|
default_headers.insert("Sec-Ch-Ua-Platform", HeaderValue::from_static("Linux"));
|
|
default_headers.insert(
|
|
"Sec-Ch-Ua",
|
|
HeaderValue::from_static("\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"138\", \"Google Chrome\";v=\"138\""),
|
|
);
|
|
|
|
default_headers.insert("Sec-Fetch-Site", HeaderValue::from_static("none"));
|
|
default_headers.insert("Sec-Fetch-Mode", HeaderValue::from_static("navigate"));
|
|
default_headers.insert("Sec-Fetch-User", HeaderValue::from_static("?1"));
|
|
default_headers.insert("Sec-Fetch-Dest", HeaderValue::from_static("document"));
|
|
|
|
// Generate the cookie store
|
|
let cookie_store = Arc::new(Jar::default());
|
|
|
|
let icon_download_timeout = Duration::from_secs(u64::from(CONFIG.icon_download_timeout()));
|
|
let pool_idle_timeout = Duration::from_secs(10);
|
|
// Reuse the client between requests
|
|
get_reqwest_client_builder(true)
|
|
.cookie_provider(Arc::clone(&cookie_store))
|
|
.timeout(icon_download_timeout)
|
|
.pool_max_idle_per_host(5) // Configure the Hyper Pool to only have max 5 idle connections
|
|
.pool_idle_timeout(pool_idle_timeout) // Configure the Hyper Pool to timeout after 10 seconds
|
|
.default_headers(default_headers.clone())
|
|
.http1_title_case_headers()
|
|
.build()
|
|
.expect("Failed to build client")
|
|
});
|
|
|
|
// The function name `icon_external` is checked in the `on_response` function in `AppHeaders`
|
|
// It is used to prevent sending a specific header which breaks icon downloads.
|
|
// If this function needs to be renamed, also adjust the code in `util.rs`
|
|
#[get("/<host>/icon.png")]
|
|
fn icon_external(host: &str) -> Cached<Option<Redirect>> {
|
|
let Ok(host) = get_valid_host(host) else {
|
|
warn!("Invalid host: {}", host.escape_debug());
|
|
return Cached::ttl(None, CONFIG.icon_cache_negttl(), true);
|
|
};
|
|
|
|
if should_block_host(&host).is_err() {
|
|
warn!("Blocked address: {host}");
|
|
return Cached::ttl(None, CONFIG.icon_cache_negttl(), true);
|
|
}
|
|
|
|
let url = CONFIG._icon_service_url().replace("{}", &host.to_string());
|
|
let redir = match CONFIG.icon_redirect_code() {
|
|
301 => Some(Redirect::moved(url)), // legacy permanent redirect
|
|
302 => Some(Redirect::found(url)), // legacy temporary redirect
|
|
307 => Some(Redirect::temporary(url)),
|
|
308 => Some(Redirect::permanent(url)),
|
|
_ => {
|
|
error!("Unexpected redirect code {}", CONFIG.icon_redirect_code());
|
|
None
|
|
}
|
|
};
|
|
Cached::ttl(redir, CONFIG.icon_cache_ttl(), true)
|
|
}
|
|
|
|
#[get("/<host>/icon.png")]
|
|
async fn icon_internal(host: &str) -> Cached<(ContentType, Vec<u8>)> {
|
|
const FALLBACK_ICON: &[u8] = include_bytes!("../static/images/fallback-icon.png");
|
|
|
|
let Ok(host) = get_valid_host(host) else {
|
|
warn!("Invalid host: {}", host.escape_debug());
|
|
return Cached::ttl(
|
|
(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()),
|
|
CONFIG.icon_cache_negttl(),
|
|
true,
|
|
);
|
|
};
|
|
|
|
if should_block_host(&host).is_err() {
|
|
warn!("Blocked address: {host}");
|
|
return Cached::ttl(
|
|
(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()),
|
|
CONFIG.icon_cache_negttl(),
|
|
true,
|
|
);
|
|
}
|
|
|
|
match get_icon(&host.to_string()).await {
|
|
Some((icon, icon_type)) => {
|
|
Cached::ttl((ContentType::new("image", icon_type), icon), CONFIG.icon_cache_ttl(), true)
|
|
}
|
|
_ => Cached::ttl((ContentType::new("image", "png"), FALLBACK_ICON.to_vec()), CONFIG.icon_cache_negttl(), true),
|
|
}
|
|
}
|
|
|
|
async fn get_icon(domain: &str) -> Option<(Vec<u8>, String)> {
|
|
let path = format!("{domain}.png");
|
|
|
|
// Check for expiration of negatively cached copy
|
|
if icon_is_negcached(&path).await {
|
|
return None;
|
|
}
|
|
|
|
if let Some(icon) = get_cached_icon(&path).await {
|
|
let icon_type = get_icon_type(&icon).unwrap_or("x-icon");
|
|
return Some((icon, icon_type.to_owned()));
|
|
}
|
|
|
|
if CONFIG.disable_icon_download() {
|
|
return None;
|
|
}
|
|
|
|
// Get the icon, or None in case of error
|
|
match download_icon(domain).await {
|
|
Ok((icon, icon_type)) => {
|
|
save_icon(&path, icon.to_vec()).await;
|
|
Some((icon.to_vec(), icon_type.unwrap_or("x-icon").to_owned()))
|
|
}
|
|
Err(e) => {
|
|
// If this error comes from the custom resolver, this means this is a blocked domain
|
|
// or non global IP, don't save the miss file in this case to avoid leaking it
|
|
if let Some(error) = CustomHttpClientError::downcast_ref(&e) {
|
|
warn!("{error}");
|
|
return None;
|
|
}
|
|
|
|
warn!("Unable to download icon: {e:?}");
|
|
let miss_indicator = path + ".miss";
|
|
save_icon(&miss_indicator, vec![]).await;
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn get_cached_icon(path: &str) -> Option<Vec<u8>> {
|
|
// Check for expiration of successfully cached copy
|
|
if icon_is_expired(path).await {
|
|
return None;
|
|
}
|
|
|
|
// Try to read the cached icon, and return it if it exists
|
|
if let Ok(operator) = CONFIG.opendal_operator_for_path_type(&PathType::IconCache)
|
|
&& let Ok(buf) = operator.read(path).await
|
|
{
|
|
return Some(buf.to_vec());
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
async fn file_is_expired(path: &str, ttl: u32) -> Result<bool, Error> {
|
|
let operator = CONFIG.opendal_operator_for_path_type(&PathType::IconCache)?;
|
|
let meta = operator.stat(path).await?;
|
|
let modified =
|
|
meta.last_modified().ok_or_else(|| std::io::Error::other(format!("No last modified time for `{path}`")))?;
|
|
let age = SystemTime::now().duration_since(modified.into())?;
|
|
|
|
Ok(ttl > 0 && u64::from(ttl) <= age.as_secs())
|
|
}
|
|
|
|
async fn icon_is_negcached(path: &str) -> bool {
|
|
let miss_indicator = path.to_owned() + ".miss";
|
|
let expired = file_is_expired(&miss_indicator, CONFIG.icon_cache_negttl()).await;
|
|
|
|
match expired {
|
|
// No longer negatively cached, drop the marker
|
|
Ok(true) => {
|
|
match CONFIG.opendal_operator_for_path_type(&PathType::IconCache) {
|
|
Ok(operator) => {
|
|
if let Err(e) = operator.delete(&miss_indicator).await {
|
|
error!("Could not remove negative cache indicator for icon {path:?}: {e:?}");
|
|
}
|
|
}
|
|
Err(e) => error!("Could not remove negative cache indicator for icon {path:?}: {e:?}"),
|
|
}
|
|
false
|
|
}
|
|
// The marker hasn't expired yet.
|
|
Ok(false) => true,
|
|
// The marker is missing or inaccessible in some way.
|
|
Err(_) => false,
|
|
}
|
|
}
|
|
|
|
async fn icon_is_expired(path: &str) -> bool {
|
|
let expired = file_is_expired(path, CONFIG.icon_cache_ttl()).await;
|
|
expired.unwrap_or(true)
|
|
}
|
|
|
|
struct Icon {
|
|
priority: u8,
|
|
href: String,
|
|
}
|
|
|
|
impl Icon {
|
|
const fn new(priority: u8, href: String) -> Self {
|
|
Self {
|
|
priority,
|
|
href,
|
|
}
|
|
}
|
|
}
|
|
|
|
fn get_favicons_node(dom: Tokenizer<StringReader<'_>, FaviconEmitter>, icons: &mut Vec<Icon>, url: &url::Url) {
|
|
const TAG_LINK: &[u8] = b"link";
|
|
const TAG_BASE: &[u8] = b"base";
|
|
const TAG_HEAD: &[u8] = b"head";
|
|
const ATTR_HREF: &[u8] = b"href";
|
|
const ATTR_SIZES: &[u8] = b"sizes";
|
|
|
|
let mut base_href = None;
|
|
let mut icon_tags = Vec::new();
|
|
for Ok(mut token) in dom {
|
|
match token.tag.name.as_slice() {
|
|
TAG_LINK => icon_tags.push(token.tag),
|
|
TAG_BASE if base_href.is_none() => {
|
|
base_href = token.tag.attributes.remove(ATTR_HREF);
|
|
}
|
|
TAG_HEAD if token.closing => break,
|
|
b"body" | b"frameset" if !token.closing => break,
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
let parsed_base = base_href.as_ref().and_then(|value| {
|
|
let href = str::from_utf8(value).ok()?;
|
|
debug!("Found base href: {}", href.escape_debug());
|
|
url.join(href).ok().filter(|base| !matches!(base.scheme(), "data" | "javascript"))
|
|
});
|
|
let base_url = parsed_base.as_ref().unwrap_or(url);
|
|
|
|
for icon_tag in icon_tags {
|
|
if let Some(icon_href) = icon_tag.attributes.get(ATTR_HREF)
|
|
&& let Ok(full_href) = base_url.join(str::from_utf8(icon_href).unwrap_or_default())
|
|
{
|
|
let sizes = icon_tag.attributes.get(ATTR_SIZES).map_or_default(|v| str::from_utf8(v).unwrap_or_default());
|
|
// Only check the full_href path during priority checks so query strings like `/path/to/icon.png?v=xxx` will be ignored
|
|
let priority = get_icon_priority(full_href.path(), sizes);
|
|
icons.push(Icon::new(priority, full_href.to_string()));
|
|
}
|
|
}
|
|
}
|
|
|
|
struct IconUrlResult {
|
|
iconlist: Vec<Icon>,
|
|
referer: String,
|
|
}
|
|
|
|
/// Returns a IconUrlResult which holds a Vector IconList and a string which holds the referer.
|
|
/// There will always two items within the iconlist which holds http(s)://domain.tld/favicon.ico.
|
|
/// This does not mean that location exists, but (it) is the default location the browser uses.
|
|
///
|
|
/// # Argument
|
|
/// * `domain` - A string which holds the domain with extension.
|
|
///
|
|
/// # Example
|
|
/// ```
|
|
/// let icon_result = get_icon_url("github.com").await?;
|
|
/// let icon_result = get_icon_url("vaultwarden.discourse.group").await?;
|
|
/// ```
|
|
async fn get_icon_url(domain: &str) -> Result<IconUrlResult, Error> {
|
|
// Default URL with secure and insecure schemes
|
|
let ssldomain = format!("https://{domain}");
|
|
let httpdomain = format!("http://{domain}");
|
|
|
|
// First check the domain as given during the request for HTTPS.
|
|
let resp = match get_page(&ssldomain).await {
|
|
Err(e) if CustomHttpClientError::downcast_ref(&e).is_none() => {
|
|
// If we get an error that is not caused by the blacklist, we retry with HTTP
|
|
match get_page(&httpdomain).await {
|
|
mut sub_resp @ Err(_) => {
|
|
if domain.parse::<IpAddr>().is_err() {
|
|
if let Some((dot, _)) = domain.rmatch_indices('.').nth(1) {
|
|
// Use the last two labels when the domain has at least two dots.
|
|
let base_domain = &domain[dot + 1..];
|
|
if get_valid_host(base_domain).is_ok() {
|
|
let sslbase = format!("https://{base_domain}");
|
|
let httpbase = format!("http://{base_domain}");
|
|
debug!("[get_icon_url]: Trying without subdomains '{base_domain}'");
|
|
|
|
sub_resp = get_page(&sslbase).or_else(|_| get_page(&httpbase)).await;
|
|
}
|
|
} else {
|
|
let www_domain = format!("www.{domain}");
|
|
if get_valid_host(&www_domain).is_ok() {
|
|
let sslwww = format!("https://{www_domain}");
|
|
let httpwww = format!("http://{www_domain}");
|
|
debug!("[get_icon_url]: Trying with www. prefix '{www_domain}'");
|
|
|
|
sub_resp = get_page(&sslwww).or_else(|_| get_page(&httpwww)).await;
|
|
}
|
|
}
|
|
}
|
|
sub_resp
|
|
}
|
|
res => res,
|
|
}
|
|
}
|
|
|
|
// If we get a result or a blacklist error, just continue
|
|
res => res,
|
|
};
|
|
|
|
// Create the iconlist
|
|
let mut iconlist: Vec<Icon> = Vec::new();
|
|
let mut referer = String::new();
|
|
|
|
if let Ok(content) = resp {
|
|
// Extract the URL from the response in case redirects occurred (like @ gitlab.com)
|
|
let url = content.url().clone();
|
|
|
|
// Set the referer to be used on the final request, some sites check this.
|
|
// Mostly used to prevent direct linking and other security reasons.
|
|
referer = url.to_string();
|
|
|
|
// Add the fallback favicon.ico and apple-touch-icon.png to the list with the domain the content responded from.
|
|
iconlist.push(Icon::new(35, String::from(url.join("/favicon.ico").unwrap())));
|
|
iconlist.push(Icon::new(40, String::from(url.join("/apple-touch-icon.png").unwrap())));
|
|
|
|
// 384KB should be more than enough for the HTML, though as we only really need the HTML header.
|
|
let limited_reader = stream_to_bytes_limit(content, 384 * 1024).await?;
|
|
let dom = Tokenizer::new_with_emitter(limited_reader.as_ref().to_reader(), FaviconEmitter::default());
|
|
get_favicons_node(dom, &mut iconlist, &url);
|
|
} else {
|
|
// Add the default favicon.ico to the list with just the given domain
|
|
iconlist.push(Icon::new(35, format!("{ssldomain}/favicon.ico")));
|
|
iconlist.push(Icon::new(40, format!("{ssldomain}/apple-touch-icon.png")));
|
|
iconlist.push(Icon::new(35, format!("{httpdomain}/favicon.ico")));
|
|
iconlist.push(Icon::new(40, format!("{httpdomain}/apple-touch-icon.png")));
|
|
}
|
|
|
|
// Sort the iconlist by priority
|
|
iconlist.sort_by_key(|x| x.priority);
|
|
|
|
// There always is an icon in the list, so no need to check if it exists, and just return the first one
|
|
Ok(IconUrlResult {
|
|
iconlist,
|
|
referer,
|
|
})
|
|
}
|
|
|
|
async fn get_page(url: &str) -> Result<Response, Error> {
|
|
get_page_with_referer(url, "").await
|
|
}
|
|
|
|
async fn get_page_with_referer(url: &str, referer: &str) -> Result<Response, Error> {
|
|
// The resolver only sees hosts needing name resolution, so IP-literal hrefs from
|
|
// attacker-controlled HTML never reach `post_resolve()`. Check them here.
|
|
let Ok(parsed_url) = url::Url::parse(url) else {
|
|
err_silent!("Invalid URL", url)
|
|
};
|
|
|
|
if !matches!(parsed_url.scheme(), "http" | "https") {
|
|
err_silent!("Invalid scheme", url)
|
|
}
|
|
|
|
let Some(host) = parsed_url.host() else {
|
|
err_silent!("Invalid host", url)
|
|
};
|
|
|
|
should_block_host(&host)?;
|
|
|
|
let mut client = CLIENT.get(url);
|
|
if !referer.is_empty() {
|
|
client = client.header("Referer", referer);
|
|
}
|
|
|
|
Ok(client.send().await?.error_for_status()?)
|
|
}
|
|
|
|
/// Returns a Integer with the priority of the type of the icon which to prefer.
|
|
/// The lower the number the better.
|
|
///
|
|
/// # Arguments
|
|
/// * `href` - A string which holds the href value or relative path.
|
|
/// * `sizes` - The size of the icon if available as a <width>x<height> value like 32x32.
|
|
///
|
|
/// # Example
|
|
/// ```
|
|
/// priority1 = get_icon_priority("http://example.com/path/to/a/favicon.png", "32x32");
|
|
/// priority2 = get_icon_priority("https://example.com/path/to/a/favicon.ico", "");
|
|
/// ```
|
|
fn get_icon_priority(href: &str, sizes: &str) -> u8 {
|
|
static PRIORITY_MAP: LazyLock<HashMap<&'static str, u8>> =
|
|
LazyLock::new(|| HashMap::from([("png", 10), ("jpg", 20), ("jpeg", 20)]));
|
|
|
|
// Check if there is a dimension set
|
|
let (width, height) = parse_sizes(sizes);
|
|
|
|
// Check if there is a size given
|
|
if width != 0 && height != 0 {
|
|
// Only allow square dimensions
|
|
if width == height {
|
|
// Change priority by given size
|
|
if width == 32 {
|
|
1
|
|
} else if width == 64 {
|
|
2
|
|
} else if (24..=192).contains(&width) {
|
|
3
|
|
} else if width == 16 {
|
|
4
|
|
} else {
|
|
5
|
|
}
|
|
// There are dimensions available, but the image is not a square
|
|
} else {
|
|
200
|
|
}
|
|
} else {
|
|
match href.rsplit_once('.') {
|
|
Some((_, extension)) => PRIORITY_MAP.get(&*extension.to_ascii_lowercase()).copied().unwrap_or(30),
|
|
None => 30,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Returns a Tuple with the width and height as a separate value extracted from the sizes attribute
|
|
/// It will return 0 for both values if no match has been found.
|
|
///
|
|
/// # Arguments
|
|
/// * `sizes` - The size of the icon if available as a <width>x<height> value like 32x32.
|
|
///
|
|
/// # Example
|
|
/// ```
|
|
/// let (width, height) = parse_sizes("64x64"); // (64, 64)
|
|
/// let (width, height) = parse_sizes("x128x128"); // (128, 128)
|
|
/// let (width, height) = parse_sizes("32"); // (0, 0)
|
|
/// ```
|
|
fn parse_sizes(sizes: &str) -> (u16, u16) {
|
|
static ICON_SIZE_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(\d+)\D+(\d+)").unwrap());
|
|
|
|
ICON_SIZE_REGEX
|
|
.captures(sizes)
|
|
.and_then(|c| Some((c[1].parse::<u16>().ok()?, c[2].parse::<u16>().ok()?)))
|
|
.unwrap_or((0, 0))
|
|
}
|
|
|
|
async fn download_icon(domain: &str) -> Result<(Bytes, Option<&str>), Error> {
|
|
let icon_result = get_icon_url(domain).await?;
|
|
|
|
let mut icons = icon_result.iconlist.iter().take(5).peekable();
|
|
while let Some(icon) = icons.next() {
|
|
let mut buffer = if icon.href.starts_with("data:image") {
|
|
let Ok(datauri) = data_url::DataUrl::process(&icon.href) else {
|
|
continue;
|
|
};
|
|
let mut body = BytesMut::new();
|
|
if datauri
|
|
.decode::<_, ()>(|bytes| {
|
|
body.extend_from_slice(bytes);
|
|
Ok(())
|
|
})
|
|
.is_err()
|
|
{
|
|
debug!("Extracted icon from data:image uri is invalid");
|
|
continue;
|
|
}
|
|
// Preserve the minimum size required for data URI icons.
|
|
if body.len() < 67 {
|
|
continue;
|
|
}
|
|
body.freeze()
|
|
} else {
|
|
debug!("Trying {}", icon.href);
|
|
let res = match get_page_with_referer(&icon.href, &icon_result.referer).await {
|
|
Ok(r) => r,
|
|
Err(e) if icons.peek().is_none() => return Err(e),
|
|
// If blacklisted stop immediately instead of checking the rest of the icons. see explanation and actual handling inside get_icon()
|
|
Err(e) if CustomHttpClientError::downcast_ref(&e).is_some() => return Err(e),
|
|
Err(e) => {
|
|
warn!("Unable to download icon: {e:?}");
|
|
continue;
|
|
}
|
|
};
|
|
stream_to_bytes_limit(res, 5120 * 1024).await?
|
|
};
|
|
|
|
let Some(icon_type) = get_icon_type(&buffer) else {
|
|
debug!("Icon candidate for {domain} is not a valid image type");
|
|
continue;
|
|
};
|
|
|
|
// Validate SVG format early incase it is invalid, it will continue to the next icon in the list
|
|
if icon_type == "svg+xml" {
|
|
let mut svg_filter = Filter::new();
|
|
svg_filter.set_data_url_filter(data_url_filter::allow_standard_images);
|
|
let mut sanitized_svg = Vec::new();
|
|
if svg_filter.filter(&*buffer, &mut sanitized_svg).is_err() || sanitized_svg.is_empty() {
|
|
debug!("SVG icon candidate for {domain} could not be sanitized");
|
|
continue;
|
|
}
|
|
buffer = sanitized_svg.into();
|
|
}
|
|
|
|
info!("Found valid icon for {domain}");
|
|
return Ok((buffer, Some(icon_type)));
|
|
}
|
|
|
|
err_silent!("Empty response or unable find a valid icon", domain);
|
|
}
|
|
|
|
async fn save_icon(path: &str, icon: Vec<u8>) {
|
|
let operator = match CONFIG.opendal_operator_for_path_type(&PathType::IconCache) {
|
|
Ok(operator) => operator,
|
|
Err(e) => {
|
|
warn!("Failed to get OpenDAL operator while saving icon: {e}");
|
|
return;
|
|
}
|
|
};
|
|
|
|
if let Err(e) = operator.write(path, icon).await {
|
|
warn!("Unable to save icon: {e:?}");
|
|
}
|
|
}
|
|
|
|
fn get_icon_type(bytes: &[u8]) -> Option<&'static str> {
|
|
fn check_svg_after_xml_declaration(bytes: &[u8]) -> Option<&'static str> {
|
|
// Look for SVG tag within the first 1KB
|
|
const SVG_MATCH: &[u8; 4] = b"<svg";
|
|
let content = &bytes[..bytes.len().min(1024)];
|
|
// Even though XML is case-sensitive we will do a case insensitive match
|
|
content.windows(SVG_MATCH.len()).any(|w| w.eq_ignore_ascii_case(SVG_MATCH)).then_some("svg+xml")
|
|
}
|
|
|
|
// Some details can be found here:
|
|
// - https://www.garykessler.net/library/file_sigs_GCK_latest.html
|
|
// - https://en.wikipedia.org/wiki/List_of_file_signatures
|
|
match bytes {
|
|
[137, 80, 78, 71, 13, 10, 26, 10, ..] => Some("png"),
|
|
[0, 0, 1, 0, n1, n2, ..] if u16::from_le_bytes([*n1, *n2]) > 0 => Some("x-icon"), // https://en.wikipedia.org/wiki/ICO_(file_format)
|
|
[82, 73, 70, 70, _, _, _, _, 87, 69, 66, 80, ..] => Some("webp"), // Only match WebP Images
|
|
[255, 216, 255, b, ..] if *b >= 0xC0 => Some("jpeg"),
|
|
[71, 73, 70, 56, 55 | 57, 97, ..] => Some("gif"),
|
|
[66, 77, _, _, _, _, 0, 0, 0, 0, ..] => Some("bmp"), // https://en.wikipedia.org/wiki/BMP_file_format
|
|
[60, 115, 118, 103, ..] => Some("svg+xml"), // Normal svg
|
|
[60, 63, 120, 109, 108, ..] => check_svg_after_xml_declaration(bytes), // An svg starting with <?xml
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// Minimize the amount of bytes to be parsed from a reqwest result.
|
|
/// This prevents very long parsing and memory usage.
|
|
async fn stream_to_bytes_limit(res: Response, max_size: usize) -> Result<Bytes, reqwest::Error> {
|
|
let mut stream = res.bytes_stream().take(max_size);
|
|
let mut buf = BytesMut::new();
|
|
let mut size = 0;
|
|
while let Some(chunk) = stream.next().await {
|
|
// It is possible that there might occur UnexpectedEof errors or others
|
|
// This is most of the time no issue, and if there is no chunked data anymore or at all parsing the HTML will not happen anyway.
|
|
// Therefore if chunk is an err, just break and continue with the data be have received.
|
|
if chunk.is_err() {
|
|
break;
|
|
}
|
|
let chunk = &chunk?;
|
|
size += chunk.len();
|
|
buf.extend(chunk);
|
|
if size >= max_size {
|
|
break;
|
|
}
|
|
}
|
|
Ok(buf.freeze())
|
|
}
|
|
|
|
/// This is an implementation of the default Cookie Jar from Reqwest and reqwest_cookie_store build by pfernie.
|
|
/// The default cookie jar used by Reqwest keeps all the cookies based upon the Max-Age or Expires which could be a long time.
|
|
/// That could be used for tracking, to prevent this we force the lifespan of the cookies to always be max two minutes.
|
|
/// A Cookie Jar is needed because some sites force a redirect with cookies to verify if a request uses cookies or not.
|
|
use cookie_store::CookieStore;
|
|
#[derive(Default)]
|
|
pub struct Jar(std::sync::RwLock<CookieStore>);
|
|
|
|
impl reqwest::cookie::CookieStore for Jar {
|
|
fn set_cookies(&self, cookie_headers: &mut dyn Iterator<Item = &HeaderValue>, url: &url::Url) {
|
|
use cookie::{Cookie as RawCookie, ParseError as RawCookieParseError, time::Duration};
|
|
|
|
let mut cookie_store = self.0.write().unwrap();
|
|
|
|
// The store only skips expired cookies when reading them, so remove them here or they are kept forever
|
|
let expired: Vec<_> = cookie_store
|
|
.iter_any()
|
|
.filter(|c| c.is_expired())
|
|
.map(|c| (String::from(&c.domain), String::from(&c.path), c.name().to_owned()))
|
|
.collect();
|
|
for (domain, path, name) in expired {
|
|
cookie_store.remove(&domain, &path, &name);
|
|
}
|
|
|
|
let cookies = cookie_headers.filter_map(|val| {
|
|
str::from_utf8(val.as_bytes())
|
|
.map_err(RawCookieParseError::from)
|
|
.and_then(RawCookie::parse)
|
|
.map(|mut c| {
|
|
c.set_expires(None);
|
|
c.set_max_age(Some(Duration::minutes(2)));
|
|
c.into_owned()
|
|
})
|
|
.ok()
|
|
});
|
|
cookie_store.store_response_cookies(cookies, url);
|
|
}
|
|
|
|
fn cookies(&self, url: &url::Url) -> Option<HeaderValue> {
|
|
let cookie_store = self.0.read().unwrap();
|
|
let mut s = String::new();
|
|
for (name, value) in cookie_store.get_request_values(url) {
|
|
if !s.is_empty() {
|
|
s.push_str("; ");
|
|
}
|
|
s.push_str(name);
|
|
s.push('=');
|
|
s.push_str(value);
|
|
}
|
|
|
|
if s.is_empty() {
|
|
return None;
|
|
}
|
|
|
|
HeaderValue::from_maybe_shared(Bytes::from(s)).ok()
|
|
}
|
|
}
|
|
|
|
/// Custom FaviconEmitter for the html5gum parser.
|
|
/// The FaviconEmitter is using an optimized version of the DefaultEmitter.
|
|
/// This prevents emitting tags like comments, doctype and also strings between the tags.
|
|
/// But it will also only emit the tags we need and only if they have the correct attributes
|
|
/// Therefore parsing the HTML content is faster.
|
|
use std::collections::BTreeMap;
|
|
|
|
#[derive(Default)]
|
|
pub struct Tag {
|
|
/// The tag's name, such as `"link"` or `"base"`.
|
|
pub name: HtmlString,
|
|
|
|
/// A mapping for any HTML attributes this start tag may have.
|
|
///
|
|
/// Duplicate attributes are ignored after the first one as per WHATWG spec.
|
|
pub attributes: BTreeMap<HtmlString, HtmlString>,
|
|
}
|
|
|
|
struct FaviconToken {
|
|
tag: Tag,
|
|
closing: bool,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
struct FaviconEmitter {
|
|
current_token: Option<FaviconToken>,
|
|
last_start_tag: HtmlString,
|
|
current_attribute: Option<(HtmlString, HtmlString)>,
|
|
emit_token: bool,
|
|
}
|
|
|
|
impl FaviconEmitter {
|
|
fn is_icon_rel(rel: &[u8]) -> bool {
|
|
let mut has_icon = false;
|
|
|
|
for keyword in rel.split(u8::is_ascii_whitespace) {
|
|
if keyword.eq_ignore_ascii_case(b"mask-icon") {
|
|
return false;
|
|
}
|
|
if keyword.eq_ignore_ascii_case(b"icon")
|
|
|| keyword.eq_ignore_ascii_case(b"favicon")
|
|
|| keyword.eq_ignore_ascii_case(b"fluid-icon")
|
|
|| keyword.eq_ignore_ascii_case(b"apple-touch-icon")
|
|
|| keyword.eq_ignore_ascii_case(b"apple-touch-icon-precomposed")
|
|
{
|
|
has_icon = true;
|
|
}
|
|
}
|
|
|
|
has_icon
|
|
}
|
|
|
|
fn flush_current_attribute(&mut self, tag_complete: bool) {
|
|
let Some(token) = &mut self.current_token else {
|
|
return;
|
|
};
|
|
|
|
if let Some((name, value)) = self.current_attribute.take() {
|
|
token.tag.attributes.entry(name).or_insert(value);
|
|
}
|
|
|
|
if tag_complete {
|
|
let attrs = &token.tag.attributes;
|
|
let href: &[u8] = b"href";
|
|
let rel: &[u8] = b"rel";
|
|
|
|
self.emit_token = match (token.closing, token.tag.name.as_slice()) {
|
|
(true, b"head") | (false, b"body" | b"frameset") => true,
|
|
(false, b"base") => attrs.contains_key(href),
|
|
(false, b"link") => {
|
|
attrs.contains_key(href) && attrs.get(rel).is_some_and(|value| Self::is_icon_rel(value))
|
|
}
|
|
_ => false,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Emitter for FaviconEmitter {
|
|
type Token = FaviconToken;
|
|
|
|
fn set_last_start_tag(&mut self, last_start_tag: Option<&[u8]>) {
|
|
self.last_start_tag.clear();
|
|
self.last_start_tag.extend(last_start_tag.unwrap_or_default());
|
|
}
|
|
|
|
fn pop_token(&mut self) -> Option<Self::Token> {
|
|
if self.emit_token {
|
|
self.emit_token = false;
|
|
return self.current_token.take();
|
|
}
|
|
None
|
|
}
|
|
|
|
fn init_start_tag(&mut self) {
|
|
self.current_token = Some(FaviconToken {
|
|
tag: Tag::default(),
|
|
closing: false,
|
|
});
|
|
}
|
|
|
|
fn init_end_tag(&mut self) {
|
|
self.current_token = Some(FaviconToken {
|
|
tag: Tag::default(),
|
|
closing: true,
|
|
});
|
|
}
|
|
|
|
fn emit_current_tag(&mut self) -> Option<html5gum::State> {
|
|
self.flush_current_attribute(true);
|
|
self.last_start_tag.clear();
|
|
match &self.current_token {
|
|
Some(token) if !token.closing => {
|
|
self.last_start_tag.extend(&*token.tag.name);
|
|
}
|
|
_ => {}
|
|
}
|
|
html5gum::naive_next_state(&self.last_start_tag)
|
|
}
|
|
|
|
fn push_tag_name(&mut self, s: &[u8]) {
|
|
if let Some(ref mut token) = self.current_token {
|
|
token.tag.name.extend(s);
|
|
}
|
|
}
|
|
|
|
fn init_attribute(&mut self) {
|
|
self.flush_current_attribute(false);
|
|
self.current_attribute = self
|
|
.current_token
|
|
.as_ref()
|
|
.filter(|token| !token.closing && matches!(token.tag.name.as_slice(), b"link" | b"base"))
|
|
.map(|_| Default::default());
|
|
}
|
|
|
|
fn push_attribute_name(&mut self, s: &[u8]) {
|
|
if let Some(attr) = &mut self.current_attribute {
|
|
attr.0.extend(s);
|
|
}
|
|
}
|
|
|
|
fn push_attribute_value(&mut self, s: &[u8]) {
|
|
if let Some(attr) = &mut self.current_attribute {
|
|
attr.1.extend(s);
|
|
}
|
|
}
|
|
|
|
fn current_is_appropriate_end_tag_token(&mut self) -> bool {
|
|
self.current_token
|
|
.as_ref()
|
|
.is_some_and(|t| t.closing && (!self.last_start_tag.is_empty() && self.last_start_tag == t.tag.name))
|
|
}
|
|
|
|
fn should_emit_errors(&mut self) -> bool {
|
|
false
|
|
}
|
|
|
|
// We do not want and need these parts of the HTML document
|
|
// These will be skipped and ignored during the tokenization and iteration.
|
|
fn emit_current_comment(&mut self) {}
|
|
fn emit_current_doctype(&mut self) {}
|
|
fn emit_current_processing_instruction(&mut self) {}
|
|
fn emit_eof(&mut self) {}
|
|
fn emit_error(&mut self, _: html5gum::Error) {}
|
|
fn emit_string(&mut self, _: &[u8]) {}
|
|
fn init_comment(&mut self) {}
|
|
fn init_doctype(&mut self) {}
|
|
fn init_processing_instruction(&mut self) {}
|
|
fn push_comment(&mut self, _: &[u8]) {}
|
|
fn push_doctype_name(&mut self, _: &[u8]) {}
|
|
fn push_doctype_public_identifier(&mut self, _: &[u8]) {}
|
|
fn push_doctype_system_identifier(&mut self, _: &[u8]) {}
|
|
fn push_processing_instruction_data(&mut self, _: &[u8]) {}
|
|
fn push_processing_instruction_target(&mut self, _: &[u8]) {}
|
|
fn set_doctype_public_identifier(&mut self, _: &[u8]) {}
|
|
fn set_doctype_system_identifier(&mut self, _: &[u8]) {}
|
|
fn set_force_quirks(&mut self) {}
|
|
fn set_self_closing(&mut self) {}
|
|
}
|
|
|