From 3a3390963c91566e1b60ee4bb3d0f7fea5a950a6 Mon Sep 17 00:00:00 2001 From: BlackDex Date: Mon, 29 Mar 2021 10:27:58 +0200 Subject: [PATCH] Icon and SMTP Debug fixes. - We need to add some feature to enable smtp debugging again. See: https://github.com/lettre/lettre/pull/584 - Upstream added the fallback icon again, probably because of caching ;). See: https://github.com/bitwarden/server/pull/1149 - Enabled gzip and brotli compression support with reqwest. Some sites seem to force this, or assume that because of the User-Agent string it is supported. This caused some failed icons. Fixes #1540 --- Cargo.lock | 148 ++++++++++++++++++++++------ Cargo.toml | 5 +- src/api/icons.rs | 25 ++--- src/static/images/fallback-icon.png | Bin 0 -> 331 bytes 4 files changed, 134 insertions(+), 44 deletions(-) create mode 100644 src/static/images/fallback-icon.png diff --git a/Cargo.lock b/Cargo.lock index feab7340..e60c8d0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,35 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192ec435945d87bc2f70992b4d818154b5feede43c09fb7592146374eac90a6" + +[[package]] +name = "alloc-stdlib" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "async-compression" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b72c1f1154e234325b50864a349b9c8e56939e266a4c307c0f159812df2f9537" +dependencies = [ + "brotli", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "atty" version = "0.2.14" @@ -159,6 +188,7 @@ dependencies = [ "serde_json", "syslog", "time 0.2.26", + "tracing", "u2f", "uuid", "yubico", @@ -194,6 +224,27 @@ dependencies = [ "byte-tools 0.3.1", ] +[[package]] +name = "brotli" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29919120f08613aadcd4383764e00526fc9f18b6c0895814faeed0dd78613e" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1052e1c3b8d4d80eb84a8b94f0a1498797b5fb96314c001156a1c761940ef4ec" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "buf_redux" version = "0.8.4" @@ -298,9 +349,9 @@ dependencies = [ [[package]] name = "const_fn" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" +checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" [[package]] name = "constant_time_eq" @@ -341,6 +392,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crypto-mac" version = "0.3.0" @@ -401,7 +461,7 @@ dependencies = [ "bitflags", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -429,7 +489,7 @@ checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -525,6 +585,18 @@ dependencies = [ "syslog", ] +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -651,7 +723,7 @@ dependencies = [ "proc-macro-hack", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -770,9 +842,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "3.5.3" +version = "3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb0867bbc5a3da37a753e78021d5fcf8a4db00e18dd2dd90fd36e24190e162d" +checksum = "580b6f551b29a3a02436318aed09ba1c58eea177dc49e39beac627ad356730a5" dependencies = [ "log 0.4.14", "pest", @@ -841,7 +913,7 @@ dependencies = [ "markup5ever", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -899,9 +971,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" +checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1" dependencies = [ "bytes 1.0.1", "futures-channel", @@ -940,7 +1012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes 1.0.1", - "hyper 0.14.4", + "hyper 0.14.5", "native-tls", "tokio", "tokio-native-tls", @@ -1225,7 +1297,7 @@ dependencies = [ "migrations_internals", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -1443,7 +1515,7 @@ checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -1718,7 +1790,7 @@ dependencies = [ "pest_meta", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -1793,7 +1865,7 @@ checksum = "a490329918e856ed1b083f244e3bfe2d8c4f336407e4ea9e1a9f479ff09049e5" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -2087,6 +2159,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" dependencies = [ + "async-compression", "base64 0.13.0", "bytes 1.0.1", "encoding_rs", @@ -2094,7 +2167,7 @@ dependencies = [ "futures-util", "http", "http-body", - "hyper 0.14.4", + "hyper 0.14.5", "hyper-tls", "ipnet", "js-sys", @@ -2109,6 +2182,7 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-util", "url 2.2.1", "wasm-bindgen", "wasm-bindgen-futures", @@ -2306,9 +2380,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d" +checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" dependencies = [ "bitflags", "core-foundation", @@ -2319,9 +2393,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee48cdde5ed250b0d3252818f646e174ab414036edb884dde62d80a3ac6082d" +checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" dependencies = [ "core-foundation-sys", "libc", @@ -2359,7 +2433,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -2482,11 +2556,10 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.3.19" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" dependencies = [ - "cfg-if 1.0.0", "libc", "winapi 0.3.9", ] @@ -2542,7 +2615,7 @@ dependencies = [ "quote 1.0.9", "serde", "serde_derive", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -2558,7 +2631,7 @@ dependencies = [ "serde_derive", "serde_json", "sha1", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -2611,9 +2684,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" +checksum = "f3a1d708c221c5a612956ef9f75b37e454e88d1f7b899fbd3a18d4252012d663" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", @@ -2718,7 +2791,7 @@ dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", "standback", - "syn 1.0.64", + "syn 1.0.65", ] [[package]] @@ -2797,10 +2870,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f" dependencies = [ "cfg-if 1.0.0", + "log 0.4.14", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" +dependencies = [ + "proc-macro2 1.0.24", + "quote 1.0.9", + "syn 1.0.65", +] + [[package]] name = "tracing-core" version = "0.1.17" @@ -3032,7 +3118,7 @@ dependencies = [ "log 0.4.14", "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", "wasm-bindgen-shared", ] @@ -3066,7 +3152,7 @@ checksum = "96eb45c1b2ee33545a813a92dbb53856418bf7eb54ab34f7f7ff1448a5b3735d" dependencies = [ "proc-macro2 1.0.24", "quote 1.0.9", - "syn 1.0.64", + "syn 1.0.65", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index bb8ad8cb..24c24eba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ rocket = { version = "0.5.0-dev", features = ["tls"], default-features = false } rocket_contrib = "0.5.0-dev" # HTTP client -reqwest = { version = "0.11.2", features = ["blocking", "json"] } +reqwest = { version = "0.11.2", features = ["blocking", "json", "gzip", "brotli"] } # multipart/form-data support multipart = { version = "0.17.1", features = ["server"], default-features = false } @@ -99,11 +99,12 @@ num-traits = "0.2.14" num-derive = "0.3.3" # Email libraries +tracing = { version = "0.1", features = ["log"] } # Needed to have lettre trace logging used when SMTP_DEBUG is enabled. lettre = { version = "0.10.0-beta.3", features = ["smtp-transport", "builder", "serde", "native-tls", "hostname", "tracing"], default-features = false } newline-converter = "0.2.0" # Template library -handlebars = { version = "3.5.3", features = ["dir_source"] } +handlebars = { version = "3.5.4", features = ["dir_source"] } # For favicon extraction from main website html5ever = "0.25.1" diff --git a/src/api/icons.rs b/src/api/icons.rs index 747569c6..6da3af0b 100644 --- a/src/api/icons.rs +++ b/src/api/icons.rs @@ -18,8 +18,6 @@ pub fn routes() -> Vec { routes![icon] } -const ALLOWED_CHARS: &str = "_-."; - static CLIENT: Lazy = Lazy::new(|| { // Generate the default headers let mut default_headers = header::HeaderMap::new(); @@ -45,13 +43,18 @@ static ICON_SIZE_REGEX: Lazy = Lazy::new(|| Regex::new(r"(?x)(\d+)\D*(\d+ static ICON_BLACKLIST_REGEX: Lazy>> = Lazy::new(|| RwLock::new(HashMap::new())); #[get("//icon.png")] -fn icon(domain: String) -> Option>>> { +fn icon(domain: String) -> Cached>> { + const FALLBACK_ICON: &[u8] = include_bytes!("../static/images/fallback-icon.png"); + if !is_valid_domain(&domain) { warn!("Invalid domain: {}", domain); - return None; + return Cached::ttl(Content(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()), CONFIG.icon_cache_negttl()); } - get_icon(&domain).map(|icon| Cached::ttl(Content(ContentType::new("image", "x-icon"), icon), CONFIG.icon_cache_ttl())) + match get_icon(&domain) { + Some(i) => Cached::ttl(Content(ContentType::new("image", "x-icon"), i), CONFIG.icon_cache_ttl()), + _ => Cached::ttl(Content(ContentType::new("image", "png"), FALLBACK_ICON.to_vec()), CONFIG.icon_cache_negttl()), + } } /// Returns if the domain provided is valid or not. @@ -59,6 +62,8 @@ fn icon(domain: String) -> Option>>> { /// This does some manual checks and makes use of Url to do some basic checking. /// domains can't be larger then 63 characters (not counting multiple subdomains) according to the RFC's, but we limit the total size to 255. fn is_valid_domain(domain: &str) -> bool { + const ALLOWED_CHARS: &str = "_-."; + // If parsing the domain fails using Url, it will not work with reqwest. if let Err(parse_error) = Url::parse(format!("https://{}", domain).as_str()) { debug!("Domain parse error: '{}' - {:?}", domain, parse_error); @@ -486,10 +491,10 @@ fn get_icon_url(domain: &str) -> Result { iconlist.sort_by_key(|x| x.priority); // There always is an icon in the list, so no need to check if it exists, and just return the first one - Ok(IconUrlResult{ + Ok(IconUrlResult { iconlist, cookies: cookie_str, - referer + referer, }) } @@ -510,9 +515,7 @@ fn get_page_with_cookies(url: &str, cookie_str: &str, referer: &str) -> Result Result, Error> { info!("Downloaded icon from {}", icon.href); res.copy_to(&mut buffer)?; break; - }, + } _ => warn!("Download failed for {}", icon.href), }; } diff --git a/src/static/images/fallback-icon.png b/src/static/images/fallback-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..e30b532e65a0304622ad2938bee02fcbc2f89987 GIT binary patch literal 331 zcmV-R0kr;!P)k@@&$eP0S64l=K_sN$BuK6XOfRIOm@a>|#`zV3>$SuCU-E zQMLhNNdZQ}mLg4h7}w;&4I~^WgDEch53KQxu_K8J#uJg;;S%H2>0%`0#3hDNkUZ3| z)RKrpF(U^@SmwIYFTOIS06B_$xJX4jq-Bc*iBq~XFz2#3&YDm1Y*5AUj*7Bt@7bor zt()Ab4=v2u=8Re9(v!>TLqviPy@%-#G5Z?8B+o8VCr^bA$ish5vPevi7gqEa3aqk6 djV-3-_yJsJ=ebCXRj>d6002ovPDHLkV1l