Merge branch 'main' into favicon-proxy

Favicons are now proxied through the Vade server which reaches out and
finds the icon to display it.

Icons are collected via either rel="icon" or rel="shortcut icon" or
/favicon.ico.

This should allow favicons to be used more reliably and bypasses CORS
mechanisms.

Healthcheck was also increased to 5 minutes from 1 minute.

Cargo.lock

@@ -629,6 +629,19 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
+[[package]]
+name = "async-compression"
+version = "0.3.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2bf394cfbbe876f0ac67b13b6ca819f9c9f2fb9ec67223cceb1555fbab1c31a"
+dependencies = [
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite 0.2.8",
+ "tokio 1.16.1",
+]
+
 [[package]]
 name = "async-stream"
 version = "0.3.2"
@@ -1008,6 +1021,33 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "cssparser"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
+dependencies = [
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa 0.4.8",
+ "matches",
+ "phf",
+ "proc-macro2",
+ "quote",
+ "smallvec",
+ "syn",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -1053,6 +1093,27 @@ version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
 
+[[package]]
+name = "dtoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
+
+[[package]]
+name = "dtoa-short"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"
+dependencies = [
+ "dtoa",
+]
+
+[[package]]
+name = "ego-tree"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
+
 [[package]]
 name = "either"
 version = "1.6.1"
@@ -1148,6 +1209,16 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
 
+[[package]]
+name = "futf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
+dependencies = [
+ "mac",
+ "new_debug_unreachable",
+]
+
 [[package]]
 name = "futures"
 version = "0.3.21"
@@ -1267,6 +1338,15 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.1.16"
@@ -1381,6 +1461,20 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "html5ever"
+version = "0.25.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "http"
 version = "0.2.6"
@@ -1679,6 +1773,26 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+
+[[package]]
+name = "markup5ever"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
+dependencies = [
+ "log",
+ "phf",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
 [[package]]
 name = "match_cfg"
 version = "0.1.0"
@@ -1826,6 +1940,18 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
 [[package]]
 name = "nom"
 version = "7.1.0"
@@ -1997,6 +2123,69 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "phf"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
+dependencies = [
+ "phf_macros",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.8.0",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
+dependencies = [
+ "phf_shared 0.8.0",
+ "rand 0.7.3",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.29"
@@ -2067,6 +2256,12 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
 
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -2141,6 +2336,7 @@ dependencies = [
  "rand_chacha 0.2.2",
  "rand_core 0.5.1",
  "rand_hc 0.2.0",
+ "rand_pcg",
 ]
 
 [[package]]
@@ -2211,6 +2407,15 @@ dependencies = [
  "rand_core 0.6.3",
 ]
 
+[[package]]
+name = "rand_pcg"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.10"
@@ -2252,6 +2457,7 @@ version = "0.11.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
 dependencies = [
+ "async-compression",
  "base64",
  "bytes 1.1.0",
  "encoding_rs",
@@ -2276,6 +2482,7 @@ dependencies = [
  "serde_urlencoded",
  "tokio 1.16.1",
  "tokio-rustls 0.23.2",
+ "tokio-util 0.6.9",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -2434,6 +2641,22 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scraper"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12"
+dependencies = [
+ "cssparser",
+ "ego-tree",
+ "getopts",
+ "html5ever",
+ "matches",
+ "selectors",
+ "smallvec",
+ "tendril",
+]
+
 [[package]]
 name = "sct"
 version = "0.6.1"
@@ -2540,6 +2763,26 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "selectors"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
+dependencies = [
+ "bitflags",
+ "cssparser",
+ "derive_more",
+ "fxhash",
+ "log",
+ "matches",
+ "phf",
+ "phf_codegen",
+ "precomputed-hash",
+ "servo_arc",
+ "smallvec",
+ "thin-slice",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -2605,6 +2848,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "servo_arc"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
+dependencies = [
+ "nodrop",
+ "stable_deref_trait",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.9.8"
@@ -2687,6 +2940,12 @@ dependencies = [
  "time 0.3.7",
 ]
 
+[[package]]
+name = "siphasher"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a86232ab60fa71287d7f2ddae4a7073f6b7aac33631c3015abb556f08c6d0a3e"
+
 [[package]]
 name = "slab"
 version = "0.4.5"
@@ -2904,6 +3163,32 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
 
+[[package]]
+name = "string_cache"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26"
+dependencies = [
+ "lazy_static",
+ "new_debug_unreachable",
+ "parking_lot",
+ "phf_shared 0.10.0",
+ "precomputed-hash",
+ "serde",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97"
+dependencies = [
+ "phf_generator",
+ "phf_shared 0.8.0",
+ "proc-macro2",
+ "quote",
+]
+
 [[package]]
 name = "stringprep"
 version = "0.1.2"
@@ -2931,6 +3216,17 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "tendril"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33"
+dependencies = [
+ "futf",
+ "mac",
+ "utf-8",
+]
+
 [[package]]
 name = "termcolor"
 version = "1.1.2"
@@ -2946,6 +3242,12 @@ version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
 
+[[package]]
+name = "thin-slice"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
+
 [[package]]
 name = "thiserror"
 version = "1.0.30"
@@ -3398,6 +3700,12 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.2"
@@ -3428,6 +3736,12 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
 [[package]]
 name = "uuid"
 version = "0.8.2"
@@ -3460,6 +3774,7 @@ dependencies = [
  "rand 0.8.4",
  "reqwest",
  "rust-embed",
+ "scraper",
  "sea-orm",
  "serde",
  "serde_json",
@@ -3470,6 +3785,7 @@ dependencies = [
  "tracing-subscriber",
  "tracing-test",
  "tracing-unwrap",
+ "url",
 ]
 
 [[package]]

Cargo.toml

@@ -29,10 +29,12 @@ rand = "0.8.4"
 tokio = { version = "1.16.1", features=["full"] }
 base64 = "0.13.0"
 sqlx = { version = "^0.5", features=["sqlite", "migrate"] }
-reqwest = { version = "0.11.9", features = ["rustls-tls"], default-features=false }
+reqwest = { version = "0.11.9", features = ["rustls-tls", "gzip"], default-features=false }
 clap = { version = "^3.0", features=["cargo", "env"] }
 actix-multipart = "0.4.0-beta.13"
 futures = "0.3.21"
+scraper = "0.12.0"
+url = "2.2.2"
 
 
 [target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.jemallocator]

src/api/mod.rs

@@ -1,5 +1,5 @@
 use actix_web::web::Data;
-use actix_web::Scope;
+use actix_web::{get, Scope};
 use actix_web_httpauth::middleware::HttpAuthentication;
 use serde::{Deserialize, Serialize};
 
@@ -153,6 +153,75 @@ async fn bg_upload(
     Ok(HttpResponse::Ok().body(""))
 }
 
+#[derive(Deserialize)]
+struct IconRequest {
+    url: String,
+}
+
+#[get("/icon_proxy")]
+async fn icon_proxy(q: web::Query<IconRequest>) -> crate::error::Result<HttpResponse> {
+    let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?;
+    let client = reqwest::Client::builder()
+        .danger_accept_invalid_certs(true)
+        .referer(false)
+        .build()
+        .unwrap();
+    let resp = client
+        .request(reqwest::Method::GET, q.url.clone())
+        .timeout(std::time::Duration::from_secs(10))
+        .send()
+        .await?;
+
+    let html = resp.text().await?;
+
+    let document = scraper::Html::parse_document(&html);
+
+    let icon_selector =
+        scraper::Selector::parse(r#"link[rel="icon"]"#).map_err(|_| "Error getting selector")?;
+    let shortcut_selector = scraper::Selector::parse(r#"link[rel="shortcut icon"]"#)
+        .map_err(|_| "Error getting selector")?;
+
+    let icon_url = if let Some(src) = document
+        .select(&icon_selector)
+        .chain(document.select(&shortcut_selector))
+        .next()
+        .and_then(|link| link.value().attr("href"))
+    {
+        url.join(src).map_err(|_| "could not map base url")?
+    } else {
+        let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?;
+        let url_string = &format!(
+            "{}://{}/favicon.ico",
+            url.scheme(),
+            url.host_str().unwrap_or_default()
+        );
+        tracing::info!("Url string: {}", url_string);
+        url::Url::parse(url_string).map_err(|_| "Couldnt parse url")?
+    };
+
+    let img_bytes = client
+        .request(reqwest::Method::GET, icon_url.clone())
+        .timeout(std::time::Duration::from_secs(5))
+        .send()
+        .await?
+        .bytes()
+        .await?;
+
+    let fpath = Path::new(
+        icon_url
+            .path_segments()
+            .and_then(|i| i.rev().next())
+            .unwrap_or("test.png"),
+    );
+    let ext = fpath
+        .extension()
+        .map(|s| s.to_str().unwrap())
+        .unwrap_or(".png");
+    Ok(HttpResponse::Ok()
+        .content_type(mime_guess::from_ext(ext).first_or_octet_stream().as_ref())
+        .body(img_bytes))
+}
+
 pub fn routes() -> Scope {
     let auth_handler = HttpAuthentication::bearer(auth::validator);
     let protected_routes = Scope::new("")
@@ -168,5 +237,6 @@ pub fn routes() -> Scope {
     Scope::new("api")
         .service(api::authorization::authorize)
         .service(api::authorization::initial_setup)
+        .service(icon_proxy)
         .service(protected_routes)
 }

src/error.rs

@@ -49,6 +49,15 @@ impl From<std::io::Error> for Error {
     }
 }
 
+impl From<String> for Error {
+    fn from(s: String) -> Self {
+        Error {
+            code: ErrorCode::Internal,
+            message: s,
+        }
+    }
+}
+
 impl From<&str> for Error {
     fn from(s: &str) -> Self {
         Error {
@@ -58,6 +67,15 @@ impl From<&str> for Error {
     }
 }
 
+impl From<reqwest::Error> for Error {
+    fn from(s: reqwest::Error) -> Self {
+        Error {
+            code: ErrorCode::Internal,
+            message: s.to_string(),
+        }
+    }
+}
+
 impl actix_web::error::ResponseError for Error {
     fn status_code(&self) -> actix_web::http::StatusCode {
         match self.code {

src/main.rs

@@ -86,10 +86,7 @@ async fn main() {
                 .await
                 .unwrap();
 
-            let client = reqwest::Client::builder()
+            let client = reqwest::Client::builder().build().unwrap();
-                .danger_accept_invalid_certs(true)
-                .build()
-                .unwrap();
 
             for app in apps {
                 match client
@@ -145,7 +142,7 @@ async fn main() {
                     }
                 }
             }
-            tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
+            tokio::time::sleep(tokio::time::Duration::from_secs(300)).await;
         }
     });
 

src/ui/components/ApplicationTile.vue

@@ -14,12 +14,7 @@ export default {
   props: ["appData"],
   computed: {
     favicon() {
-      try {
+      return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.appData.url)}`;
-      const url = new URL(this.appData.url);
-      return `${url.protocol}//${url.hostname}/favicon.ico`;
-      } catch  {
-        return "";
-      }
     }
   },
   methods: {

src/ui/components/BookmarkTile.vue

@@ -16,12 +16,7 @@ export default {
   },
   computed: {
     favicon() {
-      try {
+      return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.bookmark.url)}`;
-        const url = new URL(this.bookmark.url);
-        return `${url.protocol}//${url.hostname}/favicon.ico`;
-      } catch  {
-        return "";
-      }
     }
   },
   methods: {