diff --git a/Cargo.lock b/Cargo.lock index 18a5b59..45f57ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,6 +629,19 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-compression" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bf394cfbbe876f0ac67b13b6ca819f9c9f2fb9ec67223cceb1555fbab1c31a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite 0.2.8", + "tokio 1.16.1", +] + [[package]] name = "async-stream" version = "0.3.2" @@ -1008,6 +1021,33 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 0.4.8", + "matches", + "phf", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -1053,6 +1093,27 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "either" version = "1.6.1" @@ -1148,6 +1209,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.21" @@ -1267,6 +1338,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -1381,6 +1461,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "html5ever" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.6" @@ -1679,6 +1773,26 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -1826,6 +1940,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "7.1.0" @@ -1997,6 +2123,69 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared 0.8.0", + "proc-macro-hack", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "0.4.29" @@ -2067,6 +2256,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2141,6 +2336,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", + "rand_pcg", ] [[package]] @@ -2211,6 +2407,15 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -2252,6 +2457,7 @@ version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525" dependencies = [ + "async-compression", "base64", "bytes 1.1.0", "encoding_rs", @@ -2276,6 +2482,7 @@ dependencies = [ "serde_urlencoded", "tokio 1.16.1", "tokio-rustls 0.23.2", + "tokio-util 0.6.9", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -2434,6 +2641,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scraper" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec", + "tendril", +] + [[package]] name = "sct" version = "0.6.1" @@ -2540,6 +2763,26 @@ dependencies = [ "syn", ] +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", + "thin-slice", +] + [[package]] name = "semver" version = "0.9.0" @@ -2605,6 +2848,16 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "sha-1" version = "0.9.8" @@ -2687,6 +2940,12 @@ dependencies = [ "time 0.3.7", ] +[[package]] +name = "siphasher" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a86232ab60fa71287d7f2ddae4a7073f6b7aac33631c3015abb556f08c6d0a3e" + [[package]] name = "slab" version = "0.4.5" @@ -2904,6 +3163,32 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" +[[package]] +name = "string_cache" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" +dependencies = [ + "phf_generator", + "phf_shared 0.8.0", + "proc-macro2", + "quote", +] + [[package]] name = "stringprep" version = "0.1.2" @@ -2931,6 +3216,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tendril" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "termcolor" version = "1.1.2" @@ -2946,6 +3242,12 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + [[package]] name = "thiserror" version = "1.0.30" @@ -3398,6 +3700,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "unicode-xid" version = "0.2.2" @@ -3428,6 +3736,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "uuid" version = "0.8.2" @@ -3460,6 +3774,7 @@ dependencies = [ "rand 0.8.4", "reqwest", "rust-embed", + "scraper", "sea-orm", "serde", "serde_json", @@ -3470,6 +3785,7 @@ dependencies = [ "tracing-subscriber", "tracing-test", "tracing-unwrap", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cedc3af..c8e3501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,10 +29,12 @@ rand = "0.8.4" tokio = { version = "1.16.1", features=["full"] } base64 = "0.13.0" sqlx = { version = "^0.5", features=["sqlite", "migrate"] } -reqwest = { version = "0.11.9", features = ["rustls-tls"], default-features=false } +reqwest = { version = "0.11.9", features = ["rustls-tls", "gzip"], default-features=false } clap = { version = "^3.0", features=["cargo", "env"] } actix-multipart = "0.4.0-beta.13" futures = "0.3.21" +scraper = "0.12.0" +url = "2.2.2" [target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.jemallocator] diff --git a/src/api/mod.rs b/src/api/mod.rs index d9fd565..09bacc7 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,5 +1,5 @@ use actix_web::web::Data; -use actix_web::Scope; +use actix_web::{get, Scope}; use actix_web_httpauth::middleware::HttpAuthentication; use serde::{Deserialize, Serialize}; @@ -153,6 +153,75 @@ async fn bg_upload( Ok(HttpResponse::Ok().body("")) } +#[derive(Deserialize)] +struct IconRequest { + url: String, +} + +#[get("/icon_proxy")] +async fn icon_proxy(q: web::Query) -> crate::error::Result { + let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?; + let client = reqwest::Client::builder() + .danger_accept_invalid_certs(true) + .referer(false) + .build() + .unwrap(); + let resp = client + .request(reqwest::Method::GET, q.url.clone()) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await?; + + let html = resp.text().await?; + + let document = scraper::Html::parse_document(&html); + + let icon_selector = + scraper::Selector::parse(r#"link[rel="icon"]"#).map_err(|_| "Error getting selector")?; + let shortcut_selector = scraper::Selector::parse(r#"link[rel="shortcut icon"]"#) + .map_err(|_| "Error getting selector")?; + + let icon_url = if let Some(src) = document + .select(&icon_selector) + .chain(document.select(&shortcut_selector)) + .next() + .and_then(|link| link.value().attr("href")) + { + url.join(src).map_err(|_| "could not map base url")? + } else { + let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?; + let url_string = &format!( + "{}://{}/favicon.ico", + url.scheme(), + url.host_str().unwrap_or_default() + ); + tracing::info!("Url string: {}", url_string); + url::Url::parse(url_string).map_err(|_| "Couldnt parse url")? + }; + + let img_bytes = client + .request(reqwest::Method::GET, icon_url.clone()) + .timeout(std::time::Duration::from_secs(5)) + .send() + .await? + .bytes() + .await?; + + let fpath = Path::new( + icon_url + .path_segments() + .and_then(|i| i.rev().next()) + .unwrap_or("test.png"), + ); + let ext = fpath + .extension() + .map(|s| s.to_str().unwrap()) + .unwrap_or(".png"); + Ok(HttpResponse::Ok() + .content_type(mime_guess::from_ext(ext).first_or_octet_stream().as_ref()) + .body(img_bytes)) +} + pub fn routes() -> Scope { let auth_handler = HttpAuthentication::bearer(auth::validator); let protected_routes = Scope::new("") @@ -168,5 +237,6 @@ pub fn routes() -> Scope { Scope::new("api") .service(api::authorization::authorize) .service(api::authorization::initial_setup) + .service(icon_proxy) .service(protected_routes) } diff --git a/src/error.rs b/src/error.rs index 16c461b..67ba45e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -49,6 +49,15 @@ impl From for Error { } } +impl From for Error { + fn from(s: String) -> Self { + Error { + code: ErrorCode::Internal, + message: s, + } + } +} + impl From<&str> for Error { fn from(s: &str) -> Self { Error { @@ -58,6 +67,15 @@ impl From<&str> for Error { } } +impl From for Error { + fn from(s: reqwest::Error) -> Self { + Error { + code: ErrorCode::Internal, + message: s.to_string(), + } + } +} + impl actix_web::error::ResponseError for Error { fn status_code(&self) -> actix_web::http::StatusCode { match self.code { diff --git a/src/main.rs b/src/main.rs index e1085cb..74cb950 100644 --- a/src/main.rs +++ b/src/main.rs @@ -86,10 +86,7 @@ async fn main() { .await .unwrap(); - let client = reqwest::Client::builder() - .danger_accept_invalid_certs(true) - .build() - .unwrap(); + let client = reqwest::Client::builder().build().unwrap(); for app in apps { match client @@ -145,7 +142,7 @@ async fn main() { } } } - tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; + tokio::time::sleep(tokio::time::Duration::from_secs(300)).await; } }); diff --git a/src/ui/components/ApplicationTile.vue b/src/ui/components/ApplicationTile.vue index 8fb3dc5..33a216c 100644 --- a/src/ui/components/ApplicationTile.vue +++ b/src/ui/components/ApplicationTile.vue @@ -14,12 +14,7 @@ export default { props: ["appData"], computed: { favicon() { - try { - const url = new URL(this.appData.url); - return `${url.protocol}//${url.hostname}/favicon.ico`; - } catch { - return ""; - } + return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.appData.url)}`; } }, methods: { diff --git a/src/ui/components/BookmarkTile.vue b/src/ui/components/BookmarkTile.vue index 2913967..70ce3fd 100644 --- a/src/ui/components/BookmarkTile.vue +++ b/src/ui/components/BookmarkTile.vue @@ -16,12 +16,7 @@ export default { }, computed: { favicon() { - try { - const url = new URL(this.bookmark.url); - return `${url.protocol}//${url.hostname}/favicon.ico`; - } catch { - return ""; - } + return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.bookmark.url)}`; } }, methods: {