vade
/
app
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FavIcon Proxy 

Favicons are now proxied through the Vade server which reaches out and
finds the icon to display it.

Icons are collected via either rel="icon" or rel="shortcut icon" or
/favicon.ico.

This should allow favicons to be used more reliably and bypasses CORS
mechanisms.

Healthcheck was also increased to 5 minutes from 1 minute.
This commit is contained in:
Joe Bellus 2022-02-15 22:58:19 -05:00
parent 6532721410
commit 26e4b73519
7 changed files with 412 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

316
Cargo.lock generated
View File

@ -629,6 +629,19 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "async-compression"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2bf394cfbbe876f0ac67b13b6ca819f9c9f2fb9ec67223cceb1555fbab1c31a"
dependencies = [
"flate2",
"futures-core",
"memchr",
"pin-project-lite 0.2.8",
"tokio 1.16.1",
]
[[package]]
name = "async-stream"
version = "0.3.2"
@ -1008,6 +1021,33 @@ dependencies = [
"generic-array",
]
[[package]]
name = "cssparser"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa 0.4.8",
"matches",
"phf",
"proc-macro2",
"quote",
"smallvec",
"syn",
]
[[package]]
name = "cssparser-macros"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "derive_more"
version = "0.99.17"
@ -1053,6 +1093,27 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dtoa"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
name = "dtoa-short"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"
dependencies = [
"dtoa",
]
[[package]]
name = "ego-tree"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
[[package]]
name = "either"
version = "1.6.1"
@ -1148,6 +1209,16 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]]
name = "futf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
dependencies = [
"mac",
"new_debug_unreachable",
]
[[package]]
name = "futures"
version = "0.3.21"
@ -1267,6 +1338,15 @@ dependencies = [
"version_check",
]
[[package]]
name = "getopts"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
dependencies = [
"unicode-width",
]
[[package]]
name = "getrandom"
version = "0.1.16"
@ -1381,6 +1461,20 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "html5ever"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b"
dependencies = [
"log",
"mac",
"markup5ever",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "http"
version = "0.2.6"
@ -1679,6 +1773,26 @@ dependencies = [
"linked-hash-map",
]
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "markup5ever"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
dependencies = [
"log",
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
"tendril",
]
[[package]]
name = "match_cfg"
version = "0.1.0"
@ -1826,6 +1940,18 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
[[package]]
name = "nodrop"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "nom"
version = "7.1.0"
@ -1997,6 +2123,69 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "phf"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
dependencies = [
"phf_macros",
"phf_shared 0.8.0",
"proc-macro-hack",
]
[[package]]
name = "phf_codegen"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
dependencies = [
"phf_generator",
"phf_shared 0.8.0",
]
[[package]]
name = "phf_generator"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
dependencies = [
"phf_shared 0.8.0",
"rand 0.7.3",
]
[[package]]
name = "phf_macros"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
dependencies = [
"phf_generator",
"phf_shared 0.8.0",
"proc-macro-hack",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "phf_shared"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project"
version = "0.4.29"
@ -2067,6 +2256,12 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@ -2141,6 +2336,7 @@ dependencies = [
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc 0.2.0",
"rand_pcg",
]
[[package]]
@ -2211,6 +2407,15 @@ dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rand_pcg"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "redox_syscall"
version = "0.2.10"
@ -2252,6 +2457,7 @@ version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
dependencies = [
"async-compression",
"base64",
"bytes 1.1.0",
"encoding_rs",
@ -2276,6 +2482,7 @@ dependencies = [
"serde_urlencoded",
"tokio 1.16.1",
"tokio-rustls 0.23.2",
"tokio-util 0.6.9",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
@ -2434,6 +2641,22 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "scraper"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12"
dependencies = [
"cssparser",
"ego-tree",
"getopts",
"html5ever",
"matches",
"selectors",
"smallvec",
"tendril",
]
[[package]]
name = "sct"
version = "0.6.1"
@ -2540,6 +2763,26 @@ dependencies = [
"syn",
]
[[package]]
name = "selectors"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
dependencies = [
"bitflags",
"cssparser",
"derive_more",
"fxhash",
"log",
"matches",
"phf",
"phf_codegen",
"precomputed-hash",
"servo_arc",
"smallvec",
"thin-slice",
]
[[package]]
name = "semver"
version = "0.9.0"
@ -2605,6 +2848,16 @@ dependencies = [
"serde",
]
[[package]]
name = "servo_arc"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
dependencies = [
"nodrop",
"stable_deref_trait",
]
[[package]]
name = "sha-1"
version = "0.9.8"
@ -2687,6 +2940,12 @@ dependencies = [
"time 0.3.7",
]
[[package]]
name = "siphasher"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a86232ab60fa71287d7f2ddae4a7073f6b7aac33631c3015abb556f08c6d0a3e"
[[package]]
name = "slab"
version = "0.4.5"
@ -2904,6 +3163,32 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]]
name = "string_cache"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33994d0838dc2d152d17a62adf608a869b5e846b65b389af7f3dbc1de45c5b26"
dependencies = [
"lazy_static",
"new_debug_unreachable",
"parking_lot",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97"
dependencies = [
"phf_generator",
"phf_shared 0.8.0",
"proc-macro2",
"quote",
]
[[package]]
name = "stringprep"
version = "0.1.2"
@ -2931,6 +3216,17 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tendril"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9ef557cb397a4f0a5a3a628f06515f78563f2209e64d47055d9dc6052bf5e33"
dependencies = [
"futf",
"mac",
"utf-8",
]
[[package]]
name = "termcolor"
version = "1.1.2"
@ -2946,6 +3242,12 @@ version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
[[package]]
name = "thin-slice"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]]
name = "thiserror"
version = "1.0.30"
@ -3398,6 +3700,12 @@ version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
version = "0.2.2"
@ -3428,6 +3736,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf-8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "uuid"
version = "0.8.2"
@ -3460,6 +3774,7 @@ dependencies = [
"rand 0.8.4",
"reqwest",
"rust-embed",
"scraper",
"sea-orm",
"serde",
"serde_json",
@ -3470,6 +3785,7 @@ dependencies = [
"tracing-subscriber",
"tracing-test",
"tracing-unwrap",
"url",
]
[[package]]

4
Cargo.toml
View File

@ -29,10 +29,12 @@ rand = "0.8.4"
tokio = { version = "1.16.1", features=["full"] }
base64 = "0.13.0"
sqlx = { version = "^0.5", features=["sqlite", "migrate"] }
reqwest = { version = "0.11.9", features = ["rustls-tls"], default-features=false }
reqwest = { version = "0.11.9", features = ["rustls-tls", "gzip"], default-features=false }
clap = { version = "^3.0", features=["cargo", "env"] }
actix-multipart = "0.4.0-beta.13"
futures = "0.3.21"
scraper = "0.12.0"
url = "2.2.2"
[target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.jemallocator]

72
src/api/mod.rs
View File

@ -1,5 +1,5 @@
use actix_web::web::Data;
use actix_web::Scope;
use actix_web::{get, Scope};
use actix_web_httpauth::middleware::HttpAuthentication;
use serde::{Deserialize, Serialize};
@ -153,6 +153,75 @@ async fn bg_upload(
Ok(HttpResponse::Ok().body(""))
}
#[derive(Deserialize)]
struct IconRequest {
url: String,
}
#[get("/icon_proxy")]
async fn icon_proxy(q: web::Query<IconRequest>) -> crate::error::Result<HttpResponse> {
let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?;
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.referer(false)
.build()
.unwrap();
let resp = client
.request(reqwest::Method::GET, q.url.clone())
.timeout(std::time::Duration::from_secs(10))
.send()
.await?;
let html = resp.text().await?;
let document = scraper::Html::parse_document(&html);
let icon_selector =
scraper::Selector::parse(r#"link[rel="icon"]"#).map_err(|_| "Error getting selector")?;
let shortcut_selector = scraper::Selector::parse(r#"link[rel="shortcut icon"]"#)
.map_err(|_| "Error getting selector")?;
let icon_url = if let Some(src) = document
.select(&icon_selector)
.chain(document.select(&shortcut_selector))
.next()
.and_then(|link| link.value().attr("href"))
{
url.join(src).map_err(|_| "could not map base url")?
} else {
let url = url::Url::parse(&q.url).map_err(|_| "Could not parse url")?;
let url_string = &format!(
"{}://{}/favicon.ico",
url.scheme(),
url.host_str().unwrap_or_default()
);
tracing::info!("Url string: {}", url_string);
url::Url::parse(url_string).map_err(|_| "Couldnt parse url")?
};
let img_bytes = client
.request(reqwest::Method::GET, icon_url.clone())
.timeout(std::time::Duration::from_secs(5))
.send()
.await?
.bytes()
.await?;
let fpath = Path::new(
icon_url
.path_segments()
.and_then(|i| i.rev().next())
.unwrap_or("test.png"),
);
let ext = fpath
.extension()
.map(|s| s.to_str().unwrap())
.unwrap_or(".png");
Ok(HttpResponse::Ok()
.content_type(mime_guess::from_ext(ext).first_or_octet_stream().as_ref())
.body(img_bytes))
}
pub fn routes() -> Scope {
let auth_handler = HttpAuthentication::bearer(auth::validator);
let protected_routes = Scope::new("")
@ -168,5 +237,6 @@ pub fn routes() -> Scope {
Scope::new("api")
.service(api::authorization::authorize)
.service(api::authorization::initial_setup)
.service(icon_proxy)
.service(protected_routes)
}

18
src/error.rs
View File

@ -49,6 +49,15 @@ impl From<std::io::Error> for Error {
}
}
impl From<String> for Error {
fn from(s: String) -> Self {
Error {
code: ErrorCode::Internal,
message: s,
}
}
}
impl From<&str> for Error {
fn from(s: &str) -> Self {
Error {
@ -58,6 +67,15 @@ impl From<&str> for Error {
}
}
impl From<reqwest::Error> for Error {
fn from(s: reqwest::Error) -> Self {
Error {
code: ErrorCode::Internal,
message: s.to_string(),
}
}
}
impl actix_web::error::ResponseError for Error {
fn status_code(&self) -> actix_web::http::StatusCode {
match self.code {

7
src/main.rs
View File

@ -86,10 +86,7 @@ async fn main() {
.await
.unwrap();
let client = reqwest::Client::builder()
.danger_accept_invalid_certs(true)
.build()
.unwrap();
let client = reqwest::Client::builder().build().unwrap();
for app in apps {
match client
@ -145,7 +142,7 @@ async fn main() {
}
}
}
tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
tokio::time::sleep(tokio::time::Duration::from_secs(300)).await;
}
});

7
src/ui/components/ApplicationTile.vue
View File

@ -14,12 +14,7 @@ export default {
props: ["appData"],
computed: {
favicon() {
try {
const url = new URL(this.appData.url);
return `${url.protocol}//${url.hostname}/favicon.ico`;
} catch {
return "";
}
return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.appData.url)}`;
}
},
methods: {

7
src/ui/components/BookmarkTile.vue
View File

@ -16,12 +16,7 @@ export default {
},
computed: {
favicon() {
try {
const url = new URL(this.bookmark.url);
return `${url.protocol}//${url.hostname}/favicon.ico`;
} catch {
return "";
}
return `${this.$rootUrl()}/api/icon_proxy?url=${encodeURIComponent(this.bookmark.url)}`;
}
},
methods: {