From 63c8096a88bf7d3103d0010e2d4b94e499e863f9 Mon Sep 17 00:00:00 2001 From: Anson Date: Sun, 24 Nov 2024 22:16:59 -0700 Subject: [PATCH] Update Ghost uploader to include extra post info --- ghost-upload/Cargo.lock | 402 +++++++++++++++++++++++++++++++++++++-- ghost-upload/Cargo.toml | 5 +- ghost-upload/src/main.rs | 134 ++++++++----- 3 files changed, 479 insertions(+), 62 deletions(-) diff --git a/ghost-upload/Cargo.lock b/ghost-upload/Cargo.lock index cf9f122..99a376f 100644 --- a/ghost-upload/Cargo.lock +++ b/ghost-upload/Cargo.lock @@ -17,6 +17,19 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom 0.2.15", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -158,6 +171,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets", ] @@ -199,6 +213,29 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "cssparser" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "deranged" version = "0.3.11" @@ -208,6 +245,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "0.99.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dirs" version = "1.0.5" @@ -230,6 +278,27 @@ dependencies = [ "syn", ] +[[package]] +name = "dtoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c6ba7d4eec39eaa9ab24d44a0e73a7949a1095a8b3f3abb11eddf27dbb56a53" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -263,9 +332,9 @@ checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "feed-rs" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6511d29ecf291f32688d546c33d8e62bbebd4fe60ef40b52b20e3f3a0ec6eb56" +checksum = "d57df0b823b3055c667ddde499736e0d8d7d486b269454eebd96a0fdd59ce051" dependencies = [ "chrono", "mediatype", @@ -273,7 +342,7 @@ dependencies = [ "regex", "serde", "serde_json", - "siphasher", + "siphasher 1.0.1", "url", "uuid", ] @@ -308,6 +377,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.31" @@ -397,6 +476,24 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.1.16" @@ -433,6 +530,7 @@ dependencies = [ "jsonwebtoken", "maud", "reqwest", + "scraper", "serde", "serde_json", "tokio", @@ -481,6 +579,20 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "html5ever" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e15626aaf9c351bc696217cbe29cb9b5e86c43f8a46b5e2f5c6c5cf7cb904ce" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "1.1.0" @@ -835,6 +947,26 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c88c6129bd24319e62a0359cb6b958fa7e8be6e19bb1663bc396b90883aca5" +dependencies = [ + "log", + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "maud" version = "0.26.0" @@ -862,6 +994,9 @@ name = "mediatype" version = "0.19.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8878cd8d1b3c8c8ae4b2ba0a36652b7cf192f618a599a7fbdfa25cffd4ea72dd" +dependencies = [ + "serde", +] [[package]] name = "memchr" @@ -913,6 +1048,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "num-bigint" version = "0.4.6" @@ -1045,6 +1186,77 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_macros", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared 0.11.2", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" +dependencies = [ + "phf_generator 0.11.2", + "phf_shared 0.11.2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher 0.3.11", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher 0.3.11", +] + [[package]] name = "pin-project-lite" version = "0.2.15" @@ -1069,6 +1281,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1103,9 +1330,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "encoding_rs", "memchr", @@ -1120,6 +1347,36 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "redox_syscall" version = "0.1.57" @@ -1324,6 +1581,22 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0e749d29b2064585327af5038a5a8eb73aeebad4a3472e83531a436563f7208" +dependencies = [ + "ahash", + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "precomputed-hash", + "selectors", + "tendril", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -1348,19 +1621,38 @@ dependencies = [ ] [[package]] -name = "serde" -version = "1.0.214" +name = "selectors" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "new_debug_unreachable", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", +] + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1369,9 +1661,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -1391,6 +1683,15 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1418,6 +1719,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + [[package]] name = "siphasher" version = "1.0.1" @@ -1461,6 +1768,32 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1532,6 +1865,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + [[package]] name = "term" version = "0.5.2" @@ -1704,6 +2048,12 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "untrusted" version = "0.9.0" @@ -1719,8 +2069,15 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -2031,6 +2388,27 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.4" diff --git a/ghost-upload/Cargo.toml b/ghost-upload/Cargo.toml index c58afc1..1ddc10d 100644 --- a/ghost-upload/Cargo.toml +++ b/ghost-upload/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] reqwest = { version = "0.12", features = ["json"] } -feed-rs = "2.1" +feed-rs = "2.2" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.41", features = ["full"] } jsonwebtoken = "9.3" @@ -16,6 +16,7 @@ hex = "0.4" chrono = "0.4" futures = "0.3" maud = "0.26" +scraper = "0.21" [dev-dependencies] -clippy = "0.0" +clippy = "0.0.302" diff --git a/ghost-upload/src/main.rs b/ghost-upload/src/main.rs index 0d542f5..33b0838 100644 --- a/ghost-upload/src/main.rs +++ b/ghost-upload/src/main.rs @@ -4,8 +4,10 @@ use futures::future::join_all; use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use maud::html; use reqwest::Client; +use scraper::{Html, Selector}; use serde::{Deserialize, Serialize}; use std::env; + #[derive(Debug, Serialize, Deserialize)] struct Claims { iat: usize, @@ -26,20 +28,23 @@ struct Post { status: String, published_at: String, updated_at: String, - feature_image: String, canonical_url: String, tags: Vec, + feature_image: Option, + feature_image_alt: Option, + feature_image_caption: Option, + meta_description: Option, + custom_excerpt: Option, } impl Post { - fn new( - title: &str, - summary: &str, - link: &str, - mut categories: Vec<&str>, - image_url: &str, - pub_date: &str, - ) -> Self { + async fn new(entry: Entry) -> Post { + let title = entry.title.as_ref().unwrap().content.clone(); + + let link = entry.links.first().unwrap().href.as_str(); + let slug = get_slug(link); + + let summary = summarize_url(link).await; let html = html! { p { (summary) } iframe src=(link) style="width: 100%; height: 80vh" { } @@ -48,45 +53,77 @@ impl Post { " The above summary was made by the " a href=("https://help.kagi.com/kagi/api/summarizer.html") {"Kagi Summarizer"} } - }; + }.into_string(); - let slug = get_slug(link); - categories.push("Projects Website"); - Post { - title: title.to_string(), - slug: slug.to_string(), - html: html.into_string(), - status: "published".to_string(), - published_at: pub_date.to_string(), - updated_at: chrono::Utc::now().to_rfc3339(), - feature_image: image_url.to_string(), - canonical_url: link.to_string(), - tags: categories.into_iter().map(|c| c.to_string()).collect(), - } - } -} + let status = "published".to_owned(); -async fn entry_to_post(entry: Entry) -> Post { - let link = entry.links.first().unwrap().href.as_str(); - let summary = summarize_url(link).await; + let published_at = entry.published.unwrap().to_rfc3339(); - Post::new( - entry.title.as_ref().unwrap().content.as_str(), - summary.as_str(), - link, - entry + let updated_at = chrono::Utc::now().to_rfc3339(); + + let canonical_url = link.to_owned(); + + let mut tags: Vec = entry .categories .iter() - .map(|category| category.term.as_str()) - .collect::>(), - entry - .media - .first() - .and_then(|m| m.content.first()) - .and_then(|c| c.url.as_ref().map(|u| u.as_str())) - .unwrap_or_default(), - &entry.published.unwrap().to_rfc3339(), - ) + .map(|category| category.term.as_str().to_owned()) + .collect(); + tags.push("Projects Website".to_owned()); + + // The rest of the data is optional and has to be pulled from the documents OpenGraph + let raw_html = reqwest::get(link).await.unwrap().text().await.unwrap(); + let document = Html::parse_document(&raw_html); + let selector = Selector::parse("meta[property^='og:']").unwrap(); + + let mut feature_image = None; + let mut feature_image_alt = None; + let mut feature_image_caption = None; + let mut meta_description = None; + let mut custom_excerpt = None; + + for meta in document.select(&selector) { + match meta.value().attr("property") { + Some("og:image") => feature_image = meta.value().attr("content").map(String::from), + Some("og:image:alt") => { + // Ghost API limits this to 190 chars + feature_image_alt = meta.value().attr("content").map(|desc| { + desc.chars().take(190).collect() + }) + } + Some("og:image:description") => { + feature_image_caption = meta.value().attr("content").map(String::from) + } + Some("og:description") => { + meta_description = meta.value().attr("content").map(String::from); + + // Ghost API limits this to 300 chars + custom_excerpt = meta.value().attr("content").map(|desc| { + desc.chars().take(300).collect() + }); + } + _ => {} + } + } + + + let x = Post { + title, + slug, + html, + status, + published_at, + updated_at, + canonical_url, + tags, + feature_image, + feature_image_alt, + feature_image_caption, + meta_description, + custom_excerpt, + }; + dbg!(&x); + x + } } fn get_slug(link: &str) -> String { @@ -205,10 +242,12 @@ async fn main() { let entries = fetch_feed(feed).await; let post_exists_futures = entries.into_iter().map(|entry| { - let entry_clone = entry.clone(); // Clone entry if necessary (depends on your data structure) + let entry_clone = entry.clone(); async move { (entry_clone, check_if_post_exists(&entry).await) } }); + let post_exists_results = join_all(post_exists_futures).await; + let filtered_entries: Vec = post_exists_results .into_iter() .filter_map(|(entry, exists)| if !exists { Some(entry) } else { None }) @@ -219,7 +258,7 @@ async fn main() { return; } - let post_futures = filtered_entries.into_iter().map(entry_to_post); + let post_futures = filtered_entries.into_iter().map(Post::new); let client = Client::new(); @@ -241,9 +280,8 @@ async fn main() { println!("Post {} published successfully.", post.title); } else { println!( - "Failed to publish post {}. Status: {:?}", - &post.title, - response.status() + "Failed to publish post {}.\n\tResp: {:?}", + &post.title, response ); } }