Merge branch 'ghost-upload' into 'master'

Upload Posts to Ghost

See merge request Anson-Projects/projects!4

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.gitignore

@@ -3,3 +3,4 @@ _site/
 .quarto/
 public/
 /.quarto/
+ghost-upload/target/

.gitlab-ci.yml

@@ -33,3 +33,6 @@ pages:
       - public
   rules:
     - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
+
+include:
+  - local: ghost-upload/.gitlab-ci.yml

ghost-upload/.gitlab-ci.yml

@@ -0,0 +1,15 @@
+cache:
+  paths:
+    - ./ghost-upload/target/
+    - ./ghost-upload/cargo/
+
+publish:
+  stage: deploy
+  image: rust:latest
+  script:
+    - cd ./ghost-upload
+    - cargo run
+  needs:
+    - pages
+  rules:
+    - if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"

ghost-upload/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "ghost-upload"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+reqwest = { version = "0.11.24", features = ["json"] }
+feed-rs = "1.4.0"
+serde = { version = "1.0.197", features = ["derive"] }
+tokio = { version = "1.36.0", features = ["full"] }
+jsonwebtoken = "9.2.0"
+serde_json = "1.0.114"
+clippy = "0.0.302"
+hex = "0.4.3"
+chrono = "0.4.34"
+futures = "0.3.30"
+maud = "0.26.0"

ghost-upload/README.md

@@ -0,0 +1,3 @@
+# ghost-upload
+
+This code uploads posts from https://projects.ansonbiggs.com to https://notes.ansonbiggs.com. I couldn't figure out how to update posts, and the kagi API doesn't make it clear how long it caches results for so for now only posts that don't exist on the ghost blog will be uploaded. If you want to update content you need to manually make edits to the code and delete posts on the blog.

ghost-upload/src/main.rs

@@ -0,0 +1,249 @@
+use feed_rs::model::Entry;
+use feed_rs::parser;
+use futures::future::join_all;
+use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
+use maud::html;
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use std::env;
+#[derive(Debug, Serialize, Deserialize)]
+struct Claims {
+    iat: usize,
+    exp: usize,
+    aud: String,
+}
+
+#[derive(Serialize, Debug)]
+struct PostPayload {
+    posts: Vec<Post>,
+}
+
+#[derive(Serialize, Debug, Clone)]
+struct Post {
+    title: String,
+    slug: String,
+    html: String,
+    status: String,
+    published_at: String,
+    updated_at: String,
+    feature_image: String,
+    canonical_url: String,
+    tags: Vec<String>,
+}
+
+impl Post {
+    fn new(
+        title: &str,
+        summary: &str,
+        link: &str,
+        mut categories: Vec<&str>,
+        image_url: &str,
+        pub_date: &str,
+    ) -> Self {
+        let html = html! {
+            p { (summary) }
+            p {
+                "This content was originally posted on my projects website " a href=(link) { "here." }
+                " The above summary was made by the " a href=("https://help.kagi.com/kagi/api/summarizer.html")
+                {"Kagi Summarizer"}
+            }
+        };
+
+        let slug = get_slug(link);
+        categories.push("Projects Website");
+        Post {
+            title: title.to_string(),
+            slug: slug.to_string(),
+            html: html.into_string(),
+            status: "published".to_string(),
+            published_at: pub_date.to_string(),
+            updated_at: chrono::Utc::now().to_rfc3339(),
+            feature_image: image_url.to_string(),
+            canonical_url: link.to_string(),
+            tags: categories.into_iter().map(|c| c.to_string()).collect(),
+        }
+    }
+}
+
+async fn entry_to_post(entry: Entry) -> Post {
+    let link = entry.links.first().unwrap().href.as_str();
+    let summary = summarize_url(link).await;
+
+    Post::new(
+        entry.title.as_ref().unwrap().content.as_str(),
+        summary.as_str(),
+        link,
+        entry
+            .categories
+            .iter()
+            .map(|category| category.term.as_str())
+            .collect::<Vec<&str>>(),
+        entry
+            .media
+            .first()
+            .and_then(|m| m.content.first())
+            .and_then(|c| c.url.as_ref().map(|u| u.as_str()))
+            .unwrap_or_default(),
+        &entry.published.unwrap().to_rfc3339(),
+    )
+}
+
+fn get_slug(link: &str) -> String {
+    link.split_once("/posts/").unwrap().1.to_string()
+}
+
+async fn check_if_post_exists(entry: &Entry) -> bool {
+    let posts_url = "https://notes.ansonbiggs.com/";
+    let link = entry.links.first().unwrap().href.as_str();
+    let slug = get_slug(link);
+
+    match reqwest::get(format!("{}{}", posts_url, slug)).await {
+        Ok(response) => response.status().is_success(),
+        Err(_) => false,
+    }
+}
+
+async fn fetch_feed(url: &str) -> Vec<Entry> {
+    let content = reqwest::get(url).await.unwrap().text().await.unwrap();
+
+    let feed = parser::parse(content.as_bytes()).unwrap();
+
+    feed.entries
+}
+
+#[derive(Serialize)]
+struct KagiPayload {
+    url: String,
+    engine: String,
+    summary_type: String,
+    target_language: String,
+    cache: bool,
+}
+impl KagiPayload {
+    fn new(url: &str) -> Self {
+        // https://help.kagi.com/kagi/api/summarizer.html
+        KagiPayload {
+            url: url.to_string(),
+            engine: "agnes".to_string(), // Formal, technical, analytical summary
+            summary_type: "summary".to_string(),
+            target_language: "EN".to_string(),
+            cache: true,
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct KagiResponse {
+    data: Data,
+}
+
+#[derive(Deserialize)]
+struct Data {
+    output: String,
+}
+
+async fn summarize_url(url: &str) -> String {
+    let kagi_url = "https://kagi.com/api/v0/summarize";
+    let kagi_api_key = env::var("kagi_api_key").unwrap();
+
+    let payload = KagiPayload::new(url);
+
+    let client = Client::new();
+
+    let response = client
+        .post(kagi_url)
+        .header("Authorization", format!("Bot {}", kagi_api_key))
+        .json(&payload)
+        .send()
+        .await
+        .expect("Kagi request failed");
+
+    response
+        .json::<KagiResponse>()
+        .await
+        .expect("Kagi didn't return json")
+        .data
+        .output
+}
+#[tokio::main]
+async fn main() {
+    let ghost_api_url = "https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/?source=html";
+    let ghost_admin_api_key = env::var("admin_api_key").unwrap();
+
+    let feed = "https://projects.ansonbiggs.com/index.xml";
+
+    // Split the key into ID and SECRET
+    let (id, secret) = ghost_admin_api_key
+        .split_once(':')
+        .expect("Invalid API key format");
+
+    // Prepare JWT header and claims
+    let iat = chrono::Utc::now().timestamp() as usize;
+    let exp = iat + 5 * 60;
+    let claims = Claims {
+        iat,
+        exp,
+        aud: "v3/admin".into(),
+    };
+
+    let header = Header {
+        alg: Algorithm::HS256,
+        kid: Some(id.into()),
+        ..Default::default()
+    };
+
+    // Encode the JWT
+    let token = encode(
+        &header,
+        &claims,
+        &EncodingKey::from_secret(&hex::decode(secret).expect("Invalid secret hex")),
+    )
+    .expect("JWT encoding failed");
+
+    // Prepare the post data
+    let mut entries = fetch_feed(feed).await;
+
+    let post_exists_futures = entries.into_iter().map(|entry| {
+        let entry_clone = entry.clone(); // Clone entry if necessary (depends on your data structure)
+        async move { (entry_clone, check_if_post_exists(&entry).await) }
+    });
+    let post_exists_results = join_all(post_exists_futures).await;
+    let filtered_entries: Vec<Entry> = post_exists_results
+        .into_iter()
+        .filter_map(|(entry, exists)| if !exists { Some(entry) } else { None })
+        .collect();
+
+    if filtered_entries.is_empty() {
+        println!("Nothing to post.");
+        return;
+    }
+
+    let post_futures = filtered_entries.into_iter().map(entry_to_post);
+
+    let client = Client::new();
+
+    for post in join_all(post_futures).await {
+        let post_payload = PostPayload {
+            posts: vec![post.clone()],
+        };
+
+        let response = client
+            .post(ghost_api_url)
+            .header("Authorization", format!("Ghost {}", token))
+            .json(&post_payload)
+            .send()
+            .await
+            .expect("Request failed");
+
+        // Check the response
+        if response.status().is_success() {
+            println!("Post {} published successfully.", post.title);
+        } else {
+            println!(
+                "Failed to publish post {}. Status: {:?}",
+                &post.title,
+                response.status()
+            );
+        }
+    }
+}