mirror of
https://gitlab.com/Anson-Projects/projects.git
synced 2025-09-19 03:52:37 +00:00
Compare commits
13 Commits
feature/gh
...
890775b2bc
Author | SHA1 | Date | |
---|---|---|---|
890775b2bc | |||
788052233a | |||
1a4773b3ef | |||
84f4e48386 | |||
52229040c6 | |||
b70c57e23e | |||
f6532e4fb6 | |||
0675f1f1b7 | |||
b5a4b33b56 | |||
9fc6a9bae1 | |||
05474b986d | |||
cdb96a50b7 | |||
e233a96f55 |
@@ -1,10 +1,15 @@
|
|||||||
|
stages:
|
||||||
|
- build
|
||||||
|
- deploy
|
||||||
|
|
||||||
build:
|
build:
|
||||||
stage: build
|
stage: build
|
||||||
image:
|
image:
|
||||||
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||||
entrypoint: [""]
|
entrypoint: [""]
|
||||||
script:
|
script:
|
||||||
- /kaniko/executor
|
- >
|
||||||
|
/kaniko/executor
|
||||||
--context "${CI_PROJECT_DIR}"
|
--context "${CI_PROJECT_DIR}"
|
||||||
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
||||||
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}"
|
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}"
|
||||||
@@ -14,13 +19,15 @@ staging:
|
|||||||
stage: deploy
|
stage: deploy
|
||||||
image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}
|
image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}
|
||||||
script:
|
script:
|
||||||
- echo "Building the project with Quarto..."
|
- echo "Building the main website with Quarto..."
|
||||||
- quarto render --to html --output-dir public
|
- quarto render --to html --output-dir public
|
||||||
|
- echo "Building Ghost-optimized version..."
|
||||||
|
- quarto render --profile ghost --to html --output-dir public/ghost-content
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
|
|
||||||
pages:
|
deploy:
|
||||||
stage: deploy
|
stage: deploy
|
||||||
script:
|
script:
|
||||||
- echo "Publishing site..."
|
- echo "Publishing site..."
|
||||||
@@ -29,6 +36,35 @@ pages:
|
|||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
|
|
||||||
|
# Branch preview deployment (for testing)
|
||||||
|
preview:
|
||||||
|
stage: deploy
|
||||||
|
script:
|
||||||
|
- echo "Deploying branch preview..."
|
||||||
|
- echo "Preview available at preview URL"
|
||||||
|
needs:
|
||||||
|
- job: staging
|
||||||
|
optional: true
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- public
|
||||||
|
environment:
|
||||||
|
name: preview/$CI_COMMIT_REF_SLUG
|
||||||
|
url: https://${CI_PROJECT_PATH_SLUG}-${CI_COMMIT_REF_SLUG}.gitlab.io
|
||||||
|
rules:
|
||||||
|
- if: "$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH"
|
||||||
|
|
||||||
|
# GitLab Pages deployment (only on main branch)
|
||||||
|
pages:
|
||||||
|
stage: deploy
|
||||||
|
script:
|
||||||
|
- echo "Publishing to GitLab Pages..."
|
||||||
|
needs:
|
||||||
|
- deploy
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- public
|
||||||
rules:
|
rules:
|
||||||
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
||||||
|
|
||||||
|
46
AGENTS.md
Normal file
46
AGENTS.md
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
# Repository Guidelines
|
||||||
|
|
||||||
|
## Project Structure & Module Organization
|
||||||
|
- `ghost-upload/`: Rust automation for Ghost CMS publishing.
|
||||||
|
- `posts/`: Quarto posts with Julia/Python code per post directory.
|
||||||
|
- `public/`: Quarto build output (generated by `quarto render`).
|
||||||
|
- Root: Quarto config (`_quarto.yml`), shared assets, CI/CD, docs.
|
||||||
|
|
||||||
|
## Build, Test, and Development Commands
|
||||||
|
- Rust (`ghost-upload/`):
|
||||||
|
- Build: `cd ghost-upload && cargo build`
|
||||||
|
- Run: `cd ghost-upload && cargo run`
|
||||||
|
- Test: `cd ghost-upload && cargo test` (single: `cargo test <test_name>`)
|
||||||
|
- Lint: `cd ghost-upload && cargo clippy`
|
||||||
|
- Format: `cd ghost-upload && cargo fmt`
|
||||||
|
- Julia (root or `posts/*/`):
|
||||||
|
- Packages: `julia -e "using Pkg; Pkg.instantiate()"`
|
||||||
|
- Precompile: `julia -e "using Pkg; Pkg.precompile()"`
|
||||||
|
- Run notebook/script: `julia <filename>.jl`
|
||||||
|
- Quarto (docs/site):
|
||||||
|
- Build site: `quarto render --to html --output-dir public`
|
||||||
|
- Preview: `quarto preview`
|
||||||
|
- Check: `quarto check`
|
||||||
|
- Docker: `docker build -t projects .` then `docker run projects`
|
||||||
|
|
||||||
|
## Coding Style & Naming Conventions
|
||||||
|
- Rust: `cargo fmt`; fix all `cargo clippy` warnings. Use `?` over `unwrap()`. Imports: std → external → local. Naming: snake_case (fn/vars), PascalCase (types). Public docs with `///`.
|
||||||
|
- Julia: 4-space indent; spaces around operators; group `using` at top; snake_case; prefer pipelines `|>` for DataFrames; handle expected errors with try-catch.
|
||||||
|
- Quarto: Include title/date in YAML; set `echo: false`, `warning: false` for clean outputs; descriptive figure captions and alt text.
|
||||||
|
|
||||||
|
## Testing Guidelines
|
||||||
|
- Rust: Unit tests for core logic; add integration tests for API calls. Run with `cargo test`. Organize tests near code or in `tests/`.
|
||||||
|
- Julia: Validate transformations and plots visually; keep scripts deterministic.
|
||||||
|
- Quarto: Manually review rendered HTML for links, figures, and warnings.
|
||||||
|
|
||||||
|
## Commit & Pull Request Guidelines
|
||||||
|
- Commits: Use clear, conventional messages (e.g., `feat:`, `fix:`, `docs:`). Scope small and focused.
|
||||||
|
- PRs: Provide description, linked issues, steps to validate (commands), and screenshots of rendered docs when relevant.
|
||||||
|
|
||||||
|
## Security & Configuration
|
||||||
|
- Environment variables: `kagi_api_key`, `admin_api_key`. Export locally (e.g., `export admin_api_key=...`); never commit secrets.
|
||||||
|
- Dependencies: Keep minimal and up-to-date. Prefer configuration via env vars over hardcoded values.
|
||||||
|
|
||||||
|
## CI/CD & Deployment
|
||||||
|
- GitLab CI builds Docker, renders Quarto to static hosting; Rust runs separately for content sync. Avoid pipeline changes unless necessary; include rationale in PRs if modified.
|
||||||
|
|
21
_quarto.yml
21
_quarto.yml
@@ -1,7 +1,9 @@
|
|||||||
project:
|
project:
|
||||||
type: website
|
type: website
|
||||||
|
|
||||||
website:
|
profiles:
|
||||||
|
default:
|
||||||
|
website:
|
||||||
title: "Anson's Projects"
|
title: "Anson's Projects"
|
||||||
site-url: https://projects.ansonbiggs.com
|
site-url: https://projects.ansonbiggs.com
|
||||||
description: A Blog for Technical Topics
|
description: A Blog for Technical Topics
|
||||||
@@ -15,11 +17,26 @@ website:
|
|||||||
# - icon: gitlab
|
# - icon: gitlab
|
||||||
# href: https://gitlab.com/MisterBiggs
|
# href: https://gitlab.com/MisterBiggs
|
||||||
open-graph: true
|
open-graph: true
|
||||||
format:
|
format:
|
||||||
html:
|
html:
|
||||||
theme: zephyr
|
theme: zephyr
|
||||||
css: styles.css
|
css: styles.css
|
||||||
# toc: true
|
# toc: true
|
||||||
|
|
||||||
|
ghost:
|
||||||
|
website:
|
||||||
|
title: "Anson's Projects"
|
||||||
|
site-url: https://projects.ansonbiggs.com
|
||||||
|
description: A Blog for Technical Topics
|
||||||
|
navbar: false
|
||||||
|
open-graph: true
|
||||||
|
format:
|
||||||
|
html:
|
||||||
|
theme: none
|
||||||
|
css: ghost-iframe.css
|
||||||
|
toc: false
|
||||||
|
page-layout: article
|
||||||
|
title-block-banner: false
|
||||||
|
|
||||||
execute:
|
execute:
|
||||||
freeze: true
|
freeze: true
|
129
ghost-iframe.css
Normal file
129
ghost-iframe.css
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
/* Ghost iframe optimized styles */
|
||||||
|
body {
|
||||||
|
font-family: system-ui, -apple-system, sans-serif;
|
||||||
|
line-height: 1.6;
|
||||||
|
color: #333;
|
||||||
|
max-width: 100%;
|
||||||
|
margin: 0;
|
||||||
|
padding: 20px;
|
||||||
|
background: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Remove any potential margins/padding */
|
||||||
|
html, body {
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Ensure content flows naturally */
|
||||||
|
#quarto-content {
|
||||||
|
max-width: none;
|
||||||
|
padding: 0;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Style headings for Ghost */
|
||||||
|
h1, h2, h3, h4, h5, h6 {
|
||||||
|
margin-top: 1.5em;
|
||||||
|
margin-bottom: 0.5em;
|
||||||
|
font-weight: 600;
|
||||||
|
line-height: 1.3;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 { font-size: 2em; }
|
||||||
|
h2 { font-size: 1.5em; }
|
||||||
|
h3 { font-size: 1.25em; }
|
||||||
|
|
||||||
|
/* Code blocks */
|
||||||
|
pre {
|
||||||
|
background: #f8f9fa;
|
||||||
|
border: 1px solid #e9ecef;
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 1rem;
|
||||||
|
overflow-x: auto;
|
||||||
|
font-size: 0.875em;
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
font-family: "SF Mono", Monaco, "Cascadia Code", "Roboto Mono", Consolas, "Courier New", monospace;
|
||||||
|
background: #f1f3f4;
|
||||||
|
padding: 0.2em 0.4em;
|
||||||
|
border-radius: 3px;
|
||||||
|
font-size: 0.875em;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre code {
|
||||||
|
background: none;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Images */
|
||||||
|
img {
|
||||||
|
max-width: 100%;
|
||||||
|
height: auto;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Tables */
|
||||||
|
table {
|
||||||
|
border-collapse: collapse;
|
||||||
|
width: 100%;
|
||||||
|
margin: 1em 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
th, td {
|
||||||
|
border: 1px solid #ddd;
|
||||||
|
padding: 8px;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
th {
|
||||||
|
background-color: #f2f2f2;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Links */
|
||||||
|
a {
|
||||||
|
color: #0066cc;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
a:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Blockquotes */
|
||||||
|
blockquote {
|
||||||
|
border-left: 4px solid #ddd;
|
||||||
|
margin: 1em 0;
|
||||||
|
padding-left: 1em;
|
||||||
|
color: #666;
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Lists */
|
||||||
|
ul, ol {
|
||||||
|
padding-left: 1.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
li {
|
||||||
|
margin-bottom: 0.25em;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Remove any navbar/footer elements that might leak through */
|
||||||
|
.navbar, .nav, footer, .sidebar, .toc, .page-footer {
|
||||||
|
display: none !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Ensure responsive behavior for iframe */
|
||||||
|
@media (max-width: 768px) {
|
||||||
|
body {
|
||||||
|
padding: 15px;
|
||||||
|
font-size: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 { font-size: 1.75em; }
|
||||||
|
h2 { font-size: 1.35em; }
|
||||||
|
h3 { font-size: 1.15em; }
|
||||||
|
}
|
@@ -10,20 +10,26 @@ publish:
|
|||||||
- cd ./ghost-upload
|
- cd ./ghost-upload
|
||||||
- cargo run
|
- cargo run
|
||||||
needs:
|
needs:
|
||||||
- pages
|
- job: deploy
|
||||||
rules:
|
optional: true
|
||||||
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
- job: staging
|
||||||
|
optional: true
|
||||||
|
|
||||||
publish_update:
|
# Manual trigger to force update all Ghost posts
|
||||||
|
force-update-ghost:
|
||||||
stage: deploy
|
stage: deploy
|
||||||
image: rust:latest
|
image: rust:latest
|
||||||
variables:
|
|
||||||
UPDATE_EXISTING: "true"
|
|
||||||
script:
|
script:
|
||||||
|
- echo "🔄 Force updating all Ghost posts..."
|
||||||
- cd ./ghost-upload
|
- cd ./ghost-upload
|
||||||
- cargo run
|
- FORCE_UPDATE=true cargo run
|
||||||
needs:
|
needs:
|
||||||
- pages
|
- job: deploy
|
||||||
|
optional: true
|
||||||
|
- job: staging
|
||||||
|
optional: true
|
||||||
rules:
|
rules:
|
||||||
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
- when: manual
|
||||||
when: manual
|
allow_failure: false
|
||||||
|
variables:
|
||||||
|
FORCE_UPDATE: "true"
|
||||||
|
1072
ghost-upload/Cargo.lock
generated
1072
ghost-upload/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,17 +1,39 @@
|
|||||||
# ghost-upload
|
# ghost-upload
|
||||||
|
|
||||||
This tool uploads posts from https://projects.ansonbiggs.com to https://notes.ansonbiggs.com.
|
This tool synchronizes posts from https://projects.ansonbiggs.com to the Ghost blog at https://notes.ansonbiggs.com.
|
||||||
|
|
||||||
What's new:
|
## Features
|
||||||
- Uses the Ghost Admin API to check for existing posts by slug instead of probing the public site.
|
|
||||||
- Optional update support: set `UPDATE_EXISTING=true` to update an existing post in-place (via `PUT /ghost/api/v3/admin/posts/{id}?source=html`).
|
|
||||||
- Safer slug handling (trims trailing `/` and falls back to the last path segment).
|
|
||||||
|
|
||||||
Env vars:
|
- **Automatic sync**: Only uploads new posts by default
|
||||||
- `admin_api_key`: Ghost Admin API key in `key_id:secret` format.
|
- **Content extraction**: Fetches clean HTML content instead of using iframes
|
||||||
- `kagi_api_key`: Kagi Summarizer API key.
|
- **AI summaries**: Uses Kagi Summarizer for post summaries
|
||||||
- `UPDATE_EXISTING` (optional): if `true`/`1`, update posts that already exist in Ghost.
|
- **Force update**: Manual trigger to update all existing posts
|
||||||
|
|
||||||
Notes:
|
## Usage
|
||||||
- Updates use optimistic concurrency by sending the current `updated_at` from Ghost. If someone edits a post in Ghost after we fetch it, the update will fail with a 409 and be reported in the console.
|
|
||||||
- Summaries are always regenerated when creating or updating; if you want to avoid re-summarizing on updates, leave `UPDATE_EXISTING` unset.
|
### Normal Mode (Default)
|
||||||
|
```bash
|
||||||
|
cargo run
|
||||||
|
```
|
||||||
|
Only processes new posts that don't exist on the Ghost blog.
|
||||||
|
|
||||||
|
### Force Update Mode
|
||||||
|
```bash
|
||||||
|
FORCE_UPDATE=true cargo run
|
||||||
|
```
|
||||||
|
Updates ALL posts, including existing ones. Useful for:
|
||||||
|
- Updating content after changes
|
||||||
|
- Refreshing summaries
|
||||||
|
- Applying new styling/formatting
|
||||||
|
|
||||||
|
## CI/CD Integration
|
||||||
|
|
||||||
|
The GitLab CI pipeline includes:
|
||||||
|
- **Automatic sync**: Runs after each deployment
|
||||||
|
- **Manual force update**: Available as a manual trigger in GitLab UI
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
- `admin_api_key`: Ghost Admin API key (required)
|
||||||
|
- `kagi_api_key`: Kagi Summarizer API key (required)
|
||||||
|
- `FORCE_UPDATE`: Set to "true" to update all posts (optional)
|
@@ -1,5 +1,6 @@
|
|||||||
use feed_rs::model::Entry;
|
use feed_rs::model::Entry;
|
||||||
use feed_rs::parser;
|
use feed_rs::parser;
|
||||||
|
use futures::future::join_all;
|
||||||
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
|
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
|
||||||
use maud::html;
|
use maud::html;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
@@ -19,29 +20,6 @@ struct PostPayload {
|
|||||||
posts: Vec<Post>,
|
posts: Vec<Post>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Debug, Clone)]
|
|
||||||
struct UpdatePost {
|
|
||||||
id: String,
|
|
||||||
title: String,
|
|
||||||
slug: String,
|
|
||||||
html: String,
|
|
||||||
status: String,
|
|
||||||
published_at: String,
|
|
||||||
updated_at: String,
|
|
||||||
canonical_url: String,
|
|
||||||
tags: Vec<String>,
|
|
||||||
feature_image: Option<String>,
|
|
||||||
feature_image_alt: Option<String>,
|
|
||||||
feature_image_caption: Option<String>,
|
|
||||||
meta_description: Option<String>,
|
|
||||||
custom_excerpt: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Debug)]
|
|
||||||
struct UpdatePayload {
|
|
||||||
posts: Vec<UpdatePost>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize, Debug, Clone)]
|
#[derive(Serialize, Debug, Clone)]
|
||||||
struct Post {
|
struct Post {
|
||||||
title: String,
|
title: String,
|
||||||
@@ -67,13 +45,29 @@ impl Post {
|
|||||||
let slug = get_slug(link);
|
let slug = get_slug(link);
|
||||||
|
|
||||||
let summary = summarize_url(link).await;
|
let summary = summarize_url(link).await;
|
||||||
|
|
||||||
|
// Extract content from ghost-optimized version
|
||||||
|
let ghost_content = extract_article_content(&link).await;
|
||||||
|
|
||||||
let html = html! {
|
let html = html! {
|
||||||
|
div class="ghost-summary" {
|
||||||
|
h3 { "Summary" }
|
||||||
p { (summary) }
|
p { (summary) }
|
||||||
iframe src=(link) style="width: 100%; height: 80vh" { }
|
}
|
||||||
|
div class="ghost-content" {
|
||||||
|
(maud::PreEscaped(ghost_content))
|
||||||
|
}
|
||||||
|
div class="ghost-footer" {
|
||||||
|
hr {}
|
||||||
p {
|
p {
|
||||||
"This content was originally posted on my projects website " a href=(link) { "here." }
|
em {
|
||||||
" The above summary was made by the " a href=("https://help.kagi.com/kagi/api/summarizer.html")
|
"This content was originally posted on my projects website "
|
||||||
{"Kagi Summarizer"}
|
a href=(link) { "here" }
|
||||||
|
". The above summary was generated by the "
|
||||||
|
a href=("https://help.kagi.com/kagi/api/summarizer.html") {"Kagi Summarizer"}
|
||||||
|
"."
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}.into_string();
|
}.into_string();
|
||||||
|
|
||||||
@@ -143,54 +137,105 @@ impl Post {
|
|||||||
meta_description,
|
meta_description,
|
||||||
custom_excerpt,
|
custom_excerpt,
|
||||||
};
|
};
|
||||||
|
dbg!(&x);
|
||||||
x
|
x
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_slug(link: &str) -> String {
|
fn get_slug(link: &str) -> String {
|
||||||
// Prefer portion after "/posts/" if present, otherwise fall back to the last path segment
|
link.split_once("/posts/").unwrap().1.to_string()
|
||||||
let raw = match link.split_once("/posts/") {
|
}
|
||||||
Some((_, rest)) => rest,
|
|
||||||
None => link.rsplit('/').next().unwrap_or(link),
|
async fn extract_article_content(original_link: &str) -> String {
|
||||||
};
|
// Convert original link to ghost-content version
|
||||||
raw.trim_end_matches('/')
|
let ghost_link = original_link.replace("projects.ansonbiggs.com", "projects.ansonbiggs.com/ghost-content");
|
||||||
.to_string()
|
|
||||||
|
match reqwest::get(&ghost_link).await {
|
||||||
|
Ok(response) => {
|
||||||
|
match response.text().await {
|
||||||
|
Ok(html_content) => {
|
||||||
|
let document = Html::parse_document(&html_content);
|
||||||
|
|
||||||
|
// Try different selectors to find the main content
|
||||||
|
let content_selectors = [
|
||||||
|
"#quarto-content main",
|
||||||
|
"#quarto-content",
|
||||||
|
"main",
|
||||||
|
"article",
|
||||||
|
".content",
|
||||||
|
"body"
|
||||||
|
];
|
||||||
|
|
||||||
|
for selector_str in &content_selectors {
|
||||||
|
if let Ok(selector) = Selector::parse(selector_str) {
|
||||||
|
if let Some(element) = document.select(&selector).next() {
|
||||||
|
let content = element.inner_html();
|
||||||
|
|
||||||
|
if !content.trim().is_empty() {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback: return original content with iframe if extraction fails
|
||||||
|
format!(r#"<div class="fallback-iframe">
|
||||||
|
<p><em>Content extraction failed. Falling back to embedded view:</em></p>
|
||||||
|
<iframe src="{}" style="width: 100%; height: 80vh; border: none;" loading="lazy"></iframe>
|
||||||
|
</div>"#, original_link)
|
||||||
|
}
|
||||||
|
Err(_) => format!(r#"<p><em>Failed to fetch content. <a href="{}">View original post</a></em></p>"#, original_link)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => format!(r#"<p><em>Failed to fetch content. <a href="{}">View original post</a></em></p>"#, original_link)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn check_if_post_exists(entry: &Entry) -> bool {
|
||||||
|
let posts_url = "https://notes.ansonbiggs.com/";
|
||||||
|
let link = entry.links.first().unwrap().href.as_str();
|
||||||
|
let slug = get_slug(link);
|
||||||
|
|
||||||
|
match reqwest::get(format!("{}{}", posts_url, slug)).await {
|
||||||
|
Ok(response) => response.status().is_success(),
|
||||||
|
Err(_) => false,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
#[derive(Deserialize, Debug)]
|
||||||
struct GhostPostSummary {
|
struct GhostPostsResponse {
|
||||||
|
posts: Vec<GhostPost>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize, Debug)]
|
||||||
|
struct GhostPost {
|
||||||
id: String,
|
id: String,
|
||||||
slug: String,
|
slug: String,
|
||||||
updated_at: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
async fn get_existing_post_id(slug: &str, token: &str) -> Option<String> {
|
||||||
struct GhostPostsResponse<T> {
|
let client = Client::new();
|
||||||
posts: Vec<T>,
|
let api_url = format!("https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/slug/{}/", slug);
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_existing_post_by_slug(
|
match client
|
||||||
client: &Client,
|
.get(&api_url)
|
||||||
ghost_admin_base: &str,
|
|
||||||
token: &str,
|
|
||||||
slug: &str,
|
|
||||||
) -> Option<GhostPostSummary> {
|
|
||||||
// Use Ghost Admin API to search by slug
|
|
||||||
let url = format!(
|
|
||||||
"{}/posts/?filter=slug:{}&fields=id,slug,updated_at",
|
|
||||||
ghost_admin_base, slug
|
|
||||||
);
|
|
||||||
let resp = client
|
|
||||||
.get(url)
|
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.ok()?;
|
{
|
||||||
if !resp.status().is_success() {
|
Ok(response) => {
|
||||||
return None;
|
if response.status().is_success() {
|
||||||
|
if let Ok(ghost_response) = response.json::<GhostPostsResponse>().await {
|
||||||
|
ghost_response.posts.first().map(|post| post.id.clone())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => None,
|
||||||
}
|
}
|
||||||
let json = resp.json::<GhostPostsResponse<GhostPostSummary>>().await.ok()?;
|
|
||||||
json.posts.into_iter().next()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_feed(url: &str) -> Vec<Entry> {
|
async fn fetch_feed(url: &str) -> Vec<Entry> {
|
||||||
@@ -257,10 +302,19 @@ async fn summarize_url(url: &str) -> String {
|
|||||||
}
|
}
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
let ghost_admin_base = "https://notes.ansonbiggs.com/ghost/api/v3/admin";
|
let ghost_api_url = "https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/?source=html";
|
||||||
let ghost_posts_create_url = format!("{}/posts/?source=html", ghost_admin_base);
|
|
||||||
let ghost_admin_api_key = env::var("admin_api_key").unwrap();
|
let ghost_admin_api_key = env::var("admin_api_key").unwrap();
|
||||||
|
|
||||||
|
// Check if force update is enabled
|
||||||
|
let force_update = env::var("FORCE_UPDATE").unwrap_or_default() == "true";
|
||||||
|
|
||||||
|
if force_update {
|
||||||
|
println!("🔄 FORCE UPDATE MODE ENABLED");
|
||||||
|
println!(" This will update ALL posts, including existing ones.");
|
||||||
|
} else {
|
||||||
|
println!("📝 NORMAL MODE - Only publishing new posts");
|
||||||
|
}
|
||||||
|
|
||||||
let feed = "https://projects.ansonbiggs.com/index.xml";
|
let feed = "https://projects.ansonbiggs.com/index.xml";
|
||||||
|
|
||||||
// Split the key into ID and SECRET
|
// Split the key into ID and SECRET
|
||||||
@@ -291,87 +345,84 @@ async fn main() {
|
|||||||
)
|
)
|
||||||
.expect("JWT encoding failed");
|
.expect("JWT encoding failed");
|
||||||
|
|
||||||
let client = Client::new();
|
|
||||||
// Prepare the post data
|
// Prepare the post data
|
||||||
let entries = fetch_feed(feed).await;
|
let entries = fetch_feed(feed).await;
|
||||||
|
|
||||||
// Control whether to update existing posts via env var
|
let filtered_entries: Vec<Entry> = if force_update {
|
||||||
let update_existing = env::var("UPDATE_EXISTING").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false);
|
println!("🔄 Force update enabled - processing all {} posts", entries.len());
|
||||||
|
entries
|
||||||
|
} else {
|
||||||
|
let post_exists_futures = entries.into_iter().map(|entry| {
|
||||||
|
let entry_clone = entry.clone();
|
||||||
|
async move { (entry_clone, check_if_post_exists(&entry).await) }
|
||||||
|
});
|
||||||
|
|
||||||
for entry in entries {
|
let post_exists_results = join_all(post_exists_futures).await;
|
||||||
let link = entry.links.first().unwrap().href.as_str();
|
|
||||||
let slug = get_slug(link);
|
|
||||||
|
|
||||||
let existing = get_existing_post_by_slug(&client, ghost_admin_base, &token, &slug).await;
|
let new_entries: Vec<Entry> = post_exists_results
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(entry, exists)| if !exists { Some(entry) } else { None })
|
||||||
|
.collect();
|
||||||
|
|
||||||
match existing {
|
println!("📝 Found {} new posts to publish", new_entries.len());
|
||||||
None => {
|
new_entries
|
||||||
// Create new post
|
};
|
||||||
let post = Post::new(entry.clone()).await;
|
|
||||||
let post_payload = PostPayload { posts: vec![post.clone()] };
|
|
||||||
|
|
||||||
let response = client
|
if filtered_entries.is_empty() {
|
||||||
.post(&ghost_posts_create_url)
|
println!("Nothing to post.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let post_futures = filtered_entries.into_iter().map(Post::new);
|
||||||
|
|
||||||
|
let client = Client::new();
|
||||||
|
|
||||||
|
for post in join_all(post_futures).await {
|
||||||
|
let post_payload = PostPayload {
|
||||||
|
posts: vec![post.clone()],
|
||||||
|
};
|
||||||
|
|
||||||
|
// Check if this is an update (for force_update mode)
|
||||||
|
let (method, url) = if force_update {
|
||||||
|
if let Some(existing_id) = get_existing_post_id(&post.slug, &token).await {
|
||||||
|
println!("🔄 Updating existing post: {}", post.title);
|
||||||
|
("PUT", format!("https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/{}/", existing_id))
|
||||||
|
} else {
|
||||||
|
println!("📝 Creating new post: {}", post.title);
|
||||||
|
("POST", ghost_api_url.to_string())
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
println!("📝 Creating new post: {}", post.title);
|
||||||
|
("POST", ghost_api_url.to_string())
|
||||||
|
};
|
||||||
|
|
||||||
|
let response = match method {
|
||||||
|
"PUT" => client
|
||||||
|
.put(&url)
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
.json(&post_payload)
|
.json(&post_payload)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.expect("Request failed");
|
.expect("Request failed"),
|
||||||
|
_ => client
|
||||||
if response.status().is_success() {
|
.post(&url)
|
||||||
println!("Post {} published successfully.", post.title);
|
|
||||||
} else {
|
|
||||||
println!(
|
|
||||||
"Failed to publish post {}.\n\tStatus: {}",
|
|
||||||
&post.title,
|
|
||||||
response.status()
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(summary) => {
|
|
||||||
if !update_existing {
|
|
||||||
println!("Post '{}' exists (slug: {}), skipping.", entry.title.unwrap().content, slug);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update existing post
|
|
||||||
let post = Post::new(entry.clone()).await;
|
|
||||||
let update = UpdatePost {
|
|
||||||
id: summary.id,
|
|
||||||
title: post.title,
|
|
||||||
slug: post.slug,
|
|
||||||
html: post.html,
|
|
||||||
status: post.status,
|
|
||||||
published_at: post.published_at,
|
|
||||||
updated_at: summary.updated_at,
|
|
||||||
canonical_url: post.canonical_url,
|
|
||||||
tags: post.tags,
|
|
||||||
feature_image: post.feature_image,
|
|
||||||
feature_image_alt: post.feature_image_alt,
|
|
||||||
feature_image_caption: post.feature_image_caption,
|
|
||||||
meta_description: post.meta_description,
|
|
||||||
custom_excerpt: post.custom_excerpt,
|
|
||||||
};
|
|
||||||
|
|
||||||
let update_url = format!("{}/posts/{}/?source=html", ghost_admin_base, update.id);
|
|
||||||
let response = client
|
|
||||||
.put(update_url)
|
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
.json(&UpdatePayload { posts: vec![update] })
|
.json(&post_payload)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.expect("Update request failed");
|
.expect("Request failed"),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Check the response
|
||||||
if response.status().is_success() {
|
if response.status().is_success() {
|
||||||
println!("Post '{}' updated successfully.", entry.title.unwrap().content);
|
let action = if method == "PUT" { "updated" } else { "published" };
|
||||||
|
println!("✅ Post '{}' {} successfully.", post.title, action);
|
||||||
} else {
|
} else {
|
||||||
|
let action = if method == "PUT" { "update" } else { "publish" };
|
||||||
println!(
|
println!(
|
||||||
"Failed to update post '{}' (status: {}).",
|
"❌ Failed to {} post '{}'.\n\tStatus: {}\n\tResponse: {:?}",
|
||||||
entry.title.unwrap().content,
|
action, &post.title, response.status(), response.text().await.unwrap_or_default()
|
||||||
response.status()
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
34
test-ghost-profile.md
Normal file
34
test-ghost-profile.md
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
# Test Ghost Profile Output
|
||||||
|
|
||||||
|
This is a test document to validate our ghost profile setup.
|
||||||
|
|
||||||
|
## Content Structure
|
||||||
|
|
||||||
|
The ghost profile should:
|
||||||
|
- Remove navigation elements
|
||||||
|
- Use minimal styling from ghost-iframe.css
|
||||||
|
- Maintain clean article layout
|
||||||
|
- Remove table of contents
|
||||||
|
|
||||||
|
## Code Example
|
||||||
|
|
||||||
|
```julia
|
||||||
|
println("Hello from Julia!")
|
||||||
|
x = 1 + 1
|
||||||
|
```
|
||||||
|
|
||||||
|
## Regular Content
|
||||||
|
|
||||||
|
This is just some regular markdown content to see how it renders in the ghost profile.
|
||||||
|
|
||||||
|
- List item 1
|
||||||
|
- List item 2
|
||||||
|
- List item 3
|
||||||
|
|
||||||
|
**Bold text** and *italic text* should render properly.
|
||||||
|
|
||||||
|
[Link to main site](https://projects.ansonbiggs.com)
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
If you can see clean, minimal styling without navigation, the ghost profile is working correctly.
|
55
test-local-deployment.sh
Executable file
55
test-local-deployment.sh
Executable file
@@ -0,0 +1,55 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo "🧪 Testing local deployment simulation..."
|
||||||
|
|
||||||
|
# Create test directories
|
||||||
|
mkdir -p test-output/main
|
||||||
|
mkdir -p test-output/ghost-content
|
||||||
|
|
||||||
|
echo "📁 Simulating dual-output build..."
|
||||||
|
|
||||||
|
# Test 1: Check if ghost profile exists
|
||||||
|
if grep -q "ghost:" _quarto.yml; then
|
||||||
|
echo "✅ Ghost profile configuration found"
|
||||||
|
else
|
||||||
|
echo "❌ Ghost profile not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Test 2: Simulate content extraction
|
||||||
|
echo "🔍 Testing content extraction logic..."
|
||||||
|
cd ghost-upload
|
||||||
|
|
||||||
|
# Test with sample URL (without actually hitting network)
|
||||||
|
echo "📝 Testing Rust compilation and basic logic..."
|
||||||
|
if cargo check --quiet; then
|
||||||
|
echo "✅ Rust code compiles successfully"
|
||||||
|
else
|
||||||
|
echo "❌ Rust compilation failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
|
||||||
|
# Test 3: Check if CI would work
|
||||||
|
echo "🔧 Validating CI configuration..."
|
||||||
|
if ./validate-ghost-extraction.sh > /dev/null 2>&1; then
|
||||||
|
echo "✅ CI validation passed"
|
||||||
|
else
|
||||||
|
echo "❌ CI validation failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "🎉 Local testing complete!"
|
||||||
|
echo ""
|
||||||
|
echo "📋 What happens in CI:"
|
||||||
|
echo " 1. Builds main site → public/"
|
||||||
|
echo " 2. Builds ghost content → public/ghost-content/"
|
||||||
|
echo " 3. Rust extracts from ghost-content URLs"
|
||||||
|
echo " 4. Posts to Ghost blog with clean HTML"
|
||||||
|
echo ""
|
||||||
|
echo "🚀 Ready for branch testing in GitLab CI!"
|
||||||
|
echo " • Download artifacts to see both outputs"
|
||||||
|
echo " • Use manual trigger to test force-update"
|
||||||
|
echo " • Check ghost-content/ folder structure"
|
89
validate-ghost-extraction.sh
Executable file
89
validate-ghost-extraction.sh
Executable file
@@ -0,0 +1,89 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Simple validation script for ghost content extraction
|
||||||
|
echo "🔍 Validating ghost profile implementation..."
|
||||||
|
|
||||||
|
# Check if required files exist
|
||||||
|
echo "📁 Checking required files..."
|
||||||
|
if [ ! -f "_quarto.yml" ]; then
|
||||||
|
echo "❌ _quarto.yml not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "ghost-iframe.css" ]; then
|
||||||
|
echo "❌ ghost-iframe.css not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -f "ghost-upload/src/main.rs" ]; then
|
||||||
|
echo "❌ Rust source not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "✅ All required files present"
|
||||||
|
|
||||||
|
# Check if ghost profile is defined in _quarto.yml
|
||||||
|
echo "📋 Checking ghost profile configuration..."
|
||||||
|
if grep -q "ghost:" _quarto.yml; then
|
||||||
|
echo "✅ Ghost profile found in _quarto.yml"
|
||||||
|
else
|
||||||
|
echo "❌ Ghost profile not found in _quarto.yml"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if GitLab CI builds both versions
|
||||||
|
echo "🔧 Checking GitLab CI configuration..."
|
||||||
|
if grep -q "ghost-content" .gitlab-ci.yml; then
|
||||||
|
echo "✅ GitLab CI configured for dual output"
|
||||||
|
else
|
||||||
|
echo "❌ GitLab CI not configured for ghost-content"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if Rust code has extract_article_content function
|
||||||
|
echo "🦀 Checking Rust implementation..."
|
||||||
|
if grep -q "extract_article_content" ghost-upload/src/main.rs; then
|
||||||
|
echo "✅ Content extraction function found"
|
||||||
|
else
|
||||||
|
echo "❌ Content extraction function not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if force update functionality is available
|
||||||
|
if grep -q "FORCE_UPDATE" ghost-upload/src/main.rs; then
|
||||||
|
echo "✅ Force update functionality found"
|
||||||
|
else
|
||||||
|
echo "❌ Force update functionality not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if manual CI job is configured
|
||||||
|
if grep -q "force-update-ghost" ghost-upload/.gitlab-ci.yml; then
|
||||||
|
echo "✅ Manual force update CI job found"
|
||||||
|
else
|
||||||
|
echo "❌ Manual force update CI job not found"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Verify Rust code compiles
|
||||||
|
echo "🛠️ Building Rust code..."
|
||||||
|
cd ghost-upload
|
||||||
|
if cargo check --quiet; then
|
||||||
|
echo "✅ Rust code compiles successfully"
|
||||||
|
else
|
||||||
|
echo "❌ Rust compilation failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd ..
|
||||||
|
echo ""
|
||||||
|
echo "🎉 All validations passed!"
|
||||||
|
echo "📋 Summary of changes:"
|
||||||
|
echo " • Quarto profiles for dual-output rendering"
|
||||||
|
echo " • Ghost-optimized CSS styling"
|
||||||
|
echo " • GitLab CI builds both main site and ghost-content"
|
||||||
|
echo " • Rust extracts HTML content instead of using iframes"
|
||||||
|
echo " • Force update mode to refresh existing posts"
|
||||||
|
echo " • Manual CI trigger for content updates"
|
||||||
|
echo ""
|
||||||
|
echo "🚀 Ready for testing in CI/CD pipeline!"
|
Reference in New Issue
Block a user