mirror of
https://gitlab.com/Anson-Projects/projects.git
synced 2025-09-19 03:52:37 +00:00
Compare commits
1 Commits
06357e6d27
...
feature/gh
Author | SHA1 | Date | |
---|---|---|---|
6aeb0ea8eb |
@@ -1,15 +1,10 @@
|
|||||||
stages:
|
|
||||||
- build
|
|
||||||
- deploy
|
|
||||||
|
|
||||||
build:
|
build:
|
||||||
stage: build
|
stage: build
|
||||||
image:
|
image:
|
||||||
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
name: gcr.io/kaniko-project/executor:v1.23.2-debug
|
||||||
entrypoint: [""]
|
entrypoint: [""]
|
||||||
script:
|
script:
|
||||||
- >
|
- /kaniko/executor
|
||||||
/kaniko/executor
|
|
||||||
--context "${CI_PROJECT_DIR}"
|
--context "${CI_PROJECT_DIR}"
|
||||||
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
||||||
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}"
|
--destination "${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}"
|
||||||
@@ -19,15 +14,13 @@ staging:
|
|||||||
stage: deploy
|
stage: deploy
|
||||||
image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}
|
image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH}
|
||||||
script:
|
script:
|
||||||
- echo "Building the main website with Quarto..."
|
- echo "Building the project with Quarto..."
|
||||||
- quarto render --to html --output-dir public
|
- quarto render --to html --output-dir public
|
||||||
- echo "Building Ghost-optimized version..."
|
|
||||||
- quarto render --profile ghost --to html --output-dir public/ghost-content
|
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
|
|
||||||
deploy:
|
pages:
|
||||||
stage: deploy
|
stage: deploy
|
||||||
script:
|
script:
|
||||||
- echo "Publishing site..."
|
- echo "Publishing site..."
|
||||||
@@ -36,35 +29,6 @@ deploy:
|
|||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- public
|
- public
|
||||||
|
|
||||||
# Branch preview deployment (for testing)
|
|
||||||
preview:
|
|
||||||
stage: deploy
|
|
||||||
script:
|
|
||||||
- echo "Deploying branch preview..."
|
|
||||||
- echo "Preview available at preview URL"
|
|
||||||
needs:
|
|
||||||
- job: staging
|
|
||||||
optional: true
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- public
|
|
||||||
environment:
|
|
||||||
name: preview/$CI_COMMIT_REF_SLUG
|
|
||||||
url: https://${CI_PROJECT_PATH_SLUG}-${CI_COMMIT_REF_SLUG}.gitlab.io
|
|
||||||
rules:
|
|
||||||
- if: "$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH"
|
|
||||||
|
|
||||||
# GitLab Pages deployment (only on main branch)
|
|
||||||
pages:
|
|
||||||
stage: deploy
|
|
||||||
script:
|
|
||||||
- echo "Publishing to GitLab Pages..."
|
|
||||||
needs:
|
|
||||||
- deploy
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- public
|
|
||||||
rules:
|
rules:
|
||||||
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
||||||
|
|
||||||
|
46
AGENTS.md
46
AGENTS.md
@@ -1,46 +0,0 @@
|
|||||||
# Repository Guidelines
|
|
||||||
|
|
||||||
## Project Structure & Module Organization
|
|
||||||
- `ghost-upload/`: Rust automation for Ghost CMS publishing.
|
|
||||||
- `posts/`: Quarto posts with Julia/Python code per post directory.
|
|
||||||
- `public/`: Quarto build output (generated by `quarto render`).
|
|
||||||
- Root: Quarto config (`_quarto.yml`), shared assets, CI/CD, docs.
|
|
||||||
|
|
||||||
## Build, Test, and Development Commands
|
|
||||||
- Rust (`ghost-upload/`):
|
|
||||||
- Build: `cd ghost-upload && cargo build`
|
|
||||||
- Run: `cd ghost-upload && cargo run`
|
|
||||||
- Test: `cd ghost-upload && cargo test` (single: `cargo test <test_name>`)
|
|
||||||
- Lint: `cd ghost-upload && cargo clippy`
|
|
||||||
- Format: `cd ghost-upload && cargo fmt`
|
|
||||||
- Julia (root or `posts/*/`):
|
|
||||||
- Packages: `julia -e "using Pkg; Pkg.instantiate()"`
|
|
||||||
- Precompile: `julia -e "using Pkg; Pkg.precompile()"`
|
|
||||||
- Run notebook/script: `julia <filename>.jl`
|
|
||||||
- Quarto (docs/site):
|
|
||||||
- Build site: `quarto render --to html --output-dir public`
|
|
||||||
- Preview: `quarto preview`
|
|
||||||
- Check: `quarto check`
|
|
||||||
- Docker: `docker build -t projects .` then `docker run projects`
|
|
||||||
|
|
||||||
## Coding Style & Naming Conventions
|
|
||||||
- Rust: `cargo fmt`; fix all `cargo clippy` warnings. Use `?` over `unwrap()`. Imports: std → external → local. Naming: snake_case (fn/vars), PascalCase (types). Public docs with `///`.
|
|
||||||
- Julia: 4-space indent; spaces around operators; group `using` at top; snake_case; prefer pipelines `|>` for DataFrames; handle expected errors with try-catch.
|
|
||||||
- Quarto: Include title/date in YAML; set `echo: false`, `warning: false` for clean outputs; descriptive figure captions and alt text.
|
|
||||||
|
|
||||||
## Testing Guidelines
|
|
||||||
- Rust: Unit tests for core logic; add integration tests for API calls. Run with `cargo test`. Organize tests near code or in `tests/`.
|
|
||||||
- Julia: Validate transformations and plots visually; keep scripts deterministic.
|
|
||||||
- Quarto: Manually review rendered HTML for links, figures, and warnings.
|
|
||||||
|
|
||||||
## Commit & Pull Request Guidelines
|
|
||||||
- Commits: Use clear, conventional messages (e.g., `feat:`, `fix:`, `docs:`). Scope small and focused.
|
|
||||||
- PRs: Provide description, linked issues, steps to validate (commands), and screenshots of rendered docs when relevant.
|
|
||||||
|
|
||||||
## Security & Configuration
|
|
||||||
- Environment variables: `kagi_api_key`, `admin_api_key`. Export locally (e.g., `export admin_api_key=...`); never commit secrets.
|
|
||||||
- Dependencies: Keep minimal and up-to-date. Prefer configuration via env vars over hardcoded values.
|
|
||||||
|
|
||||||
## CI/CD & Deployment
|
|
||||||
- GitLab CI builds Docker, renders Quarto to static hosting; Rust runs separately for content sync. Avoid pipeline changes unless necessary; include rationale in PRs if modified.
|
|
||||||
|
|
55
_quarto.yml
55
_quarto.yml
@@ -1,42 +1,25 @@
|
|||||||
project:
|
project:
|
||||||
type: website
|
type: website
|
||||||
|
|
||||||
profiles:
|
website:
|
||||||
default:
|
title: "Anson's Projects"
|
||||||
website:
|
site-url: https://projects.ansonbiggs.com
|
||||||
title: "Anson's Projects"
|
description: A Blog for Technical Topics
|
||||||
site-url: https://projects.ansonbiggs.com
|
navbar:
|
||||||
description: A Blog for Technical Topics
|
left:
|
||||||
navbar:
|
- text: "About"
|
||||||
left:
|
href: about.html
|
||||||
- text: "About"
|
right:
|
||||||
href: about.html
|
- icon: rss
|
||||||
right:
|
href: index.xml
|
||||||
- icon: rss
|
# - icon: gitlab
|
||||||
href: index.xml
|
# href: https://gitlab.com/MisterBiggs
|
||||||
# - icon: gitlab
|
open-graph: true
|
||||||
# href: https://gitlab.com/MisterBiggs
|
format:
|
||||||
open-graph: true
|
html:
|
||||||
format:
|
theme: zephyr
|
||||||
html:
|
css: styles.css
|
||||||
theme: zephyr
|
# toc: true
|
||||||
css: styles.css
|
|
||||||
# toc: true
|
|
||||||
|
|
||||||
ghost:
|
|
||||||
website:
|
|
||||||
title: "Anson's Projects"
|
|
||||||
site-url: https://projects.ansonbiggs.com
|
|
||||||
description: A Blog for Technical Topics
|
|
||||||
navbar: false
|
|
||||||
open-graph: true
|
|
||||||
format:
|
|
||||||
html:
|
|
||||||
theme: none
|
|
||||||
css: ghost-iframe.css
|
|
||||||
toc: false
|
|
||||||
page-layout: article
|
|
||||||
title-block-banner: false
|
|
||||||
|
|
||||||
execute:
|
execute:
|
||||||
freeze: true
|
freeze: true
|
129
ghost-iframe.css
129
ghost-iframe.css
@@ -1,129 +0,0 @@
|
|||||||
/* Ghost iframe optimized styles */
|
|
||||||
body {
|
|
||||||
font-family: system-ui, -apple-system, sans-serif;
|
|
||||||
line-height: 1.6;
|
|
||||||
color: #333;
|
|
||||||
max-width: 100%;
|
|
||||||
margin: 0;
|
|
||||||
padding: 20px;
|
|
||||||
background: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Remove any potential margins/padding */
|
|
||||||
html, body {
|
|
||||||
margin: 0;
|
|
||||||
padding: 0;
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Ensure content flows naturally */
|
|
||||||
#quarto-content {
|
|
||||||
max-width: none;
|
|
||||||
padding: 0;
|
|
||||||
margin: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Style headings for Ghost */
|
|
||||||
h1, h2, h3, h4, h5, h6 {
|
|
||||||
margin-top: 1.5em;
|
|
||||||
margin-bottom: 0.5em;
|
|
||||||
font-weight: 600;
|
|
||||||
line-height: 1.3;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 { font-size: 2em; }
|
|
||||||
h2 { font-size: 1.5em; }
|
|
||||||
h3 { font-size: 1.25em; }
|
|
||||||
|
|
||||||
/* Code blocks */
|
|
||||||
pre {
|
|
||||||
background: #f8f9fa;
|
|
||||||
border: 1px solid #e9ecef;
|
|
||||||
border-radius: 6px;
|
|
||||||
padding: 1rem;
|
|
||||||
overflow-x: auto;
|
|
||||||
font-size: 0.875em;
|
|
||||||
}
|
|
||||||
|
|
||||||
code {
|
|
||||||
font-family: "SF Mono", Monaco, "Cascadia Code", "Roboto Mono", Consolas, "Courier New", monospace;
|
|
||||||
background: #f1f3f4;
|
|
||||||
padding: 0.2em 0.4em;
|
|
||||||
border-radius: 3px;
|
|
||||||
font-size: 0.875em;
|
|
||||||
}
|
|
||||||
|
|
||||||
pre code {
|
|
||||||
background: none;
|
|
||||||
padding: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Images */
|
|
||||||
img {
|
|
||||||
max-width: 100%;
|
|
||||||
height: auto;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Tables */
|
|
||||||
table {
|
|
||||||
border-collapse: collapse;
|
|
||||||
width: 100%;
|
|
||||||
margin: 1em 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
th, td {
|
|
||||||
border: 1px solid #ddd;
|
|
||||||
padding: 8px;
|
|
||||||
text-align: left;
|
|
||||||
}
|
|
||||||
|
|
||||||
th {
|
|
||||||
background-color: #f2f2f2;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Links */
|
|
||||||
a {
|
|
||||||
color: #0066cc;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Blockquotes */
|
|
||||||
blockquote {
|
|
||||||
border-left: 4px solid #ddd;
|
|
||||||
margin: 1em 0;
|
|
||||||
padding-left: 1em;
|
|
||||||
color: #666;
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Lists */
|
|
||||||
ul, ol {
|
|
||||||
padding-left: 1.5em;
|
|
||||||
}
|
|
||||||
|
|
||||||
li {
|
|
||||||
margin-bottom: 0.25em;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Remove any navbar/footer elements that might leak through */
|
|
||||||
.navbar, .nav, footer, .sidebar, .toc, .page-footer {
|
|
||||||
display: none !important;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Ensure responsive behavior for iframe */
|
|
||||||
@media (max-width: 768px) {
|
|
||||||
body {
|
|
||||||
padding: 15px;
|
|
||||||
font-size: 16px;
|
|
||||||
}
|
|
||||||
|
|
||||||
h1 { font-size: 1.75em; }
|
|
||||||
h2 { font-size: 1.35em; }
|
|
||||||
h3 { font-size: 1.15em; }
|
|
||||||
}
|
|
@@ -10,26 +10,20 @@ publish:
|
|||||||
- cd ./ghost-upload
|
- cd ./ghost-upload
|
||||||
- cargo run
|
- cargo run
|
||||||
needs:
|
needs:
|
||||||
- job: deploy
|
- pages
|
||||||
optional: true
|
rules:
|
||||||
- job: staging
|
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
||||||
optional: true
|
|
||||||
|
|
||||||
# Manual trigger to force update all Ghost posts
|
publish_update:
|
||||||
force-update-ghost:
|
|
||||||
stage: deploy
|
stage: deploy
|
||||||
image: rust:latest
|
image: rust:latest
|
||||||
script:
|
|
||||||
- echo "🔄 Force updating all Ghost posts..."
|
|
||||||
- cd ./ghost-upload
|
|
||||||
- FORCE_UPDATE=true cargo run
|
|
||||||
needs:
|
|
||||||
- job: deploy
|
|
||||||
optional: true
|
|
||||||
- job: staging
|
|
||||||
optional: true
|
|
||||||
rules:
|
|
||||||
- when: manual
|
|
||||||
allow_failure: false
|
|
||||||
variables:
|
variables:
|
||||||
FORCE_UPDATE: "true"
|
UPDATE_EXISTING: "true"
|
||||||
|
script:
|
||||||
|
- cd ./ghost-upload
|
||||||
|
- cargo run
|
||||||
|
needs:
|
||||||
|
- pages
|
||||||
|
rules:
|
||||||
|
- if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH"
|
||||||
|
when: manual
|
||||||
|
1074
ghost-upload/Cargo.lock
generated
1074
ghost-upload/Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,39 +1,17 @@
|
|||||||
# ghost-upload
|
# ghost-upload
|
||||||
|
|
||||||
This tool synchronizes posts from https://projects.ansonbiggs.com to the Ghost blog at https://notes.ansonbiggs.com.
|
This tool uploads posts from https://projects.ansonbiggs.com to https://notes.ansonbiggs.com.
|
||||||
|
|
||||||
## Features
|
What's new:
|
||||||
|
- Uses the Ghost Admin API to check for existing posts by slug instead of probing the public site.
|
||||||
|
- Optional update support: set `UPDATE_EXISTING=true` to update an existing post in-place (via `PUT /ghost/api/v3/admin/posts/{id}?source=html`).
|
||||||
|
- Safer slug handling (trims trailing `/` and falls back to the last path segment).
|
||||||
|
|
||||||
- **Automatic sync**: Only uploads new posts by default
|
Env vars:
|
||||||
- **Content extraction**: Fetches clean HTML content instead of using iframes
|
- `admin_api_key`: Ghost Admin API key in `key_id:secret` format.
|
||||||
- **AI summaries**: Uses Kagi Summarizer for post summaries
|
- `kagi_api_key`: Kagi Summarizer API key.
|
||||||
- **Force update**: Manual trigger to update all existing posts
|
- `UPDATE_EXISTING` (optional): if `true`/`1`, update posts that already exist in Ghost.
|
||||||
|
|
||||||
## Usage
|
Notes:
|
||||||
|
- Updates use optimistic concurrency by sending the current `updated_at` from Ghost. If someone edits a post in Ghost after we fetch it, the update will fail with a 409 and be reported in the console.
|
||||||
### Normal Mode (Default)
|
- Summaries are always regenerated when creating or updating; if you want to avoid re-summarizing on updates, leave `UPDATE_EXISTING` unset.
|
||||||
```bash
|
|
||||||
cargo run
|
|
||||||
```
|
|
||||||
Only processes new posts that don't exist on the Ghost blog.
|
|
||||||
|
|
||||||
### Force Update Mode
|
|
||||||
```bash
|
|
||||||
FORCE_UPDATE=true cargo run
|
|
||||||
```
|
|
||||||
Updates ALL posts, including existing ones. Useful for:
|
|
||||||
- Updating content after changes
|
|
||||||
- Refreshing summaries
|
|
||||||
- Applying new styling/formatting
|
|
||||||
|
|
||||||
## CI/CD Integration
|
|
||||||
|
|
||||||
The GitLab CI pipeline includes:
|
|
||||||
- **Automatic sync**: Runs after each deployment
|
|
||||||
- **Manual force update**: Available as a manual trigger in GitLab UI
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
- `admin_api_key`: Ghost Admin API key (required)
|
|
||||||
- `kagi_api_key`: Kagi Summarizer API key (required)
|
|
||||||
- `FORCE_UPDATE`: Set to "true" to update all posts (optional)
|
|
||||||
|
@@ -1,6 +1,5 @@
|
|||||||
use feed_rs::model::Entry;
|
use feed_rs::model::Entry;
|
||||||
use feed_rs::parser;
|
use feed_rs::parser;
|
||||||
use futures::future::join_all;
|
|
||||||
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
|
use jsonwebtoken::{encode, Algorithm, EncodingKey, Header};
|
||||||
use maud::html;
|
use maud::html;
|
||||||
use reqwest::Client;
|
use reqwest::Client;
|
||||||
@@ -20,6 +19,29 @@ struct PostPayload {
|
|||||||
posts: Vec<Post>,
|
posts: Vec<Post>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Debug, Clone)]
|
||||||
|
struct UpdatePost {
|
||||||
|
id: String,
|
||||||
|
title: String,
|
||||||
|
slug: String,
|
||||||
|
html: String,
|
||||||
|
status: String,
|
||||||
|
published_at: String,
|
||||||
|
updated_at: String,
|
||||||
|
canonical_url: String,
|
||||||
|
tags: Vec<String>,
|
||||||
|
feature_image: Option<String>,
|
||||||
|
feature_image_alt: Option<String>,
|
||||||
|
feature_image_caption: Option<String>,
|
||||||
|
meta_description: Option<String>,
|
||||||
|
custom_excerpt: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Debug)]
|
||||||
|
struct UpdatePayload {
|
||||||
|
posts: Vec<UpdatePost>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Debug, Clone)]
|
#[derive(Serialize, Debug, Clone)]
|
||||||
struct Post {
|
struct Post {
|
||||||
title: String,
|
title: String,
|
||||||
@@ -45,29 +67,13 @@ impl Post {
|
|||||||
let slug = get_slug(link);
|
let slug = get_slug(link);
|
||||||
|
|
||||||
let summary = summarize_url(link).await;
|
let summary = summarize_url(link).await;
|
||||||
|
|
||||||
// Extract content from ghost-optimized version
|
|
||||||
let ghost_content = extract_article_content(&link).await;
|
|
||||||
|
|
||||||
let html = html! {
|
let html = html! {
|
||||||
div class="ghost-summary" {
|
p { (summary) }
|
||||||
h3 { "Summary" }
|
iframe src=(link) style="width: 100%; height: 80vh" { }
|
||||||
p { (summary) }
|
p {
|
||||||
}
|
"This content was originally posted on my projects website " a href=(link) { "here." }
|
||||||
div class="ghost-content" {
|
" The above summary was made by the " a href=("https://help.kagi.com/kagi/api/summarizer.html")
|
||||||
(maud::PreEscaped(ghost_content))
|
{"Kagi Summarizer"}
|
||||||
}
|
|
||||||
div class="ghost-footer" {
|
|
||||||
hr {}
|
|
||||||
p {
|
|
||||||
em {
|
|
||||||
"This content was originally posted on my projects website "
|
|
||||||
a href=(link) { "here" }
|
|
||||||
". The above summary was generated by the "
|
|
||||||
a href=("https://help.kagi.com/kagi/api/summarizer.html") {"Kagi Summarizer"}
|
|
||||||
"."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}.into_string();
|
}.into_string();
|
||||||
|
|
||||||
@@ -137,105 +143,54 @@ impl Post {
|
|||||||
meta_description,
|
meta_description,
|
||||||
custom_excerpt,
|
custom_excerpt,
|
||||||
};
|
};
|
||||||
dbg!(&x);
|
|
||||||
x
|
x
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_slug(link: &str) -> String {
|
fn get_slug(link: &str) -> String {
|
||||||
link.split_once("/posts/").unwrap().1.to_string()
|
// Prefer portion after "/posts/" if present, otherwise fall back to the last path segment
|
||||||
}
|
let raw = match link.split_once("/posts/") {
|
||||||
|
Some((_, rest)) => rest,
|
||||||
async fn extract_article_content(original_link: &str) -> String {
|
None => link.rsplit('/').next().unwrap_or(link),
|
||||||
// Convert original link to ghost-content version
|
};
|
||||||
let ghost_link = original_link.replace("projects.ansonbiggs.com", "projects.ansonbiggs.com/ghost-content");
|
raw.trim_end_matches('/')
|
||||||
|
.to_string()
|
||||||
match reqwest::get(&ghost_link).await {
|
|
||||||
Ok(response) => {
|
|
||||||
match response.text().await {
|
|
||||||
Ok(html_content) => {
|
|
||||||
let document = Html::parse_document(&html_content);
|
|
||||||
|
|
||||||
// Try different selectors to find the main content
|
|
||||||
let content_selectors = [
|
|
||||||
"#quarto-content main",
|
|
||||||
"#quarto-content",
|
|
||||||
"main",
|
|
||||||
"article",
|
|
||||||
".content",
|
|
||||||
"body"
|
|
||||||
];
|
|
||||||
|
|
||||||
for selector_str in &content_selectors {
|
|
||||||
if let Ok(selector) = Selector::parse(selector_str) {
|
|
||||||
if let Some(element) = document.select(&selector).next() {
|
|
||||||
let content = element.inner_html();
|
|
||||||
|
|
||||||
if !content.trim().is_empty() {
|
|
||||||
return content;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: return original content with iframe if extraction fails
|
|
||||||
format!(r#"<div class="fallback-iframe">
|
|
||||||
<p><em>Content extraction failed. Falling back to embedded view:</em></p>
|
|
||||||
<iframe src="{}" style="width: 100%; height: 80vh; border: none;" loading="lazy"></iframe>
|
|
||||||
</div>"#, original_link)
|
|
||||||
}
|
|
||||||
Err(_) => format!(r#"<p><em>Failed to fetch content. <a href="{}">View original post</a></em></p>"#, original_link)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => format!(r#"<p><em>Failed to fetch content. <a href="{}">View original post</a></em></p>"#, original_link)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn check_if_post_exists(entry: &Entry) -> bool {
|
|
||||||
let posts_url = "https://notes.ansonbiggs.com/";
|
|
||||||
let link = entry.links.first().unwrap().href.as_str();
|
|
||||||
let slug = get_slug(link);
|
|
||||||
|
|
||||||
match reqwest::get(format!("{}{}", posts_url, slug)).await {
|
|
||||||
Ok(response) => response.status().is_success(),
|
|
||||||
Err(_) => false,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
#[derive(Deserialize, Debug)]
|
||||||
struct GhostPostsResponse {
|
struct GhostPostSummary {
|
||||||
posts: Vec<GhostPost>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize, Debug)]
|
|
||||||
struct GhostPost {
|
|
||||||
id: String,
|
id: String,
|
||||||
slug: String,
|
slug: String,
|
||||||
|
updated_at: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_existing_post_id(slug: &str, token: &str) -> Option<String> {
|
#[derive(Deserialize, Debug)]
|
||||||
let client = Client::new();
|
struct GhostPostsResponse<T> {
|
||||||
let api_url = format!("https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/slug/{}/", slug);
|
posts: Vec<T>,
|
||||||
|
}
|
||||||
|
|
||||||
match client
|
async fn get_existing_post_by_slug(
|
||||||
.get(&api_url)
|
client: &Client,
|
||||||
|
ghost_admin_base: &str,
|
||||||
|
token: &str,
|
||||||
|
slug: &str,
|
||||||
|
) -> Option<GhostPostSummary> {
|
||||||
|
// Use Ghost Admin API to search by slug
|
||||||
|
let url = format!(
|
||||||
|
"{}/posts/?filter=slug:{}&fields=id,slug,updated_at",
|
||||||
|
ghost_admin_base, slug
|
||||||
|
);
|
||||||
|
let resp = client
|
||||||
|
.get(url)
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
{
|
.ok()?;
|
||||||
Ok(response) => {
|
if !resp.status().is_success() {
|
||||||
if response.status().is_success() {
|
return None;
|
||||||
if let Ok(ghost_response) = response.json::<GhostPostsResponse>().await {
|
|
||||||
ghost_response.posts.first().map(|post| post.id.clone())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(_) => None,
|
|
||||||
}
|
}
|
||||||
|
let json = resp.json::<GhostPostsResponse<GhostPostSummary>>().await.ok()?;
|
||||||
|
json.posts.into_iter().next()
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn fetch_feed(url: &str) -> Vec<Entry> {
|
async fn fetch_feed(url: &str) -> Vec<Entry> {
|
||||||
@@ -302,19 +257,10 @@ async fn summarize_url(url: &str) -> String {
|
|||||||
}
|
}
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
let ghost_api_url = "https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/?source=html";
|
let ghost_admin_base = "https://notes.ansonbiggs.com/ghost/api/v3/admin";
|
||||||
|
let ghost_posts_create_url = format!("{}/posts/?source=html", ghost_admin_base);
|
||||||
let ghost_admin_api_key = env::var("admin_api_key").unwrap();
|
let ghost_admin_api_key = env::var("admin_api_key").unwrap();
|
||||||
|
|
||||||
// Check if force update is enabled
|
|
||||||
let force_update = env::var("FORCE_UPDATE").unwrap_or_default() == "true";
|
|
||||||
|
|
||||||
if force_update {
|
|
||||||
println!("🔄 FORCE UPDATE MODE ENABLED");
|
|
||||||
println!(" This will update ALL posts, including existing ones.");
|
|
||||||
} else {
|
|
||||||
println!("📝 NORMAL MODE - Only publishing new posts");
|
|
||||||
}
|
|
||||||
|
|
||||||
let feed = "https://projects.ansonbiggs.com/index.xml";
|
let feed = "https://projects.ansonbiggs.com/index.xml";
|
||||||
|
|
||||||
// Split the key into ID and SECRET
|
// Split the key into ID and SECRET
|
||||||
@@ -345,84 +291,87 @@ async fn main() {
|
|||||||
)
|
)
|
||||||
.expect("JWT encoding failed");
|
.expect("JWT encoding failed");
|
||||||
|
|
||||||
|
let client = Client::new();
|
||||||
// Prepare the post data
|
// Prepare the post data
|
||||||
let entries = fetch_feed(feed).await;
|
let entries = fetch_feed(feed).await;
|
||||||
|
|
||||||
let filtered_entries: Vec<Entry> = if force_update {
|
// Control whether to update existing posts via env var
|
||||||
println!("🔄 Force update enabled - processing all {} posts", entries.len());
|
let update_existing = env::var("UPDATE_EXISTING").map(|v| v == "1" || v.eq_ignore_ascii_case("true")).unwrap_or(false);
|
||||||
entries
|
|
||||||
} else {
|
|
||||||
let post_exists_futures = entries.into_iter().map(|entry| {
|
|
||||||
let entry_clone = entry.clone();
|
|
||||||
async move { (entry_clone, check_if_post_exists(&entry).await) }
|
|
||||||
});
|
|
||||||
|
|
||||||
let post_exists_results = join_all(post_exists_futures).await;
|
for entry in entries {
|
||||||
|
let link = entry.links.first().unwrap().href.as_str();
|
||||||
|
let slug = get_slug(link);
|
||||||
|
|
||||||
let new_entries: Vec<Entry> = post_exists_results
|
let existing = get_existing_post_by_slug(&client, ghost_admin_base, &token, &slug).await;
|
||||||
.into_iter()
|
|
||||||
.filter_map(|(entry, exists)| if !exists { Some(entry) } else { None })
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
println!("📝 Found {} new posts to publish", new_entries.len());
|
match existing {
|
||||||
new_entries
|
None => {
|
||||||
};
|
// Create new post
|
||||||
|
let post = Post::new(entry.clone()).await;
|
||||||
|
let post_payload = PostPayload { posts: vec![post.clone()] };
|
||||||
|
|
||||||
if filtered_entries.is_empty() {
|
let response = client
|
||||||
println!("Nothing to post.");
|
.post(&ghost_posts_create_url)
|
||||||
return;
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
}
|
.json(&post_payload)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("Request failed");
|
||||||
|
|
||||||
let post_futures = filtered_entries.into_iter().map(Post::new);
|
if response.status().is_success() {
|
||||||
|
println!("Post {} published successfully.", post.title);
|
||||||
let client = Client::new();
|
} else {
|
||||||
|
println!(
|
||||||
for post in join_all(post_futures).await {
|
"Failed to publish post {}.\n\tStatus: {}",
|
||||||
let post_payload = PostPayload {
|
&post.title,
|
||||||
posts: vec![post.clone()],
|
response.status()
|
||||||
};
|
);
|
||||||
|
}
|
||||||
// Check if this is an update (for force_update mode)
|
|
||||||
let (method, url) = if force_update {
|
|
||||||
if let Some(existing_id) = get_existing_post_id(&post.slug, &token).await {
|
|
||||||
println!("🔄 Updating existing post: {}", post.title);
|
|
||||||
("PUT", format!("https://notes.ansonbiggs.com/ghost/api/v3/admin/posts/{}/", existing_id))
|
|
||||||
} else {
|
|
||||||
println!("📝 Creating new post: {}", post.title);
|
|
||||||
("POST", ghost_api_url.to_string())
|
|
||||||
}
|
}
|
||||||
} else {
|
Some(summary) => {
|
||||||
println!("📝 Creating new post: {}", post.title);
|
if !update_existing {
|
||||||
("POST", ghost_api_url.to_string())
|
println!("Post '{}' exists (slug: {}), skipping.", entry.title.unwrap().content, slug);
|
||||||
};
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let response = match method {
|
// Update existing post
|
||||||
"PUT" => client
|
let post = Post::new(entry.clone()).await;
|
||||||
.put(&url)
|
let update = UpdatePost {
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
id: summary.id,
|
||||||
.json(&post_payload)
|
title: post.title,
|
||||||
.send()
|
slug: post.slug,
|
||||||
.await
|
html: post.html,
|
||||||
.expect("Request failed"),
|
status: post.status,
|
||||||
_ => client
|
published_at: post.published_at,
|
||||||
.post(&url)
|
updated_at: summary.updated_at,
|
||||||
.header("Authorization", format!("Ghost {}", token))
|
canonical_url: post.canonical_url,
|
||||||
.json(&post_payload)
|
tags: post.tags,
|
||||||
.send()
|
feature_image: post.feature_image,
|
||||||
.await
|
feature_image_alt: post.feature_image_alt,
|
||||||
.expect("Request failed"),
|
feature_image_caption: post.feature_image_caption,
|
||||||
};
|
meta_description: post.meta_description,
|
||||||
|
custom_excerpt: post.custom_excerpt,
|
||||||
|
};
|
||||||
|
|
||||||
// Check the response
|
let update_url = format!("{}/posts/{}/?source=html", ghost_admin_base, update.id);
|
||||||
if response.status().is_success() {
|
let response = client
|
||||||
let action = if method == "PUT" { "updated" } else { "published" };
|
.put(update_url)
|
||||||
println!("✅ Post '{}' {} successfully.", post.title, action);
|
.header("Authorization", format!("Ghost {}", token))
|
||||||
} else {
|
.json(&UpdatePayload { posts: vec![update] })
|
||||||
let action = if method == "PUT" { "update" } else { "publish" };
|
.send()
|
||||||
println!(
|
.await
|
||||||
"❌ Failed to {} post '{}'.\n\tStatus: {}\n\tResponse: {:?}",
|
.expect("Update request failed");
|
||||||
action, &post.title, response.status(), response.text().await.unwrap_or_default()
|
|
||||||
);
|
if response.status().is_success() {
|
||||||
|
println!("Post '{}' updated successfully.", entry.title.unwrap().content);
|
||||||
|
} else {
|
||||||
|
println!(
|
||||||
|
"Failed to update post '{}' (status: {}).",
|
||||||
|
entry.title.unwrap().content,
|
||||||
|
response.status()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -1,34 +0,0 @@
|
|||||||
# Test Ghost Profile Output
|
|
||||||
|
|
||||||
This is a test document to validate our ghost profile setup.
|
|
||||||
|
|
||||||
## Content Structure
|
|
||||||
|
|
||||||
The ghost profile should:
|
|
||||||
- Remove navigation elements
|
|
||||||
- Use minimal styling from ghost-iframe.css
|
|
||||||
- Maintain clean article layout
|
|
||||||
- Remove table of contents
|
|
||||||
|
|
||||||
## Code Example
|
|
||||||
|
|
||||||
```julia
|
|
||||||
println("Hello from Julia!")
|
|
||||||
x = 1 + 1
|
|
||||||
```
|
|
||||||
|
|
||||||
## Regular Content
|
|
||||||
|
|
||||||
This is just some regular markdown content to see how it renders in the ghost profile.
|
|
||||||
|
|
||||||
- List item 1
|
|
||||||
- List item 2
|
|
||||||
- List item 3
|
|
||||||
|
|
||||||
**Bold text** and *italic text* should render properly.
|
|
||||||
|
|
||||||
[Link to main site](https://projects.ansonbiggs.com)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
If you can see clean, minimal styling without navigation, the ghost profile is working correctly.
|
|
@@ -1,55 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
echo "🧪 Testing local deployment simulation..."
|
|
||||||
|
|
||||||
# Create test directories
|
|
||||||
mkdir -p test-output/main
|
|
||||||
mkdir -p test-output/ghost-content
|
|
||||||
|
|
||||||
echo "📁 Simulating dual-output build..."
|
|
||||||
|
|
||||||
# Test 1: Check if ghost profile exists
|
|
||||||
if grep -q "ghost:" _quarto.yml; then
|
|
||||||
echo "✅ Ghost profile configuration found"
|
|
||||||
else
|
|
||||||
echo "❌ Ghost profile not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Test 2: Simulate content extraction
|
|
||||||
echo "🔍 Testing content extraction logic..."
|
|
||||||
cd ghost-upload
|
|
||||||
|
|
||||||
# Test with sample URL (without actually hitting network)
|
|
||||||
echo "📝 Testing Rust compilation and basic logic..."
|
|
||||||
if cargo check --quiet; then
|
|
||||||
echo "✅ Rust code compiles successfully"
|
|
||||||
else
|
|
||||||
echo "❌ Rust compilation failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd ..
|
|
||||||
|
|
||||||
# Test 3: Check if CI would work
|
|
||||||
echo "🔧 Validating CI configuration..."
|
|
||||||
if ./validate-ghost-extraction.sh > /dev/null 2>&1; then
|
|
||||||
echo "✅ CI validation passed"
|
|
||||||
else
|
|
||||||
echo "❌ CI validation failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "🎉 Local testing complete!"
|
|
||||||
echo ""
|
|
||||||
echo "📋 What happens in CI:"
|
|
||||||
echo " 1. Builds main site → public/"
|
|
||||||
echo " 2. Builds ghost content → public/ghost-content/"
|
|
||||||
echo " 3. Rust extracts from ghost-content URLs"
|
|
||||||
echo " 4. Posts to Ghost blog with clean HTML"
|
|
||||||
echo ""
|
|
||||||
echo "🚀 Ready for branch testing in GitLab CI!"
|
|
||||||
echo " • Download artifacts to see both outputs"
|
|
||||||
echo " • Use manual trigger to test force-update"
|
|
||||||
echo " • Check ghost-content/ folder structure"
|
|
@@ -1,89 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Simple validation script for ghost content extraction
|
|
||||||
echo "🔍 Validating ghost profile implementation..."
|
|
||||||
|
|
||||||
# Check if required files exist
|
|
||||||
echo "📁 Checking required files..."
|
|
||||||
if [ ! -f "_quarto.yml" ]; then
|
|
||||||
echo "❌ _quarto.yml not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f "ghost-iframe.css" ]; then
|
|
||||||
echo "❌ ghost-iframe.css not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ ! -f "ghost-upload/src/main.rs" ]; then
|
|
||||||
echo "❌ Rust source not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "✅ All required files present"
|
|
||||||
|
|
||||||
# Check if ghost profile is defined in _quarto.yml
|
|
||||||
echo "📋 Checking ghost profile configuration..."
|
|
||||||
if grep -q "ghost:" _quarto.yml; then
|
|
||||||
echo "✅ Ghost profile found in _quarto.yml"
|
|
||||||
else
|
|
||||||
echo "❌ Ghost profile not found in _quarto.yml"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if GitLab CI builds both versions
|
|
||||||
echo "🔧 Checking GitLab CI configuration..."
|
|
||||||
if grep -q "ghost-content" .gitlab-ci.yml; then
|
|
||||||
echo "✅ GitLab CI configured for dual output"
|
|
||||||
else
|
|
||||||
echo "❌ GitLab CI not configured for ghost-content"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if Rust code has extract_article_content function
|
|
||||||
echo "🦀 Checking Rust implementation..."
|
|
||||||
if grep -q "extract_article_content" ghost-upload/src/main.rs; then
|
|
||||||
echo "✅ Content extraction function found"
|
|
||||||
else
|
|
||||||
echo "❌ Content extraction function not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if force update functionality is available
|
|
||||||
if grep -q "FORCE_UPDATE" ghost-upload/src/main.rs; then
|
|
||||||
echo "✅ Force update functionality found"
|
|
||||||
else
|
|
||||||
echo "❌ Force update functionality not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Check if manual CI job is configured
|
|
||||||
if grep -q "force-update-ghost" ghost-upload/.gitlab-ci.yml; then
|
|
||||||
echo "✅ Manual force update CI job found"
|
|
||||||
else
|
|
||||||
echo "❌ Manual force update CI job not found"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Verify Rust code compiles
|
|
||||||
echo "🛠️ Building Rust code..."
|
|
||||||
cd ghost-upload
|
|
||||||
if cargo check --quiet; then
|
|
||||||
echo "✅ Rust code compiles successfully"
|
|
||||||
else
|
|
||||||
echo "❌ Rust compilation failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
cd ..
|
|
||||||
echo ""
|
|
||||||
echo "🎉 All validations passed!"
|
|
||||||
echo "📋 Summary of changes:"
|
|
||||||
echo " • Quarto profiles for dual-output rendering"
|
|
||||||
echo " • Ghost-optimized CSS styling"
|
|
||||||
echo " • GitLab CI builds both main site and ghost-content"
|
|
||||||
echo " • Rust extracts HTML content instead of using iframes"
|
|
||||||
echo " • Force update mode to refresh existing posts"
|
|
||||||
echo " • Manual CI trigger for content updates"
|
|
||||||
echo ""
|
|
||||||
echo "🚀 Ready for testing in CI/CD pipeline!"
|
|
Reference in New Issue
Block a user