Keep only first instance of each blog

src/main.rs

@@ -56,6 +56,8 @@ fn main() -> Result<(), Box<dyn Error>> {
         posts.insert(0, post_with_image);
     }
 
+    utilities::retain_first_main_url(&mut posts);
+
     posts.truncate(16);
 
     let mut old_posts = all_posts;

src/utilities.rs

@@ -3,10 +3,12 @@ use chrono::{DateTime, Utc};
 use feed_rs::model::Entry;
 use rayon::prelude::*;
 use scraper::{Html, Selector};
+use std::collections::HashSet;
 
 use anyhow::Result;
 use std::cmp::Ordering;
 use std::fs;
+
 #[derive(Clone, PartialEq, Eq)]
 pub struct Post {
     pub title: String,
@@ -246,3 +248,8 @@ pub fn validate(post: &mut Post) {
         println!("{} is not valid", post.link.as_str());
     };
 }
+
+pub fn retain_first_main_url(posts: &mut Vec<Post>) {
+    let mut seen_urls = HashSet::new();
+    posts.retain(|post| seen_urls.insert(post.main_url.clone()));
+}