fix site upload rule

Fix sec.gov permissions issue

See merge request simple-stock-bots/simple-stock-bot!63

.gitlab-ci.yml

@@ -1,9 +1,7 @@
 stages:
   - lint
-  - build
   - build_site
   - deploy
-  - deploy_site
 
 black:
   stage: lint

common/MarketData.py

@@ -1,8 +1,8 @@
 import datetime as dt
 import logging
 import os
-from typing import Dict
 from collections import OrderedDict
+from typing import Dict
 
 import humanize
 import pandas as pd
@@ -54,9 +54,12 @@ class MarketData:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
-    def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
+    def get(self, endpoint, params=None, timeout=10, headers=None) -> dict:
         url = "https://api.marketdata.app/v1/" + endpoint
 
+        if params is None:
+            params = {}
+
         # set token param if it wasn't passed.
         params["token"] = self.MARKETDATA_TOKEN
 
@@ -64,7 +67,13 @@ class MarketData:
         # monitored even if someone doesn't make it through an affiliate link.
         params["application"] = "simplestockbot"
 
-        resp = r.get(url, params=params, timeout=timeout)
+        if headers is None:
+            headers = {}
+        headers = {"User-Agent": "Simple Stock Bot anson@ansonbiggs.com"} | headers
+
+        resp = r.get(url, params=params, timeout=timeout, headers=headers)
+
+        logging.error(resp.headers.items())
 
         # Make sure API returned a proper status code
         try:
@@ -95,7 +104,15 @@ class MarketData:
         return self.symbol_list.get(symbol.upper(), None)
 
     def get_symbol_list(self):
-        sec_resp = r.get("https://www.sec.gov/files/company_tickers.json")
+        # Doesn't use `self.get`` since needs are much different
+        sec_resp = r.get(
+            "https://www.sec.gov/files/company_tickers.json",
+            headers={
+                "User-Agent": "Simple Stock Bot anson@ansonbiggs.com",
+                "Accept-Encoding": "gzip, deflate",
+                "Host": "www.sec.gov",
+            },
+        )
         sec_resp.raise_for_status()
         sec_data = sec_resp.json()
 

common/cg_Crypto.py

@@ -7,6 +7,9 @@ import schedule
 from markdownify import markdownify
 
 from common.Symbol import Coin
+from common.utilities import rate_limited
+
+import time
 
 log = logging.getLogger(__name__)
 
@@ -24,10 +27,21 @@ class cg_Crypto:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
+    # Coingecko's rate limit is 30 requests per minute.
+    # Since there are two bots sharing the same IP, we allocate half of that limit to each bot.
+    # This results in a rate limit of 15 requests per minute for each bot.
+    # Given this, the rate limit effectively becomes 1 request every 4 seconds for each bot.
+    @rate_limited(0.25)
     def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
         url = "https://api.coingecko.com/api/v3" + endpoint
         resp = r.get(url, params=params, timeout=timeout)
         # Make sure API returned a proper status code
+
+        if resp.status_code == 429:
+            log.warning(f"CoinGecko returned 429 - Too Many Requests for endpoint: {endpoint}. Sleeping and trying again.")
+            time.sleep(10)
+            return self.get(endpoint=endpoint, params=params, timeout=timeout)
+
         try:
             resp.raise_for_status()
         except r.exceptions.HTTPError as e:

common/requirements.txt

@@ -4,4 +4,5 @@ markdownify==0.11.6
 mplfinance==0.12.10b0
 pandas==2.1.1
 requests==2.31.0
-schedule==1.2.1
+rush==2021.4.0
+schedule==1.2.1

common/utilities.py

@@ -0,0 +1,31 @@
+import time
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def rate_limited(max_per_second):
+    """
+    Decorator that ensures the wrapped function is called at most `max_per_second` times per second.
+    """
+    min_interval = 1.0 / max_per_second
+
+    def decorate(func):
+        last_called = [0.0]
+
+        def rate_limited_function(*args, **kwargs):
+            elapsed = time.time() - last_called[0]
+            left_to_wait = min_interval - elapsed
+
+            if left_to_wait > 0:
+                log.info(f"Rate limit exceeded. Waiting for {left_to_wait:.2f} seconds.")
+                time.sleep(left_to_wait)
+
+            ret = func(*args, **kwargs)
+            last_called[0] = time.time()
+
+            return ret
+
+        return rate_limited_function
+
+    return decorate

site/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 image: python:3.11
 
-build_site:
+build_mkdocs:
-  stage: build
+  stage: build_site
   script:
     - cd ./site
     - pip install -r requirements.txt
@@ -10,12 +10,12 @@ build_site:
     paths:
       - public
 
-deploy_site:
+pages:
   stage: deploy
   script:
     - echo "Publishing site..."
   dependencies:
-    - build_site
+    - build_mkdocs
   artifacts:
     paths:
       - public

site/docs/blog/posts/intro.md

@@ -25,6 +25,8 @@ Simple Stock Bot is a chatbot designed to enrich your financial discussions on T
     </div>
 </div>
 
+<!-- more -->
+
 With Simple Stock Bot, you can:
 
 - **Fetch Real-time Quotes**: Obtain the latest stock and cryptocurrency prices instantly within your group chat.

site/docs/blog/posts/marketdata-release.md

@@ -0,0 +1,40 @@
+---
+title: "v2023.1 Release! New Stock Market Data Provider and More!"
+date: 2023-10-16
+tags: [Simple Stock Bot, Telegram, Discord, MarketDataApp]
+authors: [Anson]
+description: >
+  Discover the latest updates for Simple Stock Bot, including our new integration with MarketData.app for enhanced real-time stock market insights.
+---
+
+## 🌐 General Updates:
+
+- **New Home**: We've transitioned to our fresh and updated website at [simplestockbot.com](https://simplestockbot.com/).
+- **New Data Provider**: We're excited to announce [MarketData.app](https://dashboard.marketdata.app/marketdata/aff/go/misterbiggs?keyword=web) as our new provider for real-time stock market data, ensuring timely and accurate insights for our users.
+
+<!-- more -->
+
+## 📈 MarketData.app Integration Enhancements:
+
+- We've introduced **additional options** to our MarketData.app integration, ensuring even more precise and varied financial data.
+- **Afterhours Data Fix**: Addressed an issue where after hours stock market data caused errors.
+- **Trending Symbol Accuracy**: Fixed a bug that led to the display of invalid symbols in the `/trending` command.
+
+## 🤖 Bot Improvements & Fixes:
+
+- **Unified Repository**: To streamline our development and deployment, we've merged the Discord and Telegram bots into a single monorepo. Check it out on [GitLab](https://gitlab.com/simple-stock-bots/simple-stock-bot). Contributions welcome!
+- **Inline Functionality Restoration**: Fixed the telegram bot's inline functionality.
+- **Python Telegram Bot Update**: Migrated to the latest version of `Python Telegram Bot` for superior performance and more features.
+- **Rate Limiting Addition**: Implemented rate limiting to ensure optimal performance during peak usage times.
+- **SO MUCH MORE**: [Move to Marketdata.app GitLab Issues](https://gitlab.com/simple-stock-bots/simple-stock-bot/-/milestones/3)
+
+## 📝 Documentation & Repository Overhauls:
+
+- **Centralized Documentation**: For convenience and improved maintenance, we've shifted our documentation into the monorepo.
+- **Documentation Refinement**: Updated our documentation to reflect the latest homepage details.
+
+**Special Mention**: Immense gratitude to our dedicated community for their continual feedback and unwavering support. Dive deeper into the financial realm with Simple Stock Bot!
+
+---
+
+📥 For any concerns, queries, or feedback, don't hesitate to [Contact Us](../../contact.md).

telegram/bot.py

@@ -11,27 +11,21 @@ import traceback
 from uuid import uuid4
 
 import mplfinance as mpf
-import telegram
+from T_info import T_info
-from telegram import (
-    InlineQueryResultArticle,
-    InputTextMessageContent,
-    LabeledPrice,
-    Update,
-)
 
+import telegram
+from common.symbol_router import Router
+from telegram import InlineQueryResultArticle, InputTextMessageContent, LabeledPrice, Update
 from telegram.ext import (
     Application,
     CommandHandler,
-    InlineQueryHandler,
-    PreCheckoutQueryHandler,
-    MessageHandler,
-    filters,
     ContextTypes,
+    InlineQueryHandler,
+    MessageHandler,
+    PreCheckoutQueryHandler,
+    filters,
 )
 
-from common.symbol_router import Router
-from T_info import T_info
-
 # Enable logging
 logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO)