update reqs

fix site upload rule

See merge request simple-stock-bots/simple-stock-bot!64

.devcontainer/devcontainer.json

@@ -27,7 +27,7 @@
       ]
     }
   },
-  "postCreateCommand": "pip3 install --user -r dev-reqs.txt && apt-get update && apt-get install -y nodejs npm && npm install"
+  "postCreateCommand": "pip3 install --user -r dev-reqs.txt && apt-get update && apt-get install -y nodejs npm --fix-missing && npm install"
   // Features to add to the dev container. More info: https://containers.dev/features.
   // "features": {},
   // Use 'forwardPorts' to make a list of ports inside the container available locally.

.gitlab-ci.yml

@@ -1,29 +1,21 @@
 stages:
   - lint
-  - build
   - build_site
   - deploy
-  - deploy_site
 
-black:
-  stage: lint
-  image: registry.gitlab.com/pipeline-components/black:latest
-  script:
-    - black --check --verbose -- .
+# ruff:
+#   stage: lint
+#   image: python:3.11-slim
+#   script:
+#     - pip3 install ruff
+#     - ruff . --output-format gitlab; ruff format . --diff
 
-ruff:
-  stage: lint
-  image: python:3.11-slim
-  script:
-    - pip3 install ruff
-    - ruff --output-format gitlab .
-
-prettier:
-  stage: lint
-  image: node:16-slim # Use Node.js image since prettier is a Node.js tool
-  script:
-    - npm install prettier
-    - npx prettier --check . # Adjust the path as needed
+# prettier:
+#   stage: lint
+#   image: node:16-slim # Use Node.js image since prettier is a Node.js tool
+#   script:
+#     - npm install prettier
+#     - npx prettier --check . # Adjust the path as needed
 
 include:
   - local: /site/.gitlab-ci.yml

.vscode/settings.json

@@ -3,7 +3,7 @@
   "editor.formatOnPaste": true,
   "editor.formatOnSaveMode": "modificationsIfAvailable",
   "[python]": {
-    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.defaultFormatter": "charliermarsh.ruff",
     "editor.formatOnSave": true,
     "editor.formatOnSaveMode": "file"
   }

common/MarketData.py

@@ -1,8 +1,8 @@
 import datetime as dt
 import logging
 import os
-from typing import Dict
 from collections import OrderedDict
+from typing import Dict
 
 import humanize
 import pandas as pd
@@ -10,6 +10,7 @@ import pytz
 import requests as r
 import schedule
 
+
 from common.Symbol import Stock
 
 log = logging.getLogger(__name__)
@@ -54,9 +55,12 @@ class MarketData:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
-    def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
+    def get(self, endpoint, params=None, timeout=10, headers=None) -> dict:
         url = "https://api.marketdata.app/v1/" + endpoint
 
+        if params is None:
+            params = {}
+
         # set token param if it wasn't passed.
         params["token"] = self.MARKETDATA_TOKEN
 
@@ -64,7 +68,13 @@ class MarketData:
         # monitored even if someone doesn't make it through an affiliate link.
         params["application"] = "simplestockbot"
 
-        resp = r.get(url, params=params, timeout=timeout)
+        if headers is None:
+            headers = {}
+        headers = {"User-Agent": "Simple Stock Bot anson@ansonbiggs.com"} | headers
+
+        resp = r.get(url, params=params, timeout=timeout, headers=headers)
+
+        logging.error(resp.headers.items())
 
         # Make sure API returned a proper status code
         try:
@@ -95,7 +105,15 @@ class MarketData:
         return self.symbol_list.get(symbol.upper(), None)
 
     def get_symbol_list(self):
-        sec_resp = r.get("https://www.sec.gov/files/company_tickers.json")
+        # Doesn't use `self.get()` since needs are much different
+        sec_resp = r.get(
+            "https://www.sec.gov/files/company_tickers.json",
+            headers={
+                "User-Agent": "Simple Stock Bot anson@ansonbiggs.com",
+                "Accept-Encoding": "gzip, deflate",
+                "Host": "www.sec.gov",
+            },
+        )
         sec_resp.raise_for_status()
         sec_data = sec_resp.json()
 
@@ -212,7 +230,11 @@ class MarketData:
 
         if data := self.get(
             f"stocks/candles/{resolution}/{symbol}",
-            params={"from": startTime.timestamp(), "to": now.timestamp(), "extended": True},
+            params={
+                "from": startTime.timestamp(),
+                "to": now.timestamp(),
+                "extended": True,
+            },
         ):
             data.pop("s")
             df = pd.DataFrame(data)

common/Symbol.py

@@ -26,6 +26,9 @@ class Symbol:
     def __str__(self) -> str:
         return self.id
 
+    def __hash__(self):
+        return hash(self.id)
+
 
 class Stock(Symbol):
     """Stock Market Object. Gets data from MarketData"""

common/cg_Crypto.py

@@ -5,8 +5,10 @@ import pandas as pd
 import requests as r
 import schedule
 from markdownify import markdownify
-
 from common.Symbol import Coin
+from common.utilities import rate_limited
+
+import time
 
 log = logging.getLogger(__name__)
 
@@ -24,10 +26,21 @@ class cg_Crypto:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
+    # Coingecko's rate limit is 30 requests per minute.
+    # Since there are two bots sharing the same IP, we allocate half of that limit to each bot.
+    # This results in a rate limit of 15 requests per minute for each bot.
+    # Given this, the rate limit effectively becomes 1 request every 4 seconds for each bot.
+    @rate_limited(0.25)
     def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
         url = "https://api.coingecko.com/api/v3" + endpoint
         resp = r.get(url, params=params, timeout=timeout)
         # Make sure API returned a proper status code
+
+        if resp.status_code == 429:
+            log.warning(f"CoinGecko returned 429 - Too Many Requests for endpoint: {endpoint}. Sleeping and trying again.")
+            time.sleep(10)
+            return self.get(endpoint=endpoint, params=params, timeout=timeout)
+
         try:
             resp.raise_for_status()
         except r.exceptions.HTTPError as e:

common/requirements.txt

@@ -4,4 +4,5 @@ markdownify==0.11.6
 mplfinance==0.12.10b0
 pandas==2.1.1
 requests==2.31.0
+rush==2021.4.0
 schedule==1.2.1

common/utilities.py

@@ -0,0 +1,31 @@
+import time
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def rate_limited(max_per_second):
+    """
+    Decorator that ensures the wrapped function is called at most `max_per_second` times per second.
+    """
+    min_interval = 1.0 / max_per_second
+
+    def decorate(func):
+        last_called = [0.0]
+
+        def rate_limited_function(*args, **kwargs):
+            elapsed = time.time() - last_called[0]
+            left_to_wait = min_interval - elapsed
+
+            if left_to_wait > 0:
+                log.info(f"Rate limit exceeded. Waiting for {left_to_wait:.2f} seconds.")
+                time.sleep(left_to_wait)
+
+            ret = func(*args, **kwargs)
+            last_called[0] = time.time()
+
+            return ret
+
+        return rate_limited_function
+
+    return decorate

dev-reqs.txt

@@ -1,6 +1,5 @@
 -r common/requirements.txt
 -r site/requirements.txt
-black==23.9.1
 ipython==8.16.1
 jupyter_client==8.4.0
 jupyter_core==5.4.0
@@ -8,5 +7,4 @@ pylama==8.4.1
 mypy==1.5.1
 types-cachetools==5.3.0.6
 types-pytz==2023.3.1.1
-ruff==0.0.292
-isort==5.12.0
+ruff==0.1.6

pyproject.toml

@@ -1,11 +1,2 @@
-[tool.black]
-line-length = 130
-
-[tool.flake8]
-max-line-length = 130
-
-[tool.pycodestyle]
-max_line_length = 130
-
 [tool.ruff]
 line-length = 130

site/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 image: python:3.11
 
-build_site:
-  stage: build
+build_mkdocs:
+  stage: build_site
   script:
     - cd ./site
     - pip install -r requirements.txt
@@ -10,12 +10,12 @@ build_site:
     paths:
       - public
 
-deploy_site:
+pages:
   stage: deploy
   script:
     - echo "Publishing site..."
   dependencies:
-    - build_site
+    - build_mkdocs
   artifacts:
     paths:
       - public

site/docs/blog/posts/intro.md

@@ -25,6 +25,8 @@ Simple Stock Bot is a chatbot designed to enrich your financial discussions on T
     </div>
 </div>
 
+<!-- more -->
+
 With Simple Stock Bot, you can:
 
 - **Fetch Real-time Quotes**: Obtain the latest stock and cryptocurrency prices instantly within your group chat.

site/docs/blog/posts/marketdata-release.md

@@ -0,0 +1,40 @@
+---
+title: "v2023.1 Release! New Stock Market Data Provider and More!"
+date: 2023-10-16
+tags: [Simple Stock Bot, Telegram, Discord, MarketDataApp]
+authors: [Anson]
+description: >
+  Discover the latest updates for Simple Stock Bot, including our new integration with MarketData.app for enhanced real-time stock market insights.
+---
+
+## 🌐 General Updates:
+
+- **New Home**: We've transitioned to our fresh and updated website at [simplestockbot.com](https://simplestockbot.com/).
+- **New Data Provider**: We're excited to announce [MarketData.app](https://dashboard.marketdata.app/marketdata/aff/go/misterbiggs?keyword=web) as our new provider for real-time stock market data, ensuring timely and accurate insights for our users.
+
+<!-- more -->
+
+## 📈 MarketData.app Integration Enhancements:
+
+- We've introduced **additional options** to our MarketData.app integration, ensuring even more precise and varied financial data.
+- **Afterhours Data Fix**: Addressed an issue where after hours stock market data caused errors.
+- **Trending Symbol Accuracy**: Fixed a bug that led to the display of invalid symbols in the `/trending` command.
+
+## 🤖 Bot Improvements & Fixes:
+
+- **Unified Repository**: To streamline our development and deployment, we've merged the Discord and Telegram bots into a single monorepo. Check it out on [GitLab](https://gitlab.com/simple-stock-bots/simple-stock-bot). Contributions welcome!
+- **Inline Functionality Restoration**: Fixed the telegram bot's inline functionality.
+- **Python Telegram Bot Update**: Migrated to the latest version of `Python Telegram Bot` for superior performance and more features.
+- **Rate Limiting Addition**: Implemented rate limiting to ensure optimal performance during peak usage times.
+- **SO MUCH MORE**: [Move to Marketdata.app GitLab Issues](https://gitlab.com/simple-stock-bots/simple-stock-bot/-/milestones/3)
+
+## 📝 Documentation & Repository Overhauls:
+
+- **Centralized Documentation**: For convenience and improved maintenance, we've shifted our documentation into the monorepo.
+- **Documentation Refinement**: Updated our documentation to reflect the latest homepage details.
+
+**Special Mention**: Immense gratitude to our dedicated community for their continual feedback and unwavering support. Dive deeper into the financial realm with Simple Stock Bot!
+
+---
+
+📥 For any concerns, queries, or feedback, don't hesitate to [Contact Us](../../contact.md).

site/docs/blog/posts/sunset.md

@@ -0,0 +1,14 @@
+---
+title: "Simple Bot End-of-Life"
+date: 2024-05-12
+tags: [Simple Stock Bot, Introduction, Telegram, Discord, Financial Insights]
+authors: [Anson]
+description: >
+  Simple Stock Bot is being sunset. 
+---
+
+## What is Simple Stock Bot?
+
+For full details see my full blog post: https://notes.ansonbiggs.com/simple-stock-bot-end-of-life/
+
+I am shutting down Simple Stock Bot, a popular Telegram and Discord bot that provided live stock and cryptocurrency market data. The bot was created in 2018 after my group chat lost access to the Google Allo, which had integrated market data. The bot grew to 15,000 monthly active users, but maintaining the service has become challenging due to increasing data costs and changes to the Discord API. I really appreciate the community and everyone that has donated along the way, this has been a seriously great ride. The bot will soon no longer function on Telegram and Discord.

site/mkdocs.yml

@@ -11,12 +11,13 @@ nav:
   - Home: index.md
   - Commands: commands.md
   - Self-Host: host.md
-  - Donate: donate.md
+  # - Donate: donate.md
   - Contact: contact.md
   - Blog: blog/index.md
 
 theme:
   name: material
+  custom_dir: overrides
   language: en
   features:
     - navigation.instant
@@ -65,8 +66,8 @@ plugins:
 markdown_extensions:
   - attr_list
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - admonition
   - pymdownx.details
   - pymdownx.superfences

site/overrides/main.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+
+<!-- Announcement bar -->
+{% block announce %}
+
+<div class="admonition danger" style="min-height: 20vh;">
+  <p class="admonition-title">This project is now archived!</p>
+  <p>
+    Please
+    <a href="/blog/2024/05/12/simple-bot-end-of-life/"
+      >click here for more information.</a
+    >
+  </p>
+</div>
+<div class="" admoniton></div>
+
+{% endblock %}

site/requirements.txt

@@ -1,4 +1,5 @@
-mkdocs-material==9.4.4
+mkdocs-material==9.5.22
+mkdocs-material-extensions==1.3.1
 
 # Required for Social Cards
 Pillow==10.0.1

telegram/bot.py

@@ -11,27 +11,21 @@ import traceback
 from uuid import uuid4
 
 import mplfinance as mpf
-import telegram
-from telegram import (
-    InlineQueryResultArticle,
-    InputTextMessageContent,
-    LabeledPrice,
-    Update,
-)
+from T_info import T_info
 
+import telegram
+from common.symbol_router import Router
+from telegram import InlineQueryResultArticle, InputTextMessageContent, LabeledPrice, Update
 from telegram.ext import (
     Application,
     CommandHandler,
-    InlineQueryHandler,
-    PreCheckoutQueryHandler,
-    MessageHandler,
-    filters,
     ContextTypes,
+    InlineQueryHandler,
+    MessageHandler,
+    PreCheckoutQueryHandler,
+    filters,
 )
 
-from common.symbol_router import Router
-from T_info import T_info
-
 # Enable logging
 logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO)
 

tests.py

@@ -9,9 +9,7 @@ $tsla
 /chart $tsla
 /chart $$btc
 /help
-/trending""".split(
-    "\n"
-)
+/trending""".split("\n")
 
 print("press enter to start")
 keyboard.wait("enter")