implement rate limiting

common/MarketData.py

@@ -1,8 +1,8 @@
 import datetime as dt
 import logging
 import os
-from typing import Dict
 from collections import OrderedDict
+from typing import Dict
 
 import humanize
 import pandas as pd

common/cg_Crypto.py

@@ -7,6 +7,7 @@ import schedule
 from markdownify import markdownify
 
 from common.Symbol import Coin
+from common.utilities import rate_limited
 
 log = logging.getLogger(__name__)
 
@@ -24,6 +25,11 @@ class cg_Crypto:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
+    # Coingecko's rate limit is 30 requests per minute.
+    # Since there are two bots sharing the same IP, we allocate half of that limit to each bot.
+    # This results in a rate limit of 15 requests per minute for each bot.
+    # Given this, the rate limit effectively becomes 1 request every 4 seconds for each bot.
+    @rate_limited(0.25)
     def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
         url = "https://api.coingecko.com/api/v3" + endpoint
         resp = r.get(url, params=params, timeout=timeout)

common/requirements.txt

@@ -4,4 +4,5 @@ markdownify==0.11.6
 mplfinance==0.12.10b0
 pandas==2.1.1
 requests==2.31.0
-schedule==1.2.1
+rush==2021.4.0
+schedule==1.2.1

common/utilities.py

@@ -0,0 +1,31 @@
+import logging
+import time
+
+log = logging.getLogger(__name__)
+
+
+def rate_limited(max_per_second):
+    """
+    Decorator that ensures the wrapped function is called at most `max_per_second` times per second.
+    """
+    min_interval = 1.0 / max_per_second
+
+    def decorate(func):
+        last_called = [0.0]
+
+        def rate_limited_function(*args, **kwargs):
+            elapsed = time.time() - last_called[0]
+            left_to_wait = min_interval - elapsed
+
+            if left_to_wait > 0:
+                log.info(f"Rate limit exceeded. Waiting for {left_to_wait:.2f} seconds.")
+                time.sleep(left_to_wait)
+
+            ret = func(*args, **kwargs)
+            last_called[0] = time.time()
+
+            return ret
+
+        return rate_limited_function
+
+    return decorate

telegram/bot.py

@@ -11,27 +11,16 @@ import traceback
 from uuid import uuid4
 
 import mplfinance as mpf
-import telegram
-from telegram import (
-    InlineQueryResultArticle,
-    InputTextMessageContent,
-    LabeledPrice,
-    Update,
-)
-
-from telegram.ext import (
-    Application,
-    CommandHandler,
-    InlineQueryHandler,
-    PreCheckoutQueryHandler,
-    MessageHandler,
-    filters,
-    ContextTypes,
-)
-
-from common.symbol_router import Router
 from T_info import T_info
 
+import telegram
+from common.symbol_router import Router
+from telegram import (InlineQueryResultArticle, InputTextMessageContent,
+                      LabeledPrice, Update)
+from telegram.ext import (Application, CommandHandler, ContextTypes,
+                          InlineQueryHandler, MessageHandler,
+                          PreCheckoutQueryHandler, filters)
+
 # Enable logging
 logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO)