fix site upload rule

Fix sec.gov permissions issue

See merge request simple-stock-bots/simple-stock-bot!63

.gitlab-ci.yml

@@ -1,9 +1,7 @@
 stages:
   - lint
-  - build
   - build_site
   - deploy
-  - deploy_site
 
 black:
   stage: lint

common/MarketData.py

@@ -54,9 +54,12 @@ class MarketData:
         self.get_symbol_list()
         schedule.every().day.do(self.get_symbol_list)
 
-    def get(self, endpoint, params: dict = {}, timeout=10) -> dict:
+    def get(self, endpoint, params=None, timeout=10, headers=None) -> dict:
         url = "https://api.marketdata.app/v1/" + endpoint
 
+        if params is None:
+            params = {}
+
         # set token param if it wasn't passed.
         params["token"] = self.MARKETDATA_TOKEN
 
@@ -64,7 +67,13 @@ class MarketData:
         # monitored even if someone doesn't make it through an affiliate link.
         params["application"] = "simplestockbot"
 
-        resp = r.get(url, params=params, timeout=timeout)
+        if headers is None:
+            headers = {}
+        headers = {"User-Agent": "Simple Stock Bot anson@ansonbiggs.com"} | headers
+
+        resp = r.get(url, params=params, timeout=timeout, headers=headers)
+
+        logging.error(resp.headers.items())
 
         # Make sure API returned a proper status code
         try:
@@ -95,7 +104,15 @@ class MarketData:
         return self.symbol_list.get(symbol.upper(), None)
 
     def get_symbol_list(self):
-        sec_resp = r.get("https://www.sec.gov/files/company_tickers.json")
+        # Doesn't use `self.get`` since needs are much different
+        sec_resp = r.get(
+            "https://www.sec.gov/files/company_tickers.json",
+            headers={
+                "User-Agent": "Simple Stock Bot anson@ansonbiggs.com",
+                "Accept-Encoding": "gzip, deflate",
+                "Host": "www.sec.gov",
+            },
+        )
         sec_resp.raise_for_status()
         sec_data = sec_resp.json()
 

common/cg_Crypto.py

@@ -9,6 +9,8 @@ from markdownify import markdownify
 from common.Symbol import Coin
 from common.utilities import rate_limited
 
+import time
+
 log = logging.getLogger(__name__)
 
 
@@ -34,6 +36,12 @@ class cg_Crypto:
         url = "https://api.coingecko.com/api/v3" + endpoint
         resp = r.get(url, params=params, timeout=timeout)
         # Make sure API returned a proper status code
+
+        if resp.status_code == 429:
+            log.warning(f"CoinGecko returned 429 - Too Many Requests for endpoint: {endpoint}. Sleeping and trying again.")
+            time.sleep(10)
+            return self.get(endpoint=endpoint, params=params, timeout=timeout)
+
         try:
             resp.raise_for_status()
         except r.exceptions.HTTPError as e:

site/.gitlab-ci.yml

@@ -1,7 +1,7 @@
 image: python:3.11
 
-build_site:
-  stage: build
+build_mkdocs:
+  stage: build_site
   script:
     - cd ./site
     - pip install -r requirements.txt
@@ -10,12 +10,12 @@ build_site:
     paths:
       - public
 
-deploy_site:
+pages:
   stage: deploy
   script:
     - echo "Publishing site..."
   dependencies:
-    - build_site
+    - build_mkdocs
   artifacts:
     paths:
       - public