CCF_100/Minecraft-Updater
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add more handling for HTTP 429, also reuse connection

Browse Source
This commit is contained in:
Chloe Fontenot 🏳️‍⚧️
2022-02-26 16:57:26 -06:00
parent c8713742af
commit 3d61098f9e
4 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
parsers/modrinth.py
View File

@@ -8,6 +8,7 @@ from request_wrapper import requests_wrapper as requests
import datetime
import iso8601
import pytz
from urllib.parse import unquote
debug = True
if debug == True:
import logging
@@ -18,6 +19,8 @@ if debug == True:
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
timeoutTime = 1
# Setup session, this lets the parser re-use the connection instead of establishing a new connection for EACH request, not only does this cause a HUGE performance boost, it's also nicer to the API.
session = requests.Session()
base_api_url = "https://api.modrinth.com:443/api/v1"
def failCheck(response, functOrigin):
print("Status Code is: "+str(response.status_code))
@@ -40,9 +43,9 @@ def modInfo(project):
if "modInfo" in dataCache:
print("Returning cached data!")
return dataCache["modInfo"]
response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
if failCheck(response, "modInfo") == True: #Attempt to requery API
response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
api_response = response.json()
cacheData("modInfo", api_response)
return api_response
@@ -66,9 +69,9 @@ def getAllModVersionInfo(project):
responseList = []
numberOfVersions = len(versions)
for item in range(numberOfVersions):
response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
if failCheck(response, "getAllModVersionInfo") == True: #Attempt to requery API
response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
api_response = response.json()
responseList.append(api_response)
cacheData("getAllMinecraftVersionInfo", responseList)
@@ -122,7 +125,7 @@ def getDownloadURL(project, versionID):
workingDict2 = workingList[0]
workingDict3 = workingDict2["hashes"]
#print(workingDict3)
downloadURLs[versions[item]] = workingDict2["url"]
downloadURLs[versions[item]] = unquote(workingDict2["url"])
downloadSHA1[versions[item]] = workingDict3["sha1"]
downloadFilenames[versions[item]] = workingDict2["filename"]
#print(downloadURLs)