CCF_100/Minecraft-Updater
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add more handling for HTTP 429, also reuse connection

Browse Source
This commit is contained in:
Chloe Fontenot 🏳️‍⚧️ 2022-02-26 16:57:26 -06:00
parent c8713742af
commit 3d61098f9e
4 changed files with 13 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -1,3 +1,6 @@
[submodule "request_wrapper"] [submodule "request_wrapper"]
path = request_wrapper path = request_wrapper
url = https://gitlab.com/snippets/1900824.git url = https://gitlab.com/snippets/1900824.git
[submodule "handling-http-429-with-tenacity"]
path = handling-http-429-with-tenacity
url = https://github.com/alexwlchan/handling-http-429-with-tenacity.git

1
handling-http-429-with-tenacity Submodule

@ -0,0 +1 @@
Subproject commit 6dd6bb517515b37e0d0e89aa07767b32072c5ee8

1
mcUp.py
View File

@ -10,6 +10,7 @@ import argparse
import shutil import shutil
import hashlib import hashlib
from os import error from os import error
from urllib.parse import unquote
parser = argparse.ArgumentParser(description='A command-line tool to update a Minecraft Server.') parser = argparse.ArgumentParser(description='A command-line tool to update a Minecraft Server.')
parser.add_argument('api', metavar='api', help='which API to use') parser.add_argument('api', metavar='api', help='which API to use')
parser.add_argument('project', metavar='project', help='which project to query for') parser.add_argument('project', metavar='project', help='which project to query for')

13
parsers/modrinth.py
View File

@ -8,6 +8,7 @@ from request_wrapper import requests_wrapper as requests
import datetime import datetime
import iso8601 import iso8601
import pytz import pytz
from urllib.parse import unquote
debug = True debug = True
if debug == True: if debug == True:
import logging import logging
@ -18,6 +19,8 @@ if debug == True:
requests_log.setLevel(logging.DEBUG) requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True requests_log.propagate = True
timeoutTime = 1 timeoutTime = 1
# Setup session, this lets the parser re-use the connection instead of establishing a new connection for EACH request, not only does this cause a HUGE performance boost, it's also nicer to the API.
session = requests.Session()
base_api_url = "https://api.modrinth.com:443/api/v1" base_api_url = "https://api.modrinth.com:443/api/v1"
def failCheck(response, functOrigin): def failCheck(response, functOrigin):
print("Status Code is: "+str(response.status_code)) print("Status Code is: "+str(response.status_code))
@ -40,9 +43,9 @@ def modInfo(project):
if "modInfo" in dataCache: if "modInfo" in dataCache:
print("Returning cached data!") print("Returning cached data!")
return dataCache["modInfo"] return dataCache["modInfo"]
response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET) response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
if failCheck(response, "modInfo") == True: #Attempt to requery API if failCheck(response, "modInfo") == True: #Attempt to requery API
response = requests.get(base_api_url+"/mod/"+project, family=socket.AF_INET) response = session.get(base_api_url+"/mod/"+project, family=socket.AF_INET)
api_response = response.json() api_response = response.json()
cacheData("modInfo", api_response) cacheData("modInfo", api_response)
return api_response return api_response
@ -66,9 +69,9 @@ def getAllModVersionInfo(project):
responseList = [] responseList = []
numberOfVersions = len(versions) numberOfVersions = len(versions)
for item in range(numberOfVersions): for item in range(numberOfVersions):
response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
if failCheck(response, "getAllModVersionInfo") == True: #Attempt to requery API if failCheck(response, "getAllModVersionInfo") == True: #Attempt to requery API
response = requests.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET) response = session.get(base_api_url+"/version/"+versions[item], family=socket.AF_INET)
api_response = response.json() api_response = response.json()
responseList.append(api_response) responseList.append(api_response)
cacheData("getAllMinecraftVersionInfo", responseList) cacheData("getAllMinecraftVersionInfo", responseList)
@ -122,7 +125,7 @@ def getDownloadURL(project, versionID):
workingDict2 = workingList[0] workingDict2 = workingList[0]
workingDict3 = workingDict2["hashes"] workingDict3 = workingDict2["hashes"]
#print(workingDict3) #print(workingDict3)
downloadURLs[versions[item]] = workingDict2["url"] downloadURLs[versions[item]] = unquote(workingDict2["url"])
downloadSHA1[versions[item]] = workingDict3["sha1"] downloadSHA1[versions[item]] = workingDict3["sha1"]
downloadFilenames[versions[item]] = workingDict2["filename"] downloadFilenames[versions[item]] = workingDict2["filename"]
#print(downloadURLs) #print(downloadURLs)