Skip to content

Commit

Permalink
Fix #31 - fixed api key search
Browse files Browse the repository at this point in the history
  • Loading branch information
@ScrappyCocco
ScrappyCocco committed Nov 16, 2024
1 parent 76d3d36 commit 8ce01f8
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 18 deletions.
86 changes: 70 additions & 16 deletions howlongtobeatpy/howlongtobeatpy/HTMLRequests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ def get_search_request_data(game_name: str, search_modifiers: SearchModifiers, p
'min': 0,
'max': 0
},
'rangeYear':
{
'max': "",
'min': ""
},
'gameplay': {
'perspective': "",
'flow': "",
Expand All @@ -73,14 +78,18 @@ def get_search_request_data(game_name: str, search_modifiers: SearchModifiers, p
'modifier': search_modifiers.value,
},
'users': {
'id': api_key,
'sortCategory': "postcount"
},
'filter': "",
'sort': 0,
'randomizer': 0
}
}

# If api_key is passed add it to the dict
if api_key is not None:
payload['searchOptions']['users']['id'] = api_key

return json.dumps(payload)

@staticmethod
Expand All @@ -97,10 +106,17 @@ def send_web_request(game_name: str, search_modifiers: SearchModifiers = SearchM
api_key_result = HTMLRequests.send_website_request_getcode(False)
if api_key_result is None:
api_key_result = HTMLRequests.send_website_request_getcode(True)
payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, api_key_result)
# Make the post request and return the result if is valid
search_url_with_key = HTMLRequests.SEARCH_URL
# Make the request
# The main method currently is the call to api/search/key
search_url_with_key = HTMLRequests.SEARCH_URL + "/" + api_key_result
payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, None)
resp = requests.post(search_url_with_key, headers=headers, data=payload, timeout=60)
if resp.status_code == 200:
return resp.text
# Try to call with the standard url adding the api key to the user
search_url = HTMLRequests.SEARCH_URL
payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, api_key_result)
resp = requests.post(search_url, headers=headers, data=payload, timeout=60)
if resp.status_code == 200:
return resp.text
return None
Expand All @@ -119,6 +135,22 @@ async def send_async_web_request(game_name: str, search_modifiers: SearchModifie
api_key_result = await HTMLRequests.async_send_website_request_getcode(False)
if api_key_result is None:
api_key_result = await HTMLRequests.async_send_website_request_getcode(True)
# Make the request
# The main method currently is the call to api/search/key
search_url_with_key = HTMLRequests.SEARCH_URL + "/" + api_key_result
payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, None)
async with aiohttp.ClientSession() as session:
async with session.post(search_url_with_key, headers=headers, data=payload) as resp_with_key:
if resp_with_key is not None and resp_with_key.status == 200:
return await resp_with_key.text()
else:
search_url = HTMLRequests.SEARCH_URL
payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, api_key_result)
async with session.post(search_url, headers=headers, data=payload) as resp_user_id:
if resp_user_id is not None and resp_user_id.status == 200:
return await resp_user_id.text()
return None

payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page, api_key_result)
# Make the post request and return the result if is valid
search_url_with_key = HTMLRequests.SEARCH_URL + "/" + api_key_result
Expand Down Expand Up @@ -208,11 +240,35 @@ async def async_get_game_title(game_id: int):
return HTMLRequests.__cut_game_title(text)
return None

@staticmethod
def extract_api_from_script(script_content: str):
"""
Function that extract the htlb code to use in the request from the given script
@return: the string of the api key found
"""
# Try multiple find one after the other as hltb keep changing format
# Test 1 - The API Key is in the user id in the request json
user_id_api_key_pattern = r'users\s*:\s*{\s*id\s*:\s*"([^"]+)"'
matches = re.findall(user_id_api_key_pattern, script_content)
if matches:
key = ''.join(matches)
return key
# Test 2 - The API Key is in format fetch("/api/search/".concat("X").concat("Y")...
concat_api_key_pattern = r'\/api\/search\/"(?:\.concat\("[^"]*"\))*'
matches = re.findall(concat_api_key_pattern, script_content)
if matches:
matches = str(matches).split('.concat')
matches = [re.sub(r'["\(\)\[\]\']', '', match) for match in matches[1:]]
key = ''.join(matches)
return key
# Unable to find :(
return None

@staticmethod
def send_website_request_getcode(parse_all_scripts: bool):
"""
Function that send a request to howlongtobeat to scrape the /api/search key
@return: The string key to use on /api/search
@return: The string key to use
"""
# Make the post request and return the result if is valid
headers = HTMLRequests.get_title_request_headers()
Expand All @@ -230,23 +286,22 @@ def send_website_request_getcode(parse_all_scripts: bool):
script_url = HTMLRequests.BASE_URL + script_url
script_resp = requests.get(script_url, headers=headers, timeout=60)
if script_resp.status_code == 200 and script_resp.text is not None:
pattern = r'users\s*:\s*{\s*id\s*:\s*"([^"]+)"'
matches = re.findall(pattern, script_resp.text)
for match in matches:
return match
api_key_result = HTMLRequests.extract_api_from_script(script_resp.text)
if api_key_result is not None:
return api_key_result
return None

@staticmethod
async def async_send_website_request_getcode(parse_all_scripts: bool):
"""
Function that send a request to howlongtobeat to scrape the /api/search key
@return: The string key to use on /api/search
@return: The string key to use
"""
# Make the post request and return the result if is valid
headers = HTMLRequests.get_title_request_headers()
async with aiohttp.ClientSession() as session:
async with session.get(HTMLRequests.BASE_URL, headers=headers) as resp:
if resp is not None and str(resp.status) == "200":
if resp is not None and resp.status == 200:
resp_text = await resp.text()
# Parse the HTML content using BeautifulSoup
soup = BeautifulSoup(resp_text, 'html.parser')
Expand All @@ -260,12 +315,11 @@ async def async_send_website_request_getcode(parse_all_scripts: bool):
script_url = HTMLRequests.BASE_URL + script_url
async with aiohttp.ClientSession() as session:
async with session.get(script_url, headers=headers) as script_resp:
if script_resp is not None and str(resp.status) == "200":
if script_resp is not None and resp.status == 200:
script_resp_text = await script_resp.text()
pattern = r'users\s*:\s*{\s*id\s*:\s*"([^"]+)"'
matches = re.findall(pattern, script_resp_text)
for match in matches:
return match
api_key_result = HTMLRequests.extract_api_from_script(script_resp_text)
if api_key_result is not None:
return api_key_result
else:
return None
else:
Expand Down
2 changes: 1 addition & 1 deletion howlongtobeatpy/setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
long_description = fh.read()

setup(name='howlongtobeatpy',
version='1.0.10',
version='1.0.11',
packages=find_packages(exclude=['tests']),
description='A Python API for How Long to Beat',
long_description=long_description,
Expand Down
2 changes: 1 addition & 1 deletion sonar-project.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ sonar.organization=scrappycocco-github
sonar.projectKey=ScrappyCocco_HowLongToBeat-PythonAPI

sonar.projectName=HowLongToBeat-PythonAPI
sonar.projectVersion=1.0.10
sonar.projectVersion=1.0.11
sonar.python.version=3.9

# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows.
Expand Down

0 comments on commit 8ce01f8

Please sign in to comment.