diff --git a/rdw_ingestion_tools/api/aaq/__init__.py b/rdw_ingestion_tools/api/aaq/__init__.py index 7e6a6e6..18bd6c8 100644 --- a/rdw_ingestion_tools/api/aaq/__init__.py +++ b/rdw_ingestion_tools/api/aaq/__init__.py @@ -1,5 +1,3 @@ -from collections.abc import Iterator - from httpx import Client from .. import config_from_env @@ -8,39 +6,6 @@ BASE_URL = config_from_env("AAQ_API_BASE_URL") -def get_paginated( - client: Client, - url: str, - **kwargs: str | int, -) -> Iterator[dict]: - """Paginate over pages in an AAQ endpoint up to a limit.""" - - limit: int = 100 - offset: int = 0 - - params = {"offset": offset, "limit": limit} - - while True: - print( - "Retrieving results for offsets: ", - params["offset"], - "to", - params["offset"] + limit, - sep=" ", - ) - # Need {**params, **kwargs}. mypy dislikes str|int for lines 27, 40. - response = client.get(url, params={**params, **kwargs}) - response.raise_for_status() - - result: list[dict] = response.json()["result"] - yield from result - - if len(result) < limit: - return - else: - params["offset"] += limit - - headers = { "Authorization": f"Bearer {API_KEY}", "Accept": "application/vnd.v1+json", diff --git a/rdw_ingestion_tools/api/aaq/extensions/httpx.py b/rdw_ingestion_tools/api/aaq/extensions/httpx.py new file mode 100644 index 0000000..8d985bb --- /dev/null +++ b/rdw_ingestion_tools/api/aaq/extensions/httpx.py @@ -0,0 +1,36 @@ +from collections.abc import Iterator + +from httpx import Client + + +def get_paginated( + client: Client, + url: str, + **kwargs: str | int, +) -> Iterator[dict]: + """Paginate over pages in an AAQ endpoint up to a limit.""" + + limit: int = 100 + offset: int = 0 + + params = {"offset": offset, "limit": limit} + + while True: + print( + "Retrieving results for offsets: ", + params["offset"], + "to", + params["offset"] + limit, + sep=" ", + ) + # Need {**params, **kwargs}. mypy dislikes str|int for lines 27, 40. + response = client.get(url, params={**params, **kwargs}) + response.raise_for_status() + + result: list[dict] = response.json()["result"] + yield from result + + if len(result) < limit: + return + else: + params["offset"] += limit diff --git a/rdw_ingestion_tools/api/aaq/requests/faqmatches.py b/rdw_ingestion_tools/api/aaq/requests/faqmatches.py index c20ede6..e1a7da8 100644 --- a/rdw_ingestion_tools/api/aaq/requests/faqmatches.py +++ b/rdw_ingestion_tools/api/aaq/requests/faqmatches.py @@ -2,7 +2,7 @@ from httpx import Client from pandas import DataFrame -from .. import get_paginated +from ..extensions.httpx import get_paginated @define diff --git a/rdw_ingestion_tools/api/aaq/requests/inbounds.py b/rdw_ingestion_tools/api/aaq/requests/inbounds.py index 49d222b..3bba17c 100644 --- a/rdw_ingestion_tools/api/aaq/requests/inbounds.py +++ b/rdw_ingestion_tools/api/aaq/requests/inbounds.py @@ -5,7 +5,7 @@ from httpx import Client from pandas import DataFrame -from .. import get_paginated +from ..extensions.httpx import get_paginated class FAQModel(TypedDict): diff --git a/rdw_ingestion_tools/api/aaq/requests/inbounds_ud.py b/rdw_ingestion_tools/api/aaq/requests/inbounds_ud.py index 668e1c4..2e88f83 100644 --- a/rdw_ingestion_tools/api/aaq/requests/inbounds_ud.py +++ b/rdw_ingestion_tools/api/aaq/requests/inbounds_ud.py @@ -2,7 +2,7 @@ from httpx import Client from pandas import DataFrame -from .. import get_paginated +from ..extensions.httpx import get_paginated @define diff --git a/rdw_ingestion_tools/api/aaq/requests/urgency_rules.py b/rdw_ingestion_tools/api/aaq/requests/urgency_rules.py index 4970520..0fae226 100644 --- a/rdw_ingestion_tools/api/aaq/requests/urgency_rules.py +++ b/rdw_ingestion_tools/api/aaq/requests/urgency_rules.py @@ -2,7 +2,7 @@ from httpx import Client from pandas import DataFrame -from .. import get_paginated +from ..extensions.httpx import get_paginated @define diff --git a/rdw_ingestion_tools/api/flow_results/__init__.py b/rdw_ingestion_tools/api/flow_results/__init__.py index 1bb55c6..ced8d14 100644 --- a/rdw_ingestion_tools/api/flow_results/__init__.py +++ b/rdw_ingestion_tools/api/flow_results/__init__.py @@ -1,5 +1,3 @@ -from collections.abc import Iterator - from httpx import Client from .. import config_from_env @@ -8,50 +6,6 @@ BASE_URL = config_from_env("FLOW_RESULTS_API_BASE_URL") -def get_ids(client: Client, **kwargs: str | int) -> Iterator[str]: - """Returns a list of flow id's. - - These id's are required in order to get responses from the Flow - Results API. - - """ - - params = {**kwargs} - url = "" - - response = client.get(url, params=params) - response.raise_for_status() - - for flow in response.json()["data"]: - yield flow["id"] - - -def get_paginated( - client: Client, url: str, **kwargs: str | int -) -> Iterator[list]: - """Paginate over the Flow Results Responses Endpoint. - - Each response returns a next link which is followed until - the full result set is returned. - - """ - - while True: - response = client.get(url, params={**kwargs}) - response.raise_for_status() - - data: dict = response.json()["data"] - - results: list = data["attributes"]["responses"] - yield from results - - try: - full_url = data["relationships"]["links"]["next"] - url = full_url.split("packages/")[-1] - except AttributeError: - break - - headers = {"Authorization": f"Token {API_KEY}"} client: Client = Client(base_url=BASE_URL, headers=headers) diff --git a/rdw_ingestion_tools/api/flow_results/extensions/ *Minibuf-1* b/rdw_ingestion_tools/api/flow_results/extensions/ *Minibuf-1* new file mode 100644 index 0000000..1edb35b --- /dev/null +++ b/rdw_ingestion_tools/api/flow_results/extensions/ *Minibuf-1* @@ -0,0 +1 @@ +Directory ‘/home/schalk/reach/rdw-ingestion-tools/rdw_ingestion_tools/api/flow_results/extensions/’ does not exist; create? (y or n) \ No newline at end of file diff --git a/rdw_ingestion_tools/api/flow_results/extensions/httpx.py b/rdw_ingestion_tools/api/flow_results/extensions/httpx.py new file mode 100644 index 0000000..682bb91 --- /dev/null +++ b/rdw_ingestion_tools/api/flow_results/extensions/httpx.py @@ -0,0 +1,47 @@ +from collections.abc import Iterator + +from httpx import Client + + +def get_ids(client: Client, **kwargs: str | int) -> Iterator[str]: + """Returns a list of flow id's. + + These id's are required in order to get responses from the Flow + Results API. + + """ + + params = {**kwargs} + url = "" + + response = client.get(url, params=params) + response.raise_for_status() + + for flow in response.json()["data"]: + yield flow["id"] + + +def get_paginated( + client: Client, url: str, **kwargs: str | int +) -> Iterator[list]: + """Paginate over the Flow Results Responses Endpoint. + + Each response returns a next link which is followed until + the full result set is returned. + + """ + + while True: + response = client.get(url, params={**kwargs}) + response.raise_for_status() + + data: dict = response.json()["data"] + + results: list = data["attributes"]["responses"] + yield from results + + try: + full_url = data["relationships"]["links"]["next"] + url = full_url.split("packages/")[-1] + except AttributeError: + break diff --git a/rdw_ingestion_tools/api/flow_results/requests/flows.py b/rdw_ingestion_tools/api/flow_results/requests/flows.py index 46fdaea..b0c3ca8 100644 --- a/rdw_ingestion_tools/api/flow_results/requests/flows.py +++ b/rdw_ingestion_tools/api/flow_results/requests/flows.py @@ -1,7 +1,7 @@ from attrs import define from httpx import Client -from .. import get_ids +from ..extensions.httpx import get_ids @define diff --git a/rdw_ingestion_tools/api/flow_results/requests/responses.py b/rdw_ingestion_tools/api/flow_results/requests/responses.py index f9e5237..f2e0887 100644 --- a/rdw_ingestion_tools/api/flow_results/requests/responses.py +++ b/rdw_ingestion_tools/api/flow_results/requests/responses.py @@ -3,7 +3,7 @@ from attrs import define from httpx import Client -from .. import get_ids, get_paginated +from ..extensions.httpx import get_ids, get_paginated @define