From 29742671f116d2bd31946b89f47e2c52b1e84b32 Mon Sep 17 00:00:00 2001 From: Yang Chun Ung Date: Wed, 27 Dec 2023 13:57:49 +0900 Subject: [PATCH] Make headless, data_provider url configurable --- .github/workflows/unittest.yaml | 2 ++ tests/api_test.py | 12 ++++++------ tests/data_provider_test.py | 7 ++++--- tests/graphql_test.py | 10 ++++------ tests/kms_test.py | 11 ++++++----- tests/tasks_test.py | 24 ++++++++++-------------- world_boss/app/api.py | 11 ++++++----- world_boss/app/config.py | 4 ++++ world_boss/app/data_provider.py | 17 +++++++---------- world_boss/app/graphql.py | 14 ++++++-------- world_boss/app/kms.py | 19 ++----------------- world_boss/app/raid.py | 4 ++-- world_boss/app/tasks.py | 10 +++++----- 13 files changed, 64 insertions(+), 81 deletions(-) diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 047f24e..3ac4e6e 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -40,6 +40,8 @@ jobs: KMS_KEY_ID: ${{ secrets.KMS_KEY_ID }} SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} GRAPHQL_PASSWORD: ${{ secrets.GRAPHQL_PASSWORD }} + HEADLESS_URL: ${{ secrets.HEADLESS_URL }} + DATA_PROVIDER_URL: ${{ secrets.DATA_PROVIDER_URL }} run: | poetry run pytest --redis-exec=$(which redis-server) --cov world_boss --cov-report=xml - name: Upload coverage to Codecov diff --git a/tests/api_test.py b/tests/api_test.py index c71a385..9bfe300 100644 --- a/tests/api_test.py +++ b/tests/api_test.py @@ -11,9 +11,9 @@ from starlette.testclient import TestClient from world_boss.app.cache import cache_exists, set_to_cache -from world_boss.app.data_provider import DATA_PROVIDER_URLS +from world_boss.app.config import config from world_boss.app.enums import NetworkType -from world_boss.app.kms import HEADLESS_URLS, MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount @@ -79,7 +79,7 @@ def test_count_total_users( ): httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossTotalUsers": 100}}, ) with unittest.mock.patch( @@ -115,13 +115,13 @@ def test_generate_ranking_rewards_csv( ] httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossRankingRewards": requested_rewards}}, ) httpx_mock.add_response( method="POST", - url=MINER_URLS[NetworkType.MAIN], + url=config.headless_url, json={ "data": { "stateQuery": { @@ -261,7 +261,7 @@ def test_stage_transactions( task_id = req.json() task: AsyncResult = AsyncResult(task_id) task.get(timeout=30) - assert m.call_count == len(HEADLESS_URLS[network_type]) * len(fx_transactions) + assert m.call_count == len(fx_transactions) m2.assert_called_once_with( channel="channel_id", text=f"stage {len(fx_transactions)} transactions" ) diff --git a/tests/data_provider_test.py b/tests/data_provider_test.py index 2e31056..88b82ff 100644 --- a/tests/data_provider_test.py +++ b/tests/data_provider_test.py @@ -5,7 +5,8 @@ from pytest_httpx import HTTPXMock from world_boss.app.cache import cache_exists, set_to_cache -from world_boss.app.data_provider import DATA_PROVIDER_URLS, data_provider_client +from world_boss.app.config import config +from world_boss.app.data_provider import data_provider_client from world_boss.app.enums import NetworkType from world_boss.app.stubs import RankingRewardDictionary @@ -15,7 +16,7 @@ def test_get_total_users_count( network_type: NetworkType, raid_id: int, expected_count: int ): - result = data_provider_client.get_total_users_count(raid_id, network_type) + result = data_provider_client.get_total_users_count(raid_id) assert result == expected_count @@ -60,7 +61,7 @@ def test_get_ranking_rewards_error( cache_key = f"world_boss_{raid_id}_{network_type}_{offset}_{limit}" httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[network_type], + url=config.data_provider_url, json={ "errors": [{"message": "can't receive"}], "data": {"worldBossRankingRewards": None}, diff --git a/tests/graphql_test.py b/tests/graphql_test.py index 5ade34c..47a18db 100644 --- a/tests/graphql_test.py +++ b/tests/graphql_test.py @@ -6,9 +6,7 @@ from pytest_httpx import HTTPXMock from world_boss.app.config import config -from world_boss.app.data_provider import DATA_PROVIDER_URLS from world_boss.app.enums import NetworkType -from world_boss.app.kms import HEADLESS_URLS, MINER_URLS from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount @@ -27,7 +25,7 @@ def test_next_tx_nonce(fx_session, fx_test_client): def test_count_total_users(fx_test_client, httpx_mock: HTTPXMock): httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossTotalUsers": 100}}, ) query = "query { countTotalUsers(seasonId: 1) }" @@ -86,13 +84,13 @@ def test_generate_ranking_rewards_csv( ] httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossRankingRewards": requested_rewards}}, ) httpx_mock.add_response( method="POST", - url=MINER_URLS[NetworkType.MAIN], + url=config.headless_url, json={ "data": { "stateQuery": { @@ -295,7 +293,7 @@ def test_stage_transactions( task_id = req.json()["data"]["stageTransactions"] task: AsyncResult = AsyncResult(task_id) task.get(timeout=30) - assert m.call_count == len(HEADLESS_URLS[network_type]) * len(fx_transactions) + assert m.call_count == len(fx_transactions) m2.assert_called_once_with( channel=config.slack_channel_id, text=f"stage {len(fx_transactions)} transactions", diff --git a/tests/kms_test.py b/tests/kms_test.py index 98e1cf9..a6c17bf 100644 --- a/tests/kms_test.py +++ b/tests/kms_test.py @@ -6,8 +6,9 @@ import pytest from gql.transport.exceptions import TransportQueryError +from world_boss.app.config import config from world_boss.app.enums import NetworkType -from world_boss.app.kms import HEADLESS_URLS, MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount from world_boss.app.raid import get_currencies from world_boss.app.stubs import AmountDictionary, Recipient @@ -76,7 +77,7 @@ async def test_check_transaction_status_async(fx_session, fx_mainnet_transaction ], ) def test_prepare_reward_assets(fx_app, network_type: NetworkType): - headless_url = MINER_URLS[network_type] + headless_url = config.headless_url assets: List[AmountDictionary] = [ {"decimalPlaces": 18, "ticker": "CRYSTAL", "quantity": 109380000}, {"decimalPlaces": 0, "ticker": "RUNESTONE_FENRIR1", "quantity": 406545}, @@ -94,7 +95,7 @@ def test_stage_transaction(fx_session, fx_mainnet_transactions): tx = fx_mainnet_transactions[0] fx_session.add(tx) fx_session.flush() - urls = HEADLESS_URLS[NetworkType.INTERNAL] + urls = [config.headless_url] for url in urls: with pytest.raises(TransportQueryError) as e: signer.stage_transaction(url, tx) @@ -105,14 +106,14 @@ def test_query_transaction_result(fx_session, fx_mainnet_transactions): tx = fx_mainnet_transactions[0] fx_session.add(tx) fx_session.flush() - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url signer.query_transaction_result(url, tx.tx_id, fx_session) transaction = fx_session.query(Transaction).one() assert transaction.tx_result == "INCLUDED" def test_query_balance(fx_session): - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url reward = WorldBossReward() reward.avatar_address = "avatar_address" reward.agent_address = "agent_address" diff --git a/tests/tasks_test.py b/tests/tasks_test.py index 37b7c61..f2278e4 100644 --- a/tests/tasks_test.py +++ b/tests/tasks_test.py @@ -5,9 +5,9 @@ import pytest from pytest_httpx import HTTPXMock -from world_boss.app.data_provider import DATA_PROVIDER_URLS +from world_boss.app.config import config from world_boss.app.enums import NetworkType -from world_boss.app.kms import MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount from world_boss.app.stubs import ( RankingRewardDictionary, @@ -30,7 +30,7 @@ def test_count_users(celery_session_worker, httpx_mock: HTTPXMock): httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossTotalUsers": 100}}, ) with unittest.mock.patch("world_boss.app.tasks.client.chat_postMessage") as m: @@ -91,13 +91,13 @@ def test_get_ranking_rewards( redisdb.set(rewards_cache_key, json.dumps(cached_rewards)) httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={"data": {"worldBossRankingRewards": requested_rewards}}, ) redisdb.set(addresses_cache_key, json.dumps(cached_addresses)) httpx_mock.add_response( method="POST", - url=MINER_URLS[NetworkType.MAIN], + url=config.headless_url, json={ "data": { "stateQuery": { @@ -136,7 +136,7 @@ def test_get_ranking_rewards_error( httpx_mock.add_response( method="POST", - url=DATA_PROVIDER_URLS[NetworkType.MAIN], + url=config.data_provider_url, json={ "errors": [{"message": "can't receive"}], "data": {"worldBossRankingRewards": None}, @@ -180,7 +180,7 @@ def test_sign_transfer_assets( nonce, [], "memo", - MINER_URLS[NetworkType.MAIN], + config.headless_url, max_nonce, nonce_list, ).get(timeout=10) @@ -251,7 +251,7 @@ def test_stage_transaction( transaction.nonce = 1 fx_session.add(transaction) fx_session.commit() - url = MINER_URLS[network_type] + url = config.headless_url with unittest.mock.patch( "world_boss.app.tasks.signer.stage_transaction", return_value="tx_id" ) as m: @@ -279,9 +279,7 @@ def test_query_tx_result(celery_session_worker, fx_session, fx_transactions): fx_session.commit() for tx_id in tx_ids: - _, result = query_tx_result.delay(MINER_URLS[NetworkType.MAIN], tx_id).get( - timeout=10 - ) + _, result = query_tx_result.delay(config.headless_url, tx_id).get(timeout=10) tx = fx_session.query(Transaction).filter_by(tx_id=tx_id).one() assert result == "INCLUDED" assert tx.tx_result == "INCLUDED" @@ -306,9 +304,7 @@ def test_upload_result( ) def test_check_signer_balance(celery_session_worker, ticker: str, decimal_places: int): currency = {"ticker": ticker, "decimalPlaces": decimal_places, "minters": []} - result = check_signer_balance.delay(MINER_URLS[NetworkType.MAIN], currency).get( - timeout=10 - ) + result = check_signer_balance.delay(config.headless_url, currency).get(timeout=10) assert result == f"0 {ticker}" diff --git a/world_boss/app/api.py b/world_boss/app/api.py index 865482e..890347f 100644 --- a/world_boss/app/api.py +++ b/world_boss/app/api.py @@ -9,8 +9,9 @@ from sqlalchemy.orm import Session from starlette.responses import JSONResponse, Response +from world_boss.app.config import config from world_boss.app.enums import NetworkType -from world_boss.app.kms import HEADLESS_URLS, MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction from world_boss.app.orm import SessionLocal from world_boss.app.raid import ( @@ -131,7 +132,7 @@ async def prepare_transfer_assets( assert len(recipient_map[k]) <= 100 # insert tables memo = "world boss ranking rewards by world boss signer" - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url task = chord( sign_transfer_assets.s( time_stamp, @@ -185,7 +186,7 @@ async def stage_transactions( network_type = NetworkType.INTERNAL if text.lower() == "main": network_type = NetworkType.MAIN - headless_urls = HEADLESS_URLS[network_type] + headless_urls = [config.headless_url] task = chord( stage_transaction.s(headless_url, nonce) for headless_url in headless_urls @@ -202,7 +203,7 @@ async def transaction_result( db: Session = Depends(get_db), ): tx_ids = db.query(Transaction.tx_id).filter_by(tx_result=None) - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url task = chord(query_tx_result.s(url, str(tx_id)) for tx_id, in tx_ids)( upload_tx_result.s(channel_id) ) @@ -215,7 +216,7 @@ async def check_balance( request: Request, channel_id: Annotated[str, Form()], db: Session = Depends(get_db) ): currencies = get_currencies(db) - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url task = chord(check_signer_balance.s(url, currency) for currency in currencies)( upload_balance_result.s(channel_id) ) diff --git a/world_boss/app/config.py b/world_boss/app/config.py index 5a7f4e7..755547e 100644 --- a/world_boss/app/config.py +++ b/world_boss/app/config.py @@ -26,6 +26,8 @@ class Settings(BaseSettings): sentry_sample_rate: float = 0.1 slack_channel_id: str graphql_password: str + headless_url: str + data_provider_url: str class Config: env_file = ".env" @@ -58,6 +60,8 @@ class Config: "sentry_sample_rate": {"env": "SENTRY_SAMPLE_RATE"}, "slack_channel_id": {"env": "SLACK_CHANNEL_ID"}, "graphql_password": {"env": "GRAPHQL_PASSWORD"}, + "headless_url": {"env": "HEADLESS_URL"}, + "data_provider_url": {"env": "DATA_PROVIDER_URL"}, } diff --git a/world_boss/app/data_provider.py b/world_boss/app/data_provider.py index 83d354c..29ceae6 100644 --- a/world_boss/app/data_provider.py +++ b/world_boss/app/data_provider.py @@ -3,18 +3,16 @@ import httpx +from world_boss.app.config import config from world_boss.app.enums import NetworkType -__all__ = ["DATA_PROVIDER_URLS", "DataProviderClient", "data_provider_client"] +__all__ = ["DataProviderClient", "data_provider_client"] from world_boss.app.cache import cache_exists, get_from_cache, set_to_cache from world_boss.app.stubs import RankingRewardDictionary TOTAL_USER_QUERY = "query($raidId: Int!) { worldBossTotalUsers(raidId: $raidId) }" -DATA_PROVIDER_URLS: dict[NetworkType, str] = { - NetworkType.MAIN: "https://api.9c.gg/graphql", - NetworkType.INTERNAL: "https://api.9c.gg/graphql", -} +DATA_PROVIDER_URL: str = config.data_provider_url RANKING_REWARDS_QUERY = """ query($raidId: Int!, $limit: Int!, $offset: Int!) { worldBossRankingRewards(raidId: $raidId, limit: $limit, offset: $offset) { @@ -39,16 +37,16 @@ class DataProviderClient: def __init__(self): self._client = httpx.Client(timeout=None) - def _query(self, network_type: NetworkType, query: str, variables: dict): + def _query(self, query: str, variables: dict): result = self._client.post( - DATA_PROVIDER_URLS[network_type], + DATA_PROVIDER_URL, json={"query": query, "variables": variables}, ) return result.json() - def get_total_users_count(self, raid_id: int, network_type: NetworkType) -> int: + def get_total_users_count(self, raid_id: int) -> int: variables = {"raidId": raid_id} - result = self._query(network_type, TOTAL_USER_QUERY, variables) + result = self._query(TOTAL_USER_QUERY, variables) return result["data"]["worldBossTotalUsers"] def get_ranking_rewards( @@ -60,7 +58,6 @@ def get_ranking_rewards( rewards = json.loads(cached_value) else: result = self._query( - network_type, RANKING_REWARDS_QUERY, {"raidId": raid_id, "offset": offset, "limit": limit}, ) diff --git a/world_boss/app/graphql.py b/world_boss/app/graphql.py index b0d17eb..cf34a6e 100644 --- a/world_boss/app/graphql.py +++ b/world_boss/app/graphql.py @@ -13,8 +13,7 @@ from world_boss.app.api import get_db from world_boss.app.config import config from world_boss.app.data_provider import data_provider_client -from world_boss.app.enums import NetworkType -from world_boss.app.kms import HEADLESS_URLS, MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction from world_boss.app.raid import ( get_currencies, @@ -60,12 +59,12 @@ def next_tx_nonce(self, info: Info) -> int: @strawberry.field def count_total_users(self, season_id: int) -> int: - return data_provider_client.get_total_users_count(season_id, NetworkType.MAIN) + return data_provider_client.get_total_users_count(season_id) @strawberry.field def check_balance(self, info: Info) -> typing.List[str]: currencies = get_currencies(info.context["db"]) - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url result = [] for currency in currencies: result.append(signer.query_balance(url, currency)) @@ -124,7 +123,7 @@ def prepare_transfer_assets( assert len(recipient_map[k]) <= 100 # insert tables memo = "world boss ranking rewards by world boss signer" - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url task = chord( sign_transfer_assets.s( time_stamp, @@ -152,8 +151,7 @@ def stage_transactions(self, password: str, info: Info) -> str: .filter_by(signer=signer.address, tx_result=None) .all() ) - network_type = NetworkType.MAIN - headless_urls = HEADLESS_URLS[network_type] + headless_urls = [config.headless_url] task = chord( stage_transaction.s(headless_url, nonce) for headless_url in headless_urls @@ -169,7 +167,7 @@ def stage_transactions(self, password: str, info: Info) -> str: def transaction_result(self, password: str, info: Info) -> str: db = info.context["db"] tx_ids = db.query(Transaction.tx_id).filter_by(tx_result=None) - url = MINER_URLS[NetworkType.MAIN] + url = config.headless_url task = chord(query_tx_result.s(url, str(tx_id)) for tx_id, in tx_ids)( upload_tx_result.s(config.slack_channel_id) ) diff --git a/world_boss/app/kms.py b/world_boss/app/kms.py index a4ada61..7ae59a6 100644 --- a/world_boss/app/kms.py +++ b/world_boss/app/kms.py @@ -21,21 +21,6 @@ from world_boss.app.models import Transaction from world_boss.app.stubs import AmountDictionary, CurrencyDictionary, Recipient -MINER_URLS: dict[NetworkType, str] = { - NetworkType.MAIN: "https://9c-main-full-state.nine-chronicles.com/graphql", - NetworkType.INTERNAL: "http://9c-internal-validator-5.nine-chronicles.com/graphql", -} - -HEADLESS_URLS: dict[NetworkType, typing.List[str]] = { - NetworkType.MAIN: [ - MINER_URLS[NetworkType.MAIN], - ], - NetworkType.INTERNAL: [ - MINER_URLS[NetworkType.INTERNAL], - "http://9c-internal-rpc-1.nine-chronicles.com/graphql", - ], -} - class KmsWorldBossSigner: def __init__(self, key_id: str): @@ -232,7 +217,7 @@ def query_balance(self, headless_url: str, currency: CurrencyDictionary) -> str: return f"{balance} {ticker}" async def stage_transactions_async(self, network_type: NetworkType, db: Session): - headless_urls = HEADLESS_URLS[network_type] + headless_urls = [config.headless_url] transactions = ( db.query(Transaction).filter_by(tx_result=None).order_by(Transaction.nonce) ) @@ -265,7 +250,7 @@ async def stage_transaction_async( async def check_transaction_status_async( self, network_type: NetworkType, db: Session ): - headless_url = MINER_URLS[network_type] + headless_url = config.headless_url transactions = ( db.query(Transaction).filter_by(tx_result=None).order_by(Transaction.nonce) ) diff --git a/world_boss/app/raid.py b/world_boss/app/raid.py index 35ec169..899959d 100644 --- a/world_boss/app/raid.py +++ b/world_boss/app/raid.py @@ -10,8 +10,8 @@ from starlette.responses import Response from world_boss.app.cache import cache_exists, get_from_cache, set_to_cache +from world_boss.app.config import config from world_boss.app.enums import NetworkType -from world_boss.app.kms import MINER_URLS from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount from world_boss.app.schemas import WorldBossRewardSchema from world_boss.app.stubs import ( @@ -95,7 +95,7 @@ def update_agent_address( for avatar_address in query_keys: variables[f"arg{avatar_address}"] = avatar_address req = http_client.post( - MINER_URLS[NetworkType.MAIN], json={"query": query, "variables": variables} + config.headless_url, json={"query": query, "variables": variables} ) query_result = req.json() agents = query_result["data"]["stateQuery"] diff --git a/world_boss/app/tasks.py b/world_boss/app/tasks.py index 5357f44..8817b4c 100644 --- a/world_boss/app/tasks.py +++ b/world_boss/app/tasks.py @@ -8,9 +8,9 @@ from sqlalchemy.orm import sessionmaker from world_boss.app.config import config -from world_boss.app.data_provider import DATA_PROVIDER_URLS, data_provider_client +from world_boss.app.data_provider import data_provider_client from world_boss.app.enums import NetworkType -from world_boss.app.kms import MINER_URLS, signer +from world_boss.app.kms import signer from world_boss.app.models import Transaction, WorldBossReward, WorldBossRewardAmount from world_boss.app.raid import ( get_assets, @@ -37,7 +37,7 @@ @celery.task() def count_users(channel_id: str, raid_id: int): - total_count = data_provider_client.get_total_users_count(raid_id, NetworkType.MAIN) + total_count = data_provider_client.get_total_users_count(raid_id) client.chat_postMessage( channel=channel_id, text=f"world boss season {raid_id} total users: {total_count}", @@ -59,7 +59,7 @@ def get_ranking_rewards( except Exception as e: client.chat_postMessage( channel=channel_id, - text=f"failed to get rewards from {DATA_PROVIDER_URLS[NetworkType.MAIN]} exc: {e}", + text=f"failed to get rewards from {config.data_provider_url} exc: {e}", ) raise e rewards = update_agent_address(result, raid_id, NetworkType.MAIN, offset, size) @@ -140,7 +140,7 @@ def insert_world_boss_rewards(rows: List[RecipientRow]): def upload_prepare_reward_assets(channel_id: str, raid_id: int): with TaskSessionLocal() as db: assets = get_assets(raid_id, db) - result = signer.prepare_reward_assets(MINER_URLS[NetworkType.MAIN], assets) + result = signer.prepare_reward_assets(config.headless_url, assets) decoded = bencodex.loads(bytes.fromhex(result)) client.chat_postMessage( channel=channel_id,