From 9877787a62794f5e26e40cc15f610b81bb0bd984 Mon Sep 17 00:00:00 2001 From: wavey Date: Sat, 29 Jan 2022 22:02:39 -0500 Subject: [PATCH 01/86] feat: push StrategyReported event data to postgres --- scripts/collect_reports.py | 356 +++++++++++++++++++++++++++++++++++++ yearn/db/models.py | 73 ++++++++ 2 files changed, 429 insertions(+) create mode 100644 scripts/collect_reports.py diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py new file mode 100644 index 000000000..3828869a7 --- /dev/null +++ b/scripts/collect_reports.py @@ -0,0 +1,356 @@ +import logging +import time, os +from dotenv import load_dotenv +from datetime import datetime, timedelta, timezone +from itertools import count +from brownie import chain, interface, web3, Contract, ZERO_ADDRESS +from web3._utils.events import construct_event_topic_set +from yearn.utils import closest_block_after_timestamp, contract, contract_creation_block +from yearn.prices import magic, constants +from yearn.db.models import Reports, Transactions, Session, engine, select +from sqlalchemy import select as Select, desc, asc +from yearn.networks import Network +from yearn.events import create_filter, decode_logs +import warnings +warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") +warnings.filterwarnings("ignore", ".*It has been discarded*") + + +CHAIN_VALUES = { + Network.Mainnet: { + "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), + "START_BLOCK": 11772924, + "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", + "REGISTRY_HELPER_ADDRESS": "0x52CbF68959e082565e7fd4bBb23D9Ccfb8C8C057", + "LENS_ADDRESS": "0x5b4F3BE554a88Bd0f8d8769B9260be865ba03B4a", + "LENS_DEPLOY_BLOCK": 12707450, + "VAULT_ADDRESS030": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", + "KEEPER_CALL_CONTRACT": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "KEEPER_TOKEN": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", + }, + Network.Fantom: { + "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), + "START_BLOCK": 16241109, + "REGISTRY_ADDRESS": "0x727fe1759430df13655ddb0731dE0D0FDE929b04", + "REGISTRY_HELPER_ADDRESS": "0x8CC45f739104b3Bdb98BFfFaF2423cC0f817ccc1", + "REGISTRY_HELPER_DEPLOY_BLOCK": 18456459, + "LENS_ADDRESS": "0x97D0bE2a72fc4Db90eD9Dbc2Ea7F03B4968f6938", + "LENS_DEPLOY_BLOCK": 18842673, + "VAULT_ADDRESS030": "0x637eC617c86D24E421328e6CAEa1d92114892439", + "VAULT_ADDRESS031": "0x637eC617c86D24E421328e6CAEa1d92114892439", + "KEEPER_CALL_CONTRACT": "0x39cAcdb557CA1C4a6555E00203B4a00B1c1a94f8", + "KEEPER_TOKEN": "", + "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", + }, + Network.Arbitrum: { + "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), + "START_BLOCK": 4841854, + "REGISTRY_ADDRESS": "", + "REGISTRY_HELPER_ADDRESS": "", + "LENS_ADDRESS": "", + "VAULT_ADDRESS030": "", + "VAULT_ADDRESS031": "", + "KEEPER_CALL_CONTRACT": "", + "KEEPER_TOKEN": "", + "YEARN_TREASURY": "", + } +} + + +# Primary vault interface +vault = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS031"]) +vault = web3.eth.contract(str(vault), abi=vault.abi) +topics = construct_event_topic_set( + vault.events.StrategyReported().abi, web3.codec, {} +) +# Deprecated vault interface +if chain.id == 1: + vault_v030 = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) + vault_v030 = web3.eth.contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"], abi=vault_v030.abi) + topics_v030 = construct_event_topic_set( + vault_v030.events.StrategyReported().abi, web3.codec, {} + ) + +def main(dynamically_find_multi_harvest=False): + print(f"dynamic multi_harvest detection is enabled: {dynamically_find_multi_harvest}") + interval_seconds = 25 + + last_reported_block, last_reported_block030 = last_harvest_block() + + print("latest block (v0.3.1+ API)",last_reported_block) + print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) + if chain.id == 1: + print("latest block (v0.3.0 API)",last_reported_block030) + print("blocks behind (v0.3.0+ API)", chain.height - last_reported_block030) + event_filter = web3.eth.filter({'topics': topics, "fromBlock": last_reported_block + 1}) + if chain.id == 1: + event_filter_v030 = web3.eth.filter({'topics': topics_v030, "fromBlock": last_reported_block030 + 1}) + + while True: # Keep this as a long-running script + events_to_process = [] + transaction_hashes = [] + if dynamically_find_multi_harvest: + # The code below is used to populate the "multi_harvest" property # + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(0, len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + + if chain.id == 1: # No old vaults deployed anywhere other than mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(0, len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + + for e in events_to_process: + handle_event(e.event, e.multi_harvest, e.isOldApi) + time.sleep(interval_seconds) + else: + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + handle_event(e.event, e.multi_harvest, e.isOldApi) + + if chain.id == 1: # Old vault API exists only on Ethereum mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) + handle_event(e.event, e.multi_harvest, e.isOldApi) + + time.sleep(interval_seconds) + +def handle_event(event, multi_harvest, isOldApi): + txn_hash = event.transaction_hash.hex() + tx = web3.eth.getTransactionReceipt(txn_hash) + gas_price = web3.eth.getTransaction(txn_hash).gasPrice + # TODO: Detect if endorsed ✅ # Could use review + # TODO: Lookup last harvest ✅ + # TODO: Lookup last harvest on chain id for each abi type ✅ + # TODO: Block duplicates on insert ✅ + # TODO: Refactor to two tables: Transactions + Reports ✅ + # TODO: Add keep3r payment logic ✅ + # TODO: Add logger + # TODO: Make it multichain compatible (better varible lookups) ✅ + # TODO: Add APY figures ✅ + # TODO: Get hosting + ts = chain[event.block_number].timestamp + dt = datetime.utcfromtimestamp(ts).strftime("%m/%d/%Y, %H:%M:%S") + r = Reports() + r.multi_harvest = multi_harvest + r.chain_id = chain.id + if isOldApi: + r.strategy_address, r.gain, r.loss, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = event.values() + else: + r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = event.values() + if check_endorsed(r.strategy_address, event.block_number) == False: + print(f"skipping: not endorsed. strategy {r.strategy_address} txn hash {txn_hash}. chain id {r.chain_id} sync {event.block_number} / {chain.height}.") + return + + txn_record_exists = transaction_record_exists(txn_hash) + if not transaction_record_exists(txn_hash): + t = Transactions() + t.chain_id = chain.id + t.txn_hash = txn_hash + t.block = event.block_number + t.txn_to = tx.to + t.txn_from = tx["from"] + t.txn_gas_used = tx.gasUsed + t.txn_gas_price = gas_price + t.eth_price_at_block = magic.get_price(constants.weth, t.block) + t.call_cost_eth = gas_price * tx.gasUsed / 1e18 + t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth + if chain.id == 1: + t.kp3r_price_at_block = magic.get_price("0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", t.block) + t.kp3r_paid = get_keeper_payment(tx) + t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block / 1e18 + t.keeper_called = t.kp3r_paid > 0 + if chain.id == 250: + if t.txn_to == "0x39cAcdb557CA1C4a6555E00203B4a00B1c1a94f8": + t.keeper_called = True + else: + t.keeper_called = False + t.date = datetime.utcfromtimestamp(ts) + t.date_string = dt + t.timestamp = ts + t.updated_timestamp = datetime.now() + + r.vault_address = event.address + r.block = event.block_number + r.txn_hash = txn_hash + strategy = contract(r.strategy_address) + vault = contract(r.vault_address) + + r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address) + r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want + r.want_token = strategy.want() + r.want_price_at_block = magic.get_price(r.want_token, r.block) + r.vault_api = vault.apiVersion() + r.vault_decimals = vault.decimals() + r.want_gain_usd = r.gain / 10**r.vault_decimals * r.want_price_at_block + r.vault_name = vault.name() + r.strategy_name = strategy.name() + r.strategy_api = strategy.apiVersion() + r.vault_symbol = vault.symbol() + r.date = datetime.utcfromtimestamp(ts) + r.date_string = dt + r.timestamp = ts + r.updated_timestamp = datetime.now() + + with Session(engine) as session: + query = select(Reports).where( + Reports.chain_id == chain.id, Reports.strategy_address == r.strategy_address + ).order_by(desc(Reports.block)) + previous_report = session.exec(query).first() + if previous_report != None: + previous_report_id = previous_report.id + r.previous_report_id = previous_report_id + r.rough_apr_pre_fee, r.rough_apr_post_fee = compute_apr(r, previous_report) + # Insert to database + session.add(r) + if not txn_record_exists: + session.add(t) + session.commit() + print(f"report added. strategy {r.strategy_address} txn hash {r.txn_hash}. chain id {r.chain_id} sync {r.block} / {chain.height}.") + +def transaction_record_exists(txn_hash): + with Session(engine) as session: + query = select(Transactions).where( + Transactions.txn_hash == txn_hash + ) + result = session.exec(query).first() + if result == None: + return False + return True + +def last_harvest_block(): + with Session(engine) as session: + query = select(Reports.block).where( + Reports.chain_id == chain.id, Reports.vault_api != "0.3.0" + ).order_by(desc(Reports.block)) + result1 = session.exec(query).first() + if result1 == None: + result1 = CHAIN_VALUES[chain.id]["START_BLOCK"] + if chain.id == 1: + query = select(Reports.block).where( + Reports.chain_id == chain.id, Reports.vault_api == "0.3.0" + ).order_by(desc(Reports.block)) + result2 = session.exec(query).first() + if result2 == None: + result2 = CHAIN_VALUES[chain.id]["START_BLOCK"] + else: + result2 = 0 + + return result1, result2 + +def get_keeper_payment(tx): + kp3r_token = "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44" + token = contract(kp3r_token) + token = web3.eth.contract(str(kp3r_token), abi=token.abi) + decoded_events = token.events.Transfer().processReceipt(tx) + amount = 0 + for e in decoded_events: + if e.address == kp3r_token: + sender, receiver, token_amount = e.args.values() + if receiver == tx["from"]: + amount = token_amount + return amount + +def compute_apr(report, previous_report): + # ADD pre-fee and post-fee APR + SECONDS_IN_A_YEAR = 31557600 + seconds_between_reports = report.timestamp - previous_report.timestamp + pre_fee_apr = 0 + post_fee_apr = 0 + if int(previous_report.total_debt) == 0 or seconds_between_reports == 0: + return 0, 0 + else: + pre_fee_apr = report.gain / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + if report.gain_post_fees != 0: + post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + return pre_fee_apr, post_fee_apr + +def parse_fees(tx, vault_address, strategy_address): + treasury = CHAIN_VALUES[chain.id]["YEARN_TREASURY"] + token = contract(vault_address) + token = web3.eth.contract(str(vault_address), abi=token.abi) + decoded_events = token.events.Transfer().processReceipt(tx) + amount = 0 + gov_fee_in_underlying = 0 + strategist_fee_in_underlying = 0 + counter = 0 + """ + Using the counter, we will keep track to ensure the expected sequence of fee Transfer events is followed. + Fee transfers always follow this sequence: + 1. mint + 2. transfer to strategy + 3. transfer to treasury + """ + for e in decoded_events: + if e.address == vault_address: + sender, receiver, token_amount = e.args.values() + if sender == ZERO_ADDRESS: + counter = 1 + continue + if receiver == strategy_address and counter == 1: + counter = 2 + strategist_fee_in_underlying = ( + token_amount * ( + contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / + 10 ** contract(vault_address).decimals() + ) + ) + continue + elif counter == 1: + counter = 0 + if receiver == treasury and counter == 2: + counter = 0 + gov_fee_in_underlying = ( + token_amount * ( + contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / + 10 ** contract(vault_address).decimals() + ) + ) + continue + elif counter == 1 or counter == 2: + counter = 0 + return gov_fee_in_underlying, strategist_fee_in_underlying + + +def check_endorsed(strategy_address, block): + lens = contract(CHAIN_VALUES[chain.id]["LENS_ADDRESS"]) + deploy_block = contract_creation_block(strategy_address) + if deploy_block > CHAIN_VALUES[chain.id]["LENS_DEPLOY_BLOCK"]: + # If deployed after lens, we can use lens to lookup + prod_strats = list(lens.assetsStrategiesAddresses.call(block_identifier=block)) + if strategy_address in prod_strats: + return True + else: + return False + else: + # Must lookup using alternate logic. + # Here we make the assumption that if at time of harvest, vault.rewards() != Treasury, it is not endorsed. + # Further, let's use registry helper's list of endorsed vaults to match against. + try: + vault_address = contract(strategy_address).vault() + if contract(vault_address).rewards(block_identifier=block) != CHAIN_VALUES[chain.id]["YEARN_TREASURY"]: + return False + registry_helper = contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]) + vaults = registry_helper.getVaults() + if vault_address in vaults: + return True + else: + return False + except: + return False \ No newline at end of file diff --git a/yearn/db/models.py b/yearn/db/models.py index edefda49b..b4c15ba21 100644 --- a/yearn/db/models.py +++ b/yearn/db/models.py @@ -1,6 +1,9 @@ import os from datetime import datetime from typing import List, Optional +from dotenv import load_dotenv + +load_dotenv() from sqlmodel import ( Column, @@ -33,6 +36,27 @@ class Snapshot(SQLModel, table=True): block_id: int = Field(foreign_key="block.id") block: Block = Relationship(back_populates="snapshots") +class Transactions(SQLModel, table=True): + txn_hash: str = Field(primary_key=True) + chain_id: int + # Transaction fields + block: int + txn_to: str + txn_from: str + txn_gas_used: int + txn_gas_price: int + eth_price_at_block: float + call_cost_usd: float + call_cost_eth: float + kp3r_price_at_block: float + kp3r_paid: int + kp3r_paid_usd: float + keeper_called: bool + # Date fields + date: datetime + date_string: str + timestamp: str + updated_timestamp: datetime pguser = os.environ.get('PGUSER', 'postgres') pgpassword = os.environ.get('PGPASSWORD', 'yearn') @@ -40,6 +64,55 @@ class Snapshot(SQLModel, table=True): pgdatabase = os.environ.get('PGDATABASE', 'yearn') dsn = f'postgresql://{pguser}:{pgpassword}@{pghost}:5432/{pgdatabase}' +reports: List["Reports"] = Relationship(back_populates="txn") + +class Reports(SQLModel, table=True): + id: int = Field(primary_key=True) + chain_id: int + # Transaction fields + block: int + txn_hash: str + txn_hash: str = Field(default=None, foreign_key="transactions.txn_hash") + txn: Transactions = Relationship(back_populates="reports") + # StrategyReported fields + vault_address: str + strategy_address: str + gain: int + loss: int + debt_paid: int + total_gain: int + total_loss: int + total_debt: int + debt_added: int + debt_ratio: int + # Looked-up fields + want_token: str + want_price_at_block: int + want_gain_usd: int + gov_fee_in_want: int + strategist_fee_in_want: int + gain_post_fees: int + rough_apr_pre_fee: float + rough_apr_post_fee: float + vault_api: str + vault_name: str + vault_symbol: str + vault_decimals: int + strategy_name: str + strategy_api: str + previous_report_id: int + multi_harvest: bool + # Date fields + date: datetime + date_string: str + timestamp: str + updated_timestamp: datetime + +user = os.environ.get('POSTGRES_USER') +password = os.environ.get('POSTGRES_PASS') +host = os.environ.get('POSTGRES_HOST') + +dsn = f'postgresql://{user}:{password}@{host}:5432/reports' engine = create_engine(dsn, echo=False) # SQLModel.metadata.drop_all(engine) From 470b61ca975bcc1863f9c64313b10a69487baa24 Mon Sep 17 00:00:00 2001 From: wavey Date: Sat, 29 Jan 2022 22:10:48 -0500 Subject: [PATCH 02/86] chore: remove unused imports --- scripts/collect_reports.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 3828869a7..4caaa0d78 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -1,16 +1,15 @@ import logging import time, os from dotenv import load_dotenv -from datetime import datetime, timedelta, timezone -from itertools import count -from brownie import chain, interface, web3, Contract, ZERO_ADDRESS +from datetime import datetime, timezone +from brownie import chain, web3, Contract, ZERO_ADDRESS from web3._utils.events import construct_event_topic_set -from yearn.utils import closest_block_after_timestamp, contract, contract_creation_block +from yearn.utils import contract, contract_creation_block from yearn.prices import magic, constants from yearn.db.models import Reports, Transactions, Session, engine, select -from sqlalchemy import select as Select, desc, asc +from sqlalchemy import desc, asc from yearn.networks import Network -from yearn.events import create_filter, decode_logs +from yearn.events import decode_logs import warnings warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") warnings.filterwarnings("ignore", ".*It has been discarded*") @@ -135,16 +134,6 @@ def handle_event(event, multi_harvest, isOldApi): txn_hash = event.transaction_hash.hex() tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice - # TODO: Detect if endorsed ✅ # Could use review - # TODO: Lookup last harvest ✅ - # TODO: Lookup last harvest on chain id for each abi type ✅ - # TODO: Block duplicates on insert ✅ - # TODO: Refactor to two tables: Transactions + Reports ✅ - # TODO: Add keep3r payment logic ✅ - # TODO: Add logger - # TODO: Make it multichain compatible (better varible lookups) ✅ - # TODO: Add APY figures ✅ - # TODO: Get hosting ts = chain[event.block_number].timestamp dt = datetime.utcfromtimestamp(ts).strftime("%m/%d/%Y, %H:%M:%S") r = Reports() @@ -159,7 +148,7 @@ def handle_event(event, multi_harvest, isOldApi): return txn_record_exists = transaction_record_exists(txn_hash) - if not transaction_record_exists(txn_hash): + if not txn_record_exists: t = Transactions() t.chain_id = chain.id t.txn_hash = txn_hash From 8eb319f84712ae8a4fbfeb5747abd2460a949106 Mon Sep 17 00:00:00 2001 From: wavey Date: Sat, 29 Jan 2022 23:49:02 -0500 Subject: [PATCH 03/86] fix: add Event object back --- scripts/collect_reports.py | 2 +- yearn/db/models.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 4caaa0d78..d8cbaa2e6 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -6,7 +6,7 @@ from web3._utils.events import construct_event_topic_set from yearn.utils import contract, contract_creation_block from yearn.prices import magic, constants -from yearn.db.models import Reports, Transactions, Session, engine, select +from yearn.db.models import Reports, Event, Transactions, Session, engine, select from sqlalchemy import desc, asc from yearn.networks import Network from yearn.events import decode_logs diff --git a/yearn/db/models.py b/yearn/db/models.py index b4c15ba21..52b35621c 100644 --- a/yearn/db/models.py +++ b/yearn/db/models.py @@ -16,7 +16,16 @@ select, ) - +class Event(object): + isOldApi = False + event = None + txn_hash = "" + multi_harvest = False + def __init__(self, isOldApi, event, txn_hash): + self.isOldApi = isOldApi + self.event = event + self.txn_hash = txn_hash + class Block(SQLModel, table=True): id: int = Field(primary_key=True) chain_id: int From 30228e840205f97e58ef5d4e46a914dfeb9dbf5e Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 15 Feb 2022 08:13:39 -0500 Subject: [PATCH 04/86] chore: prepare rebase --- scripts/collect_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index d8cbaa2e6..705d0b496 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -184,6 +184,7 @@ def handle_event(event, multi_harvest, isOldApi): r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.want_token = strategy.want() + print(r.want_token) r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() r.vault_decimals = vault.decimals() From 05a04eb2ed65f2813959a6a39a32f51e4c3ef11b Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 15 Feb 2022 17:53:42 -0500 Subject: [PATCH 05/86] feat: improve endorsement logic --- scripts/collect_reports.py | 102 ++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 40 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 705d0b496..8f4c3d610 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -12,6 +12,7 @@ from yearn.events import decode_logs import warnings warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") +warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") warnings.filterwarnings("ignore", ".*It has been discarded*") @@ -20,13 +21,14 @@ "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), "START_BLOCK": 11772924, "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", + "REGISTRY_DEPLOY_BLOCK": 12045555, "REGISTRY_HELPER_ADDRESS": "0x52CbF68959e082565e7fd4bBb23D9Ccfb8C8C057", "LENS_ADDRESS": "0x5b4F3BE554a88Bd0f8d8769B9260be865ba03B4a", "LENS_DEPLOY_BLOCK": 12707450, "VAULT_ADDRESS030": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", "KEEPER_CALL_CONTRACT": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", - "KEEPER_TOKEN": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "KEEPER_TOKEN": "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", }, Network.Fantom: { @@ -139,11 +141,11 @@ def handle_event(event, multi_harvest, isOldApi): r = Reports() r.multi_harvest = multi_harvest r.chain_id = chain.id - if isOldApi: - r.strategy_address, r.gain, r.loss, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = event.values() - else: - r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = event.values() - if check_endorsed(r.strategy_address, event.block_number) == False: + r.vault_address = event.address + vault = contract(r.vault_address) + r.vault_decimals = vault.decimals() + r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) + if check_endorsed(r.vault_address, event.block_number) == False: print(f"skipping: not endorsed. strategy {r.strategy_address} txn hash {txn_hash}. chain id {r.chain_id} sync {event.block_number} / {chain.height}.") return @@ -161,12 +163,12 @@ def handle_event(event, multi_harvest, isOldApi): t.call_cost_eth = gas_price * tx.gasUsed / 1e18 t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth if chain.id == 1: - t.kp3r_price_at_block = magic.get_price("0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", t.block) + t.kp3r_price_at_block = magic.get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) t.kp3r_paid = get_keeper_payment(tx) t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block / 1e18 t.keeper_called = t.kp3r_paid > 0 if chain.id == 250: - if t.txn_to == "0x39cAcdb557CA1C4a6555E00203B4a00B1c1a94f8": + if t.txn_to == CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"]: t.keeper_called = True else: t.keeper_called = False @@ -175,20 +177,17 @@ def handle_event(event, multi_harvest, isOldApi): t.timestamp = ts t.updated_timestamp = datetime.now() - r.vault_address = event.address r.block = event.block_number r.txn_hash = txn_hash strategy = contract(r.strategy_address) - vault = contract(r.vault_address) + - r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address) + r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.want_token = strategy.want() - print(r.want_token) r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() - r.vault_decimals = vault.decimals() - r.want_gain_usd = r.gain / 10**r.vault_decimals * r.want_price_at_block + r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() r.strategy_name = strategy.name() r.strategy_api = strategy.apiVersion() @@ -245,14 +244,16 @@ def last_harvest_block(): return result1, result2 def get_keeper_payment(tx): - kp3r_token = "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44" + kp3r_token = CHAIN_VALUES[chain.id]["KEEPER_TOKEN"] token = contract(kp3r_token) + denominator = 10 ** token.decimals() token = web3.eth.contract(str(kp3r_token), abi=token.abi) decoded_events = token.events.Transfer().processReceipt(tx) amount = 0 for e in decoded_events: if e.address == kp3r_token: sender, receiver, token_amount = e.args.values() + token_amount = token_amount / denominator if receiver == tx["from"]: amount = token_amount return amount @@ -271,7 +272,8 @@ def compute_apr(report, previous_report): post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) return pre_fee_apr, post_fee_apr -def parse_fees(tx, vault_address, strategy_address): +def parse_fees(tx, vault_address, strategy_address, decimals): + denominator = 10 ** decimals treasury = CHAIN_VALUES[chain.id]["YEARN_TREASURY"] token = contract(vault_address) token = web3.eth.contract(str(vault_address), abi=token.abi) @@ -290,6 +292,7 @@ def parse_fees(tx, vault_address, strategy_address): for e in decoded_events: if e.address == vault_address: sender, receiver, token_amount = e.args.values() + token_amount = token_amount / denominator if sender == ZERO_ADDRESS: counter = 1 continue @@ -298,7 +301,7 @@ def parse_fees(tx, vault_address, strategy_address): strategist_fee_in_underlying = ( token_amount * ( contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / - 10 ** contract(vault_address).decimals() + denominator ) ) continue @@ -309,7 +312,7 @@ def parse_fees(tx, vault_address, strategy_address): gov_fee_in_underlying = ( token_amount * ( contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / - 10 ** contract(vault_address).decimals() + denominator ) ) continue @@ -318,29 +321,48 @@ def parse_fees(tx, vault_address, strategy_address): return gov_fee_in_underlying, strategist_fee_in_underlying -def check_endorsed(strategy_address, block): - lens = contract(CHAIN_VALUES[chain.id]["LENS_ADDRESS"]) - deploy_block = contract_creation_block(strategy_address) - if deploy_block > CHAIN_VALUES[chain.id]["LENS_DEPLOY_BLOCK"]: - # If deployed after lens, we can use lens to lookup - prod_strats = list(lens.assetsStrategiesAddresses.call(block_identifier=block)) - if strategy_address in prod_strats: - return True - else: - return False +def check_endorsed(vault_address, block): + registry = contract(CHAIN_VALUES[chain.id]["REGISTRY_ADDRESS"]) + harvest_block = block + endorsed_vaults = [] + deploy_block = CHAIN_VALUES[chain.id]["REGISTRY_DEPLOY_BLOCK"] + block = deploy_block if block < deploy_block else block + num_tokens = registry.numTokens(block_identifier=block) + for i in range(0, num_tokens): + t = registry.tokens(i, block_identifier=block) + for n in range(0, 20): + v = registry.vaults(t, n, block_identifier=block) + if v == ZERO_ADDRESS: + break + endorsed_vaults.append(v) + if vault_address in endorsed_vaults: + return True else: - # Must lookup using alternate logic. - # Here we make the assumption that if at time of harvest, vault.rewards() != Treasury, it is not endorsed. - # Further, let's use registry helper's list of endorsed vaults to match against. - try: - vault_address = contract(strategy_address).vault() - if contract(vault_address).rewards(block_identifier=block) != CHAIN_VALUES[chain.id]["YEARN_TREASURY"]: - return False - registry_helper = contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]) - vaults = registry_helper.getVaults() - if vault_address in vaults: + endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) + if vault_address not in endorsed_vaults: + return False + else: + rewards = contract(vault_address, block_identifier=harvest_block).rewards() + if rewards == CHAIN_VALUES[chain.id]["YEARN_TREASURY"]: return True else: return False - except: - return False \ No newline at end of file + +def normalize_event_values(vals, decimals): + denominator = 10**decimals + if len(vals) == 8: + strategy_address, gain, loss, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + debt_paid = 0 + if len(vals) == 9: + strategy_address, gain, loss, debt_paid, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + return ( + strategy_address, + gain/denominator, + loss/denominator, + debt_paid/denominator, + total_gain/denominator, + total_loss/denominator, + total_debt/denominator, + debt_added/denominator, + debt_ratio + ) \ No newline at end of file From cc872152fa7f508b801157b618386fe83c4630ed Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 15 Feb 2022 18:51:04 -0500 Subject: [PATCH 06/86] feat: normalize kp3r payments --- .gitignore | 4 ++++ scripts/collect_reports.py | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 8b3eb0552..0fb613fff 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,8 @@ generated .DS_Store package*.json hardhat.config.js +<<<<<<< HEAD .idea +======= +setup_db.py +>>>>>>> 0c30e13 (feat: normalize kp3r payments) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 8f4c3d610..533f4e392 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -158,14 +158,14 @@ def handle_event(event, multi_harvest, isOldApi): t.txn_to = tx.to t.txn_from = tx["from"] t.txn_gas_used = tx.gasUsed - t.txn_gas_price = gas_price + t.txn_gas_price = gas_price / 1e9 # Use gwei t.eth_price_at_block = magic.get_price(constants.weth, t.block) t.call_cost_eth = gas_price * tx.gasUsed / 1e18 t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth if chain.id == 1: t.kp3r_price_at_block = magic.get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) - t.kp3r_paid = get_keeper_payment(tx) - t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block / 1e18 + t.kp3r_paid = get_keeper_payment(tx) / 1e18 + t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block t.keeper_called = t.kp3r_paid > 0 if chain.id == 250: if t.txn_to == CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"]: @@ -342,7 +342,7 @@ def check_endorsed(vault_address, block): if vault_address not in endorsed_vaults: return False else: - rewards = contract(vault_address, block_identifier=harvest_block).rewards() + rewards = contract(vault_address).rewards(block_identifier=harvest_block) if rewards == CHAIN_VALUES[chain.id]["YEARN_TREASURY"]: return True else: From 19517fc49f5a1c66399d16e9d89f09e4ec3b7b8f Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 15 Feb 2022 23:15:17 -0500 Subject: [PATCH 07/86] feat: added ftm registry block --- scripts/collect_reports.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 533f4e392..fa5fb055e 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -33,8 +33,9 @@ }, Network.Fantom: { "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), - "START_BLOCK": 16241109, + "START_BLOCK": 18450847, "REGISTRY_ADDRESS": "0x727fe1759430df13655ddb0731dE0D0FDE929b04", + "REGISTRY_DEPLOY_BLOCK": 18455565, "REGISTRY_HELPER_ADDRESS": "0x8CC45f739104b3Bdb98BFfFaF2423cC0f817ccc1", "REGISTRY_HELPER_DEPLOY_BLOCK": 18456459, "LENS_ADDRESS": "0x97D0bE2a72fc4Db90eD9Dbc2Ea7F03B4968f6938", @@ -49,6 +50,7 @@ "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), "START_BLOCK": 4841854, "REGISTRY_ADDRESS": "", + "REGISTRY_DEPLOY_BLOCK": 12045555, "REGISTRY_HELPER_ADDRESS": "", "LENS_ADDRESS": "", "VAULT_ADDRESS030": "", @@ -142,7 +144,10 @@ def handle_event(event, multi_harvest, isOldApi): r.multi_harvest = multi_harvest r.chain_id = chain.id r.vault_address = event.address - vault = contract(r.vault_address) + try: + vault = contract(r.vault_address) + except ValueError: + return r.vault_decimals = vault.decimals() r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) if check_endorsed(r.vault_address, event.block_number) == False: @@ -185,6 +190,7 @@ def handle_event(event, multi_harvest, isOldApi): r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.want_token = strategy.want() + print(r.want_token) r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() r.want_gain_usd = r.gain * r.want_price_at_block From 78d5e06d8e87d1c0606d335661c19d8b6414aa4b Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 16 Feb 2022 07:16:07 -0500 Subject: [PATCH 08/86] feat: gitignore cleanup --- .gitignore | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 0fb613fff..6fd0884c5 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,5 @@ generated .DS_Store package*.json hardhat.config.js -<<<<<<< HEAD .idea -======= -setup_db.py ->>>>>>> 0c30e13 (feat: normalize kp3r payments) +setup_db.py \ No newline at end of file From 97db955c29f1c8edc37bd698ce6801c91e4c8f7d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 16 Feb 2022 08:36:52 -0500 Subject: [PATCH 09/86] fix: fix reports model --- yearn/db/models.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/yearn/db/models.py b/yearn/db/models.py index 52b35621c..2da35f4e3 100644 --- a/yearn/db/models.py +++ b/yearn/db/models.py @@ -66,14 +66,8 @@ class Transactions(SQLModel, table=True): date_string: str timestamp: str updated_timestamp: datetime + reports: List["Reports"] = Relationship(back_populates="txn") -pguser = os.environ.get('PGUSER', 'postgres') -pgpassword = os.environ.get('PGPASSWORD', 'yearn') -pghost = os.environ.get('PGHOST', 'localhost') -pgdatabase = os.environ.get('PGDATABASE', 'yearn') - -dsn = f'postgresql://{pguser}:{pgpassword}@{pghost}:5432/{pgdatabase}' -reports: List["Reports"] = Relationship(back_populates="txn") class Reports(SQLModel, table=True): id: int = Field(primary_key=True) @@ -117,10 +111,17 @@ class Reports(SQLModel, table=True): timestamp: str updated_timestamp: datetime + + +pguser = os.environ.get('PGUSER', 'postgres') +pgpassword = os.environ.get('PGPASSWORD', 'yearn') +pghost = os.environ.get('PGHOST', 'localhost') +pgdatabase = os.environ.get('PGDATABASE', 'yearn') +dsn = f'postgresql://{pguser}:{pgpassword}@{pghost}:5432/{pgdatabase}' + user = os.environ.get('POSTGRES_USER') password = os.environ.get('POSTGRES_PASS') host = os.environ.get('POSTGRES_HOST') - dsn = f'postgresql://{user}:{password}@{host}:5432/reports' engine = create_engine(dsn, echo=False) From 8014c14e14cab3de7067f59f4d5d8c9a5b38e7b6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 22 Feb 2022 00:48:11 -0500 Subject: [PATCH 10/86] feat: cache vault endorsement block --- scripts/collect_reports.py | 63 ++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index fa5fb055e..174cf5f27 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -1,6 +1,7 @@ import logging import time, os from dotenv import load_dotenv +from yearn.cache import memory from datetime import datetime, timezone from brownie import chain, web3, Contract, ZERO_ADDRESS from web3._utils.events import construct_event_topic_set @@ -120,22 +121,30 @@ def main(dynamically_find_multi_harvest=False): events_to_process.append(e) for e in events_to_process: - handle_event(e.event, e.multi_harvest, e.isOldApi) + handle_event(e.event, e.multi_harvest) time.sleep(interval_seconds) else: for strategy_report_event in decode_logs(event_filter.get_new_entries()): e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) - handle_event(e.event, e.multi_harvest, e.isOldApi) + handle_event(e.event, e.multi_harvest) if chain.id == 1: # Old vault API exists only on Ethereum mainnet for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) - handle_event(e.event, e.multi_harvest, e.isOldApi) + handle_event(e.event, e.multi_harvest) time.sleep(interval_seconds) -def handle_event(event, multi_harvest, isOldApi): +def handle_event(event, multi_harvest): + endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() + if event.address not in endorsed_vaults: + print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + if get_vault_endorsement_block(event.address) > event.block_number: + print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice ts = chain[event.block_number].timestamp @@ -150,9 +159,7 @@ def handle_event(event, multi_harvest, isOldApi): return r.vault_decimals = vault.decimals() r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) - if check_endorsed(r.vault_address, event.block_number) == False: - print(f"skipping: not endorsed. strategy {r.strategy_address} txn hash {txn_hash}. chain id {r.chain_id} sync {event.block_number} / {chain.height}.") - return + txn_record_exists = transaction_record_exists(txn_hash) if not txn_record_exists: @@ -190,7 +197,6 @@ def handle_event(event, multi_harvest, isOldApi): r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.want_token = strategy.want() - print(r.want_token) r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() r.want_gain_usd = r.gain * r.want_price_at_block @@ -326,33 +332,24 @@ def parse_fees(tx, vault_address, strategy_address, decimals): counter = 0 return gov_fee_in_underlying, strategist_fee_in_underlying - -def check_endorsed(vault_address, block): +@memory.cache() +def get_vault_endorsement_block(vault_address): + token = contract(vault_address).token() registry = contract(CHAIN_VALUES[chain.id]["REGISTRY_ADDRESS"]) - harvest_block = block - endorsed_vaults = [] - deploy_block = CHAIN_VALUES[chain.id]["REGISTRY_DEPLOY_BLOCK"] - block = deploy_block if block < deploy_block else block - num_tokens = registry.numTokens(block_identifier=block) - for i in range(0, num_tokens): - t = registry.tokens(i, block_identifier=block) - for n in range(0, 20): - v = registry.vaults(t, n, block_identifier=block) - if v == ZERO_ADDRESS: - break - endorsed_vaults.append(v) - if vault_address in endorsed_vaults: - return True - else: - endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) - if vault_address not in endorsed_vaults: - return False - else: - rewards = contract(vault_address).rewards(block_identifier=harvest_block) - if rewards == CHAIN_VALUES[chain.id]["YEARN_TREASURY"]: - return True + height = chain.height + lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height + + while hi - lo > 1: + mid = lo + (hi - lo) // 2 + try: + num_vaults = registry.numVaults(token, block_identifier=mid) + if registry.vaults(token, num_vaults-1, block_identifier=mid) == vault_address: + hi = mid else: - return False + lo = mid + except: + lo = mid + return hi def normalize_event_values(vals, decimals): denominator = 10**decimals From 80450a42c9f9f14c36a9431a9c65596711f0bd43 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 24 Feb 2022 16:44:01 -0500 Subject: [PATCH 11/86] feat: add discord message support --- scripts/collect_reports.py | 153 +++++++++++++++++++++++++++++++++---- yearn/db/models.py | 2 + 2 files changed, 139 insertions(+), 16 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 174cf5f27..d2a24bf7a 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -1,7 +1,10 @@ import logging import time, os +import telebot +from discordwebhook import Discord from dotenv import load_dotenv from yearn.cache import memory +import pandas as pd from datetime import datetime, timezone from brownie import chain, web3, Contract, ZERO_ADDRESS from web3._utils.events import construct_event_topic_set @@ -16,21 +19,48 @@ warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") warnings.filterwarnings("ignore", ".*It has been discarded*") +telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') +mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') +discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') +discord_ftm = os.environ.get('DISCORD_CHANNEL_250') +bot = telebot.TeleBot(telegram_key) +alerts_enabled = True + +OLD_REGISTRY_ENDORSEMENT_BLOCKS = { + "0xE14d13d8B3b85aF791b2AADD661cDBd5E6097Db1": 11999957, + "0xdCD90C7f6324cfa40d7169ef80b12031770B4325": 11720423, + "0x986b4AFF588a109c09B50A03f42E4110E29D353F": 11881934, + "0xcB550A6D4C8e3517A939BC79d0c7093eb7cF56B5": 11770630, + "0xa9fE4601811213c340e850ea305481afF02f5b28": 11927501, + "0xB8C3B7A2A618C552C23B1E4701109a9E756Bab67": 12019352, + "0xBFa4D8AA6d8a379aBFe7793399D3DdaCC5bBECBB": 11579535, + "0x19D3364A399d251E894aC732651be8B0E4e85001": 11682465, + "0xe11ba472F74869176652C35D30dB89854b5ae84D": 11631914, + "0xe2F6b9773BF3A015E2aA70741Bde1498bdB9425b": 11579535, + "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9": 11682465, + "0x27b7b1ad7288079A66d12350c828D3C00A6F07d7": 12089661, +} + CHAIN_VALUES = { Network.Mainnet: { "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), - "START_BLOCK": 11772924, + "START_BLOCK": 11563389, "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", "REGISTRY_DEPLOY_BLOCK": 12045555, "REGISTRY_HELPER_ADDRESS": "0x52CbF68959e082565e7fd4bBb23D9Ccfb8C8C057", "LENS_ADDRESS": "0x5b4F3BE554a88Bd0f8d8769B9260be865ba03B4a", "LENS_DEPLOY_BLOCK": 12707450, - "VAULT_ADDRESS030": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "VAULT_ADDRESS030": "0x19D3364A399d251E894aC732651be8B0E4e85001", "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", "KEEPER_CALL_CONTRACT": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", "KEEPER_TOKEN": "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", + "STRATEGIST_MULTISIG": "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", + "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", + "EXPLORER_URL": "https://etherscan.io/", }, Network.Fantom: { "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), @@ -46,6 +76,9 @@ "KEEPER_CALL_CONTRACT": "0x39cAcdb557CA1C4a6555E00203B4a00B1c1a94f8", "KEEPER_TOKEN": "", "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", + "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", + "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", + "EXPLORER_URL": "https://ftmscan.com/", }, Network.Arbitrum: { "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), @@ -59,6 +92,8 @@ "KEEPER_CALL_CONTRACT": "", "KEEPER_TOKEN": "", "YEARN_TREASURY": "", + "STRATEGIST_MULTISIG": "", + "GOVERNANCE_MULTISIG": "", } } @@ -71,6 +106,7 @@ ) # Deprecated vault interface if chain.id == 1: + print(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) vault_v030 = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) vault_v030 = web3.eth.contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"], abi=vault_v030.abi) topics_v030 = construct_event_topic_set( @@ -87,7 +123,7 @@ def main(dynamically_find_multi_harvest=False): print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) if chain.id == 1: print("latest block (v0.3.0 API)",last_reported_block030) - print("blocks behind (v0.3.0+ API)", chain.height - last_reported_block030) + print("blocks behind (v0.3.0 API)", chain.height - last_reported_block030) event_filter = web3.eth.filter({'topics': topics, "fromBlock": last_reported_block + 1}) if chain.id == 1: event_filter_v030 = web3.eth.filter({'topics': topics_v030, "fromBlock": last_reported_block030 + 1}) @@ -160,9 +196,9 @@ def handle_event(event, multi_harvest): r.vault_decimals = vault.decimals() r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) - - txn_record_exists = transaction_record_exists(txn_hash) - if not txn_record_exists: + txn_record_exists = False + t = transaction_record_exists(txn_hash) + if not t: t = Transactions() t.chain_id = chain.id t.txn_hash = txn_hash @@ -188,7 +224,8 @@ def handle_event(event, multi_harvest): t.date_string = dt t.timestamp = ts t.updated_timestamp = datetime.now() - + else: + txn_record_exists = True r.block = event.block_number r.txn_hash = txn_hash strategy = contract(r.strategy_address) @@ -196,6 +233,7 @@ def handle_event(event, multi_harvest): r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want + r.token_symbol = contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() @@ -203,6 +241,7 @@ def handle_event(event, multi_harvest): r.vault_name = vault.name() r.strategy_name = strategy.name() r.strategy_api = strategy.apiVersion() + r.strategist = strategy.strategist() r.vault_symbol = vault.symbol() r.date = datetime.utcfromtimestamp(ts) r.date_string = dt @@ -219,11 +258,19 @@ def handle_event(event, multi_harvest): r.previous_report_id = previous_report_id r.rough_apr_pre_fee, r.rough_apr_post_fee = compute_apr(r, previous_report) # Insert to database - session.add(r) - if not txn_record_exists: - session.add(t) - session.commit() - print(f"report added. strategy {r.strategy_address} txn hash {r.txn_hash}. chain id {r.chain_id} sync {r.block} / {chain.height}.") + insert_success = False + try: + session.add(r) + if not txn_record_exists: + session.add(t) + session.commit() + print(f"report added. strategy {r.strategy_address} txn hash {r.txn_hash}. chain id {r.chain_id} sync {r.block} / {chain.height}.") + insert_success = True + except: + print(f"skipped duplicate record. strategy: {r.strategy_address} at tx hash: {r.txn_hash}") + pass + if insert_success: + prepare_alerts(r, t) def transaction_record_exists(txn_hash): with Session(engine) as session: @@ -233,7 +280,7 @@ def transaction_record_exists(txn_hash): result = session.exec(query).first() if result == None: return False - return True + return result def last_harvest_block(): with Session(engine) as session: @@ -271,7 +318,6 @@ def get_keeper_payment(tx): return amount def compute_apr(report, previous_report): - # ADD pre-fee and post-fee APR SECONDS_IN_A_YEAR = 31557600 seconds_between_reports = report.timestamp - previous_report.timestamp pre_fee_apr = 0 @@ -335,10 +381,14 @@ def parse_fees(tx, vault_address, strategy_address, decimals): @memory.cache() def get_vault_endorsement_block(vault_address): token = contract(vault_address).token() + try: + block = OLD_REGISTRY_ENDORSEMENT_BLOCKS[vault_address] + return block + except KeyError: + pass registry = contract(CHAIN_VALUES[chain.id]["REGISTRY_ADDRESS"]) height = chain.height lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height - while hi - lo > 1: mid = lo + (hi - lo) // 2 try: @@ -368,4 +418,75 @@ def normalize_event_values(vals, decimals): total_debt/denominator, debt_added/denominator, debt_ratio - ) \ No newline at end of file + ) + +def prepare_alerts(r, t): + if alerts_enabled: + m = format_public_telegram(r, t) + # Send to chain specific channels + if chain.id == 1: + bot.send_message(mainnet_public_channel, m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=discord_mainnet) + if chain.id == 250: + bot.send_message(ftm_public_channel, m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=discord_ftm) + discord.post( + embeds=[{ + "title": "New harvest", + "description": m + }], + ) + + # Send to dev channel + m = m + format_dev_telegram(r, t) + bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + +def format_public_telegram(r, t): + explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] + sms = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] + gov = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] + from_indicator = "" + + if t.txn_from == sms or t.txn_from == gov: + from_indicator = "✍ " + + elif t.txn_from == r.strategist: + from_indicator = "🧠 " + + elif t.keeper_called: + from_indicator = "🤖 " + + message = "" + message += from_indicator + message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' + message += f'📅 {r.date_string} UTC \n\n' + net_profit_want = "{:,.2f}".format(r.gain - r.loss) + net_profit_usd = "{:,.2f}".format(r.want_gain_usd) + message += f'💰 Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n\n' + txn_cost_str = "${:,.2f}".format(t.call_cost_usd) + message += f'💸 Transaction Cost: {txn_cost_str} \n\n' + message += f'🔗 [View on Explorer]({explorer}tx/{r.txn_hash})' + if r.multi_harvest: + message += "\n\n_part of a single txn with multiple harvests_" + print(message) + return message + +def format_dev_telegram(r, t): + message = '\n\n' + df = pd.DataFrame(index=['']) + df[r.vault_name + " " + r.vault_api] = r.vault_address + df["Strategy Address"] = r.strategy_address + df["Gain"] = "{:,.2f}".format(r.gain) + " (" + "${:,.2f}".format(r.gain * r.want_price_at_block) + ")" + df["Loss"] = "{:,.2f}".format(r.loss) + " (" + "${:,.2f}".format(r.loss * r.want_price_at_block) + ")" + df["Debt Paid"] = "{:,.2f}".format(r.debt_paid) + " (" + "${:,.2f}".format(r.debt_paid * r.want_price_at_block) + ")" + df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " (" + "${:,.2f}".format(r.debt_added * r.want_price_at_block) + ")" + df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " (" + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + ")" + df["Debt Ratio"] = r.debt_ratio + df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" + df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" + df["Pre-fee APR"] = "{:.2%}".format(r.rough_apr_pre_fee) + df["Post-fee APR"] = "{:.2%}".format(r.rough_apr_post_fee) + message2 = f"```{df.T.to_string()}\n```" + return message + message2 + + diff --git a/yearn/db/models.py b/yearn/db/models.py index 2da35f4e3..4e772a94d 100644 --- a/yearn/db/models.py +++ b/yearn/db/models.py @@ -90,6 +90,7 @@ class Reports(SQLModel, table=True): debt_ratio: int # Looked-up fields want_token: str + token_symbol: str want_price_at_block: int want_gain_usd: int gov_fee_in_want: int @@ -103,6 +104,7 @@ class Reports(SQLModel, table=True): vault_decimals: int strategy_name: str strategy_api: str + strategist: str previous_report_id: int multi_harvest: bool # Date fields From ac214c5da8b8333fe84d7ace6780ca1be9cbe1e8 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 24 Feb 2022 16:49:24 -0500 Subject: [PATCH 12/86] fix: point to proper mainnet chan --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index d2a24bf7a..e38d17ff4 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -20,7 +20,7 @@ warnings.filterwarnings("ignore", ".*It has been discarded*") telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') -mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') From 21016fd1ef441927333bf72d6541159e5e6e0867 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 24 Feb 2022 17:30:35 -0500 Subject: [PATCH 13/86] feat: caller emojis --- scripts/collect_reports.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index e38d17ff4..452229138 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -55,7 +55,7 @@ "LENS_DEPLOY_BLOCK": 12707450, "VAULT_ADDRESS030": "0x19D3364A399d251E894aC732651be8B0E4e85001", "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", - "KEEPER_CALL_CONTRACT": "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9", + "KEEPER_CALL_CONTRACT": "0x2150b45626199CFa5089368BDcA30cd0bfB152D6", "KEEPER_TOKEN": "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", "STRATEGIST_MULTISIG": "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", @@ -73,7 +73,7 @@ "LENS_DEPLOY_BLOCK": 18842673, "VAULT_ADDRESS030": "0x637eC617c86D24E421328e6CAEa1d92114892439", "VAULT_ADDRESS031": "0x637eC617c86D24E421328e6CAEa1d92114892439", - "KEEPER_CALL_CONTRACT": "0x39cAcdb557CA1C4a6555E00203B4a00B1c1a94f8", + "KEEPER_CALL_CONTRACT": "0x000004e4d96d663C809Cbc8D773a764A89D0b37f", "KEEPER_TOKEN": "", "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", @@ -438,22 +438,23 @@ def prepare_alerts(r, t): ) # Send to dev channel - m = m + format_dev_telegram(r, t) + m = f"CHAIN {chain.id}\n\n" + m + format_dev_telegram(r, t) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] sms = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] - gov = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] + gov = CHAIN_VALUES[chain.id]["GOVERNANCE_MULTISIG"] + keeper = CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"] from_indicator = "" - if t.txn_from == sms or t.txn_from == gov: + if t.txn_to == sms or t.txn_to == gov: from_indicator = "✍ " - elif t.txn_from == r.strategist: + elif t.txn_from == r.strategist and t.txn_to != sms: from_indicator = "🧠 " - elif t.keeper_called: + elif t.keeper_called or t.txn_from == keeper or t.txn_to == keeper: from_indicator = "🤖 " message = "" From 32b3512cb33e47cfa154aca49b04ec42ae4eb9a8 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 24 Feb 2022 17:31:25 -0500 Subject: [PATCH 14/86] feat: chain id text --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 452229138..248a0cca0 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -438,7 +438,7 @@ def prepare_alerts(r, t): ) # Send to dev channel - m = f"CHAIN {chain.id}\n\n" + m + format_dev_telegram(r, t) + m = f"Chain ID: {chain.id}\n\n" + m + format_dev_telegram(r, t) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): From 6640c55fe324bf94fb318918303e0f96689cfe4a Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 25 Feb 2022 16:01:18 -0500 Subject: [PATCH 15/86] feat: add time since last report --- scripts/collect_reports.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 248a0cca0..eaca7aae3 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -26,7 +26,7 @@ discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') discord_ftm = os.environ.get('DISCORD_CHANNEL_250') bot = telebot.TeleBot(telegram_key) -alerts_enabled = True +alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False OLD_REGISTRY_ENDORSEMENT_BLOCKS = { "0xE14d13d8B3b85aF791b2AADD661cDBd5E6097Db1": 11999957, @@ -439,6 +439,7 @@ def prepare_alerts(r, t): # Send to dev channel m = f"Chain ID: {chain.id}\n\n" + m + format_dev_telegram(r, t) + print(f"Chain ID: {chain.id}\n\n" + m + format_dev_telegram(r, t)) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): @@ -469,14 +470,20 @@ def format_public_telegram(r, t): message += f'🔗 [View on Explorer]({explorer}tx/{r.txn_hash})' if r.multi_harvest: message += "\n\n_part of a single txn with multiple harvests_" - print(message) return message def format_dev_telegram(r, t): message = '\n\n' df = pd.DataFrame(index=['']) + last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] + if last_harvest_ts == 0: + time_since_last_report = "n/a" + else: + seconds_since_report = int(time.time() - last_harvest_ts) + time_since_last_report = "%dd, %dhr, %dm" % dhms_from_seconds(seconds_since_report) df[r.vault_name + " " + r.vault_api] = r.vault_address df["Strategy Address"] = r.strategy_address + df["Last Report"] = time_since_last_report df["Gain"] = "{:,.2f}".format(r.gain) + " (" + "${:,.2f}".format(r.gain * r.want_price_at_block) + ")" df["Loss"] = "{:,.2f}".format(r.loss) + " (" + "${:,.2f}".format(r.loss * r.want_price_at_block) + ")" df["Debt Paid"] = "{:,.2f}".format(r.debt_paid) + " (" + "${:,.2f}".format(r.debt_paid * r.want_price_at_block) + ")" @@ -490,4 +497,8 @@ def format_dev_telegram(r, t): message2 = f"```{df.T.to_string()}\n```" return message + message2 - +def dhms_from_seconds(seconds): + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + days, hours = divmod(hours, 24) + return (days, hours, minutes) From 1d15195581b0a6d5ab4709334049ed34809ceee7 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 25 Feb 2022 20:58:40 -0500 Subject: [PATCH 16/86] fix: handle None type on apr fields --- scripts/collect_reports.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index eaca7aae3..8f05658e1 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -492,8 +492,14 @@ def format_dev_telegram(r, t): df["Debt Ratio"] = r.debt_ratio df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" - df["Pre-fee APR"] = "{:.2%}".format(r.rough_apr_pre_fee) - df["Post-fee APR"] = "{:.2%}".format(r.rough_apr_post_fee) + prefee = "n/a" + postfee = "n/a" + if r.rough_apr_pre_fee is not None: + prefee = "{:.2%}".format(r.rough_apr_pre_fee) + if r.rough_apr_post_fee is not None: + postfee = "{:.2%}".format(r.rough_apr_post_fee) + df["Pre-fee APR"] = prefee + df["Post-fee APR"] = postfee message2 = f"```{df.T.to_string()}\n```" return message + message2 From 7a0f76e578f15af3ead1d749f7b5f6506d5e0925 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 28 Feb 2022 18:11:35 -0500 Subject: [PATCH 17/86] feat: add daily harvest report script --- scripts/daily_harvest_report.py | 92 +++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/daily_harvest_report.py diff --git a/scripts/daily_harvest_report.py b/scripts/daily_harvest_report.py new file mode 100644 index 000000000..19974c647 --- /dev/null +++ b/scripts/daily_harvest_report.py @@ -0,0 +1,92 @@ +import time, os +from datetime import datetime +import telebot +from discordwebhook import Discord +from yearn.db.models import Reports, Event, Transactions, Session, engine, select +from sqlalchemy import desc, asc + +telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') +mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') +ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') +discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') +discord_ftm = os.environ.get('DISCORD_CHANNEL_250') +discord_daily_report = os.environ.get('DISCORD_CHANNEL_DAILY_REPORT') +discord = Discord(url=discord_daily_report) +bot = telebot.TeleBot(telegram_key) +alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False + +RESULTS = { + 1: { + "network_symbol": "ETH", + "network_name": "Ethereum", + "telegram_channel": mainnet_public_channel, + "profit_usd": 0, + "num_harvests": 0, + "txn_cost_eth": 0, + "txn_cost_usd": 0, + "message": "" + }, + 250: { + "network_symbol": "FTM", + "network_name": "Fantom", + "telegram_channel": ftm_public_channel, + "profit_usd": 0, + "num_harvests": 0, + "txn_cost_eth": 0, + "txn_cost_usd": 0, + "message": "" + }, + 42161: { + "network_symbol": "ARRB", + "network_name": "Arbitrum", + "telegram_channel": 0, + "profit_usd": 0, + "num_harvests": 0, + "txn_cost_eth": 0, + "txn_cost_usd": 0, + "message": "" + } +} + +def main(): + DAY_IN_SECONDS = 60 * 60 * 24 + current_time = int(time.time()) + yesterday = current_time - DAY_IN_SECONDS + with Session(engine) as session: + query = select(Reports, Transactions).join(Transactions).where( + Reports.timestamp > yesterday + ).order_by(desc(Reports.block)) + results = session.exec(query) + for report, txn in results: + RESULTS[txn.chain_id]["profit_usd"] = RESULTS[txn.chain_id]["profit_usd"] + report.want_gain_usd + RESULTS[txn.chain_id]["num_harvests"] = RESULTS[txn.chain_id]["num_harvests"] + 1 + RESULTS[txn.chain_id]["txn_cost_eth"] = RESULTS[txn.chain_id]["txn_cost_eth"] + txn.call_cost_eth + RESULTS[txn.chain_id]["txn_cost_usd"] = RESULTS[txn.chain_id]["txn_cost_usd"] + txn.call_cost_usd + # Build Messages + cumulative_message = "" + for chain in RESULTS.keys(): + print(RESULTS[chain]["network_symbol"]) + + message = f'📃 End of Day Report --- {datetime.utcfromtimestamp(current_time - 1000).strftime("%m-%d-%Y")} \n\n' + message += f'💰 ${"{:,.2f}".format(RESULTS[chain]["profit_usd"])} harvested\n\n' + message += f'💸 ${"{:,.2f}".format(RESULTS[chain]["txn_cost_usd"])} in transaction fees\n\n' + message += f'👨‍🌾 {RESULTS[chain]["num_harvests"]} strategies harvested' + + cumulative_message += f'--- {RESULTS[chain]["network_name"]} ---' + cumulative_message += f'\n💰 ${"{:,.2f}".format(RESULTS[chain]["profit_usd"])} harvested' + cumulative_message += f'\n💸 ${"{:,.2f}".format(RESULTS[chain]["txn_cost_usd"])} in transaction fees' + cumulative_message += f'\n👨‍🌾 {RESULTS[chain]["num_harvests"]} strategies harvested\n\n' + RESULTS[chain]["message"] = message + channel = RESULTS[chain]["telegram_channel"] + print() + if channel != 0: + bot.send_message(channel, message, parse_mode="markdown", disable_web_page_preview = True) + print(message) + date_banner = f'📃 End of Day Report --- {datetime.utcfromtimestamp(current_time - 1000).strftime("%m-%d-%Y")} \n\n' + discord.post( + embeds=[{ + "title": date_banner, + "description": cumulative_message + }], + ) \ No newline at end of file From 828995cefe800d87cca727ffb000a7f669e6f80e Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 28 Feb 2022 18:25:59 -0500 Subject: [PATCH 18/86] feat: add cumulative report to dev channel --- scripts/daily_harvest_report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/daily_harvest_report.py b/scripts/daily_harvest_report.py index 19974c647..9acd09f12 100644 --- a/scripts/daily_harvest_report.py +++ b/scripts/daily_harvest_report.py @@ -89,4 +89,5 @@ def main(): "title": date_banner, "description": cumulative_message }], - ) \ No newline at end of file + ) + bot.send_message(dev_channel, date_banner + cumulative_message, parse_mode="markdown", disable_web_page_preview = True) \ No newline at end of file From 6ea33182e89fd363209fb45d4c54c20453d75604 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 11 Mar 2022 13:21:37 -0500 Subject: [PATCH 19/86] feat: add tenderly link and chain identifiers on messages --- scripts/collect_reports.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 8f05658e1..6dc9af696 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -46,6 +46,9 @@ CHAIN_VALUES = { Network.Mainnet: { + "NETWORK_NAME": "Ethereum Mainnet", + "NETWORK_SYMBOL": "ETH", + "EMOJI": "🇪🇹", "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), "START_BLOCK": 11563389, "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", @@ -63,6 +66,9 @@ "EXPLORER_URL": "https://etherscan.io/", }, Network.Fantom: { + "NETWORK_NAME": "Fantom", + "NETWORK_SYMBOL": "FTM", + "EMOJI": "👻", "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), "START_BLOCK": 18450847, "REGISTRY_ADDRESS": "0x727fe1759430df13655ddb0731dE0D0FDE929b04", @@ -81,19 +87,22 @@ "EXPLORER_URL": "https://ftmscan.com/", }, Network.Arbitrum: { + "NETWORK_NAME": "Arbitrum", + "NETWORK_SYMBOL": "ARRB", + "EMOJI": "🤠", "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), "START_BLOCK": 4841854, "REGISTRY_ADDRESS": "", "REGISTRY_DEPLOY_BLOCK": 12045555, - "REGISTRY_HELPER_ADDRESS": "", - "LENS_ADDRESS": "", - "VAULT_ADDRESS030": "", - "VAULT_ADDRESS031": "", + "REGISTRY_HELPER_ADDRESS": "0x237C3623bed7D115Fc77fEB08Dd27E16982d972B", + "LENS_ADDRESS": "0xcAd10033C86B0C1ED6bfcCAa2FF6779938558E9f", + "VAULT_ADDRESS030": "0x239e14A19DFF93a17339DCC444f74406C17f8E67", + "VAULT_ADDRESS031": "0x239e14A19DFF93a17339DCC444f74406C17f8E67", "KEEPER_CALL_CONTRACT": "", "KEEPER_TOKEN": "", - "YEARN_TREASURY": "", - "STRATEGIST_MULTISIG": "", - "GOVERNANCE_MULTISIG": "", + "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", + "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", + "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", } } @@ -172,6 +181,7 @@ def main(dynamically_find_multi_harvest=False): time.sleep(interval_seconds) def handle_event(event, multi_harvest): + # exception because skeletor didnt verify contract endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() if event.address not in endorsed_vaults: @@ -438,8 +448,8 @@ def prepare_alerts(r, t): ) # Send to dev channel - m = f"Chain ID: {chain.id}\n\n" + m + format_dev_telegram(r, t) - print(f"Chain ID: {chain.id}\n\n" + m + format_dev_telegram(r, t)) + m = f'Network: {CHAIN_VALUES[chain.id]["NETWORK_EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + print(f'Chain ID: {chain.id}\n\n' + m + format_dev_telegram(r, t)) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): @@ -463,7 +473,7 @@ def format_public_telegram(r, t): message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' message += f'📅 {r.date_string} UTC \n\n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) - net_profit_usd = "{:,.2f}".format(r.want_gain_usd) + net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) message += f'💰 Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) message += f'💸 Transaction Cost: {txn_cost_str} \n\n' @@ -473,7 +483,7 @@ def format_public_telegram(r, t): return message def format_dev_telegram(r, t): - message = '\n\n' + message = '/ [Tenderly](https://dashboard.tenderly.co/yearn/sms/tx/{r.chain_id}/{r.txn_hash})\n\n' df = pd.DataFrame(index=['']) last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] if last_harvest_ts == 0: From 49b475250352d5450a438d03199b9c4c3cf50c68 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 11 Mar 2022 13:58:50 -0500 Subject: [PATCH 20/86] fix: emoji name fix --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 6dc9af696..327ee42ed 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -448,7 +448,7 @@ def prepare_alerts(r, t): ) # Send to dev channel - m = f'Network: {CHAIN_VALUES[chain.id]["NETWORK_EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) print(f'Chain ID: {chain.id}\n\n' + m + format_dev_telegram(r, t)) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) From 19e0abe534565fb11fc1a68f9b0c2856d7fc299d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 11 Mar 2022 15:38:30 -0500 Subject: [PATCH 21/86] fix: tenderly url --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 327ee42ed..4ab4344e8 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -483,7 +483,7 @@ def format_public_telegram(r, t): return message def format_dev_telegram(r, t): - message = '/ [Tenderly](https://dashboard.tenderly.co/yearn/sms/tx/{r.chain_id}/{r.txn_hash})\n\n' + message = f' / [Tenderly](https://dashboard.tenderly.co/yearn/sms/tx/{r.chain_id}/{r.txn_hash})\n\n' df = pd.DataFrame(index=['']) last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] if last_harvest_ts == 0: From 41f121d6e19299b37867302d2edf4fd31e023b92 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 11 Mar 2022 20:25:36 -0500 Subject: [PATCH 22/86] fix: tenderly url string --- scripts/collect_reports.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 4ab4344e8..677d6ac11 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -64,6 +64,7 @@ "STRATEGIST_MULTISIG": "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", "EXPLORER_URL": "https://etherscan.io/", + "TENDERLY_CHAIN_IDENTIFIER": "mainnet", }, Network.Fantom: { "NETWORK_NAME": "Fantom", @@ -85,6 +86,7 @@ "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", "EXPLORER_URL": "https://ftmscan.com/", + "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", }, Network.Arbitrum: { "NETWORK_NAME": "Arbitrum", @@ -103,6 +105,7 @@ "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", + "TENDERLY_CHAIN_IDENTIFIER": "", } } @@ -483,7 +486,8 @@ def format_public_telegram(r, t): return message def format_dev_telegram(r, t): - message = f' / [Tenderly](https://dashboard.tenderly.co/yearn/sms/tx/{r.chain_id}/{r.txn_hash})\n\n' + tenderly_str = CHAIN_VALUES[chain.id]["TENDERLY_CHAIN_IDENTIFIER"] + message = f' / [Tenderly](https://dashboard.tenderly.co/tx/{tenderly_str}/{r.txn_hash})\n\n' df = pd.DataFrame(index=['']) last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] if last_harvest_ts == 0: From ffdb5728c9b618ab833458db1dd46d4c8fc5cb00 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 11 Mar 2022 21:13:06 -0500 Subject: [PATCH 23/86] fix: tenderly url string --- scripts/collect_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 677d6ac11..ef36a028a 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -86,7 +86,7 @@ "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", "EXPLORER_URL": "https://ftmscan.com/", - "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", + "TENDERLY_CHAIN_IDENTIFIER": "fantom", }, Network.Arbitrum: { "NETWORK_NAME": "Arbitrum", @@ -105,7 +105,7 @@ "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", - "TENDERLY_CHAIN_IDENTIFIER": "", + "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", } } From f78da8993d418d57fdfa91e88358d7fa425df632 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 17 Mar 2022 14:18:18 -0400 Subject: [PATCH 24/86] feat: arbi support --- brownie-config.yaml | 1 + scripts/collect_reports.py | 32 ++++++++++++++++++-------------- yearn/utils.py | 14 +++++++++++++- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/brownie-config.yaml b/brownie-config.yaml index a9f0e2928..8f863923c 100644 --- a/brownie-config.yaml +++ b/brownie-config.yaml @@ -7,3 +7,4 @@ compiler: solc: use_latest_patch: - '0x514910771AF9Ca656af840dff83E8264EcF986CA' + - '0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f' diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index ef36a028a..382b22dc2 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -19,14 +19,16 @@ warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") warnings.filterwarnings("ignore", ".*It has been discarded*") + +# mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') +# ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +# discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') +# discord_ftm = os.environ.get('DISCORD_CHANNEL_250') + telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') -mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') -ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') -dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') -discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') -discord_ftm = os.environ.get('DISCORD_CHANNEL_250') bot = telebot.TeleBot(telegram_key) alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False +dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') OLD_REGISTRY_ENDORSEMENT_BLOCKS = { "0xE14d13d8B3b85aF791b2AADD661cDBd5E6097Db1": 11999957, @@ -65,6 +67,8 @@ "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", "EXPLORER_URL": "https://etherscan.io/", "TENDERLY_CHAIN_IDENTIFIER": "mainnet", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_1'), }, Network.Fantom: { "NETWORK_NAME": "Fantom", @@ -80,13 +84,15 @@ "LENS_DEPLOY_BLOCK": 18842673, "VAULT_ADDRESS030": "0x637eC617c86D24E421328e6CAEa1d92114892439", "VAULT_ADDRESS031": "0x637eC617c86D24E421328e6CAEa1d92114892439", - "KEEPER_CALL_CONTRACT": "0x000004e4d96d663C809Cbc8D773a764A89D0b37f", + "KEEPER_CALL_CONTRACT": "0x57419fb50fa588fc165acc26449b2bf4c7731458", "KEEPER_TOKEN": "", "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", "EXPLORER_URL": "https://ftmscan.com/", "TENDERLY_CHAIN_IDENTIFIER": "fantom", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_250'), }, Network.Arbitrum: { "NETWORK_NAME": "Arbitrum", @@ -94,7 +100,7 @@ "EMOJI": "🤠", "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), "START_BLOCK": 4841854, - "REGISTRY_ADDRESS": "", + "REGISTRY_ADDRESS": "0x3199437193625DCcD6F9C9e98BDf93582200Eb1f", "REGISTRY_DEPLOY_BLOCK": 12045555, "REGISTRY_HELPER_ADDRESS": "0x237C3623bed7D115Fc77fEB08Dd27E16982d972B", "LENS_ADDRESS": "0xcAd10033C86B0C1ED6bfcCAa2FF6779938558E9f", @@ -106,6 +112,8 @@ "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_42161_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_42161'), } } @@ -228,7 +236,7 @@ def handle_event(event, multi_harvest): t.kp3r_paid = get_keeper_payment(tx) / 1e18 t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block t.keeper_called = t.kp3r_paid > 0 - if chain.id == 250: + else: if t.txn_to == CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"]: t.keeper_called = True else: @@ -437,12 +445,8 @@ def prepare_alerts(r, t): if alerts_enabled: m = format_public_telegram(r, t) # Send to chain specific channels - if chain.id == 1: - bot.send_message(mainnet_public_channel, m, parse_mode="markdown", disable_web_page_preview = True) - discord = Discord(url=discord_mainnet) - if chain.id == 250: - bot.send_message(ftm_public_channel, m, parse_mode="markdown", disable_web_page_preview = True) - discord = Discord(url=discord_ftm) + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) discord.post( embeds=[{ "title": "New harvest", diff --git a/yearn/utils.py b/yearn/utils.py index b991eb74d..f3a043c16 100644 --- a/yearn/utils.py +++ b/yearn/utils.py @@ -2,7 +2,7 @@ from functools import lru_cache import threading -from brownie import Contract, chain, web3 +from brownie import Contract, chain, web3, interface from yearn.cache import memory from yearn.exceptions import ArchiveNodeRequired @@ -16,6 +16,13 @@ Network.Arbitrum: 0, } +_erc20 = lru_cache(maxsize=None)(interface.ERC20) + +PREFER_INTERFACE = { + Network.Arbitrum: { + "0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f": _erc20, # empty ABI for WBTC when compiling the contract + } +} def safe_views(abi): return [ @@ -120,6 +127,11 @@ def __call__(self, *args, **kwargs): def contract(address): with _contract_lock: + if chain.id in PREFER_INTERFACE: + if address in PREFER_INTERFACE[chain.id]: + _interface = PREFER_INTERFACE[chain.id][address] + return _interface(address) + return _contract(address) From 6ba435a27d9ac023d7d385f26ad5b9709f98f201 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 24 Mar 2022 20:13:45 -0400 Subject: [PATCH 25/86] feat add explorer URL to arbi --- scripts/collect_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 382b22dc2..9ecd38863 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -111,6 +111,7 @@ "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", + "EXPLORER_URL": "https://arbiscan.io/", "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_42161_PUBLIC'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_42161'), From 6bd8e728ae0fb769afe160d1235fc1a189a1fa1d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 4 Apr 2022 11:22:32 -0400 Subject: [PATCH 26/86] feat: gitignore --- .gitignore | 3 ++- scripts/collect_reports.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6fd0884c5..29e2bb48b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__ +scripts/fees.py .history .hypothesis/ build/ @@ -21,4 +22,4 @@ generated package*.json hardhat.config.js .idea -setup_db.py \ No newline at end of file +setup_db.py diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 9ecd38863..e5891554b 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -457,7 +457,6 @@ def prepare_alerts(r, t): # Send to dev channel m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) - print(f'Chain ID: {chain.id}\n\n' + m + format_dev_telegram(r, t)) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): From c8e953bcaf5f7da85d855894722cfa504961003b Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 11 Apr 2022 19:55:45 -0400 Subject: [PATCH 27/86] feat: correct treasury fee bug after strategist fee rug --- .gitignore | 2 ++ scripts/collect_reports.py | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 29e2bb48b..8f2ac39ab 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ package*.json hardhat.config.js .idea setup_db.py +fees2.py +scripts/tracking \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index e5891554b..4520492a6 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -197,6 +197,7 @@ def handle_event(event, multi_harvest): endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() if event.address not in endorsed_vaults: + print("trying",event.address) print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") return if get_vault_endorsement_block(event.address) > event.block_number: @@ -385,9 +386,7 @@ def parse_fees(tx, vault_address, strategy_address, decimals): ) ) continue - elif counter == 1: - counter = 0 - if receiver == treasury and counter == 2: + if receiver == treasury and (counter == 1 or counter == 2): counter = 0 gov_fee_in_underlying = ( token_amount * ( From 0fc4ebe363b723556a719bd6d113c878bc72c953 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 20 Apr 2022 16:34:14 -0400 Subject: [PATCH 28/86] feat: remove strategist fee from output --- scripts/collect_reports.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 4520492a6..dac5d7c33 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -127,7 +127,6 @@ ) # Deprecated vault interface if chain.id == 1: - print(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) vault_v030 = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) vault_v030 = web3.eth.contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"], abi=vault_v030.abi) topics_v030 = construct_event_topic_set( @@ -508,7 +507,7 @@ def format_dev_telegram(r, t): df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " (" + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + ")" df["Debt Ratio"] = r.debt_ratio df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" - df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" + #df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" prefee = "n/a" postfee = "n/a" if r.rough_apr_pre_fee is not None: From 76d77fb4c130d6fb55d1ce3c17de481d2b0115bf Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 22 Apr 2022 10:52:01 -0400 Subject: [PATCH 29/86] feat: reth price band-aid fix --- scripts/add_single_report.py | 531 +++++++++++++++++++++++++++++++++++ scripts/collect_reports.py | 6 +- 2 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 scripts/add_single_report.py diff --git a/scripts/add_single_report.py b/scripts/add_single_report.py new file mode 100644 index 000000000..18ab21d8b --- /dev/null +++ b/scripts/add_single_report.py @@ -0,0 +1,531 @@ +import logging +import time, os +import telebot +from discordwebhook import Discord +from dotenv import load_dotenv +from yearn.cache import memory +import pandas as pd +from datetime import datetime, timezone +from brownie import chain, web3, Contract, ZERO_ADDRESS +from web3._utils.events import construct_event_topic_set +from yearn.utils import contract, contract_creation_block +from yearn.prices import magic, constants +from yearn.db.models import Reports, Event, Transactions, Session, engine, select +from sqlalchemy import desc, asc +from yearn.networks import Network +from yearn.events import decode_logs +import warnings +warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") +warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") +warnings.filterwarnings("ignore", ".*It has been discarded*") + + +# mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') +# ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +# discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') +# discord_ftm = os.environ.get('DISCORD_CHANNEL_250') + +telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') +bot = telebot.TeleBot(telegram_key) +alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False +dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') + +OLD_REGISTRY_ENDORSEMENT_BLOCKS = { + "0xE14d13d8B3b85aF791b2AADD661cDBd5E6097Db1": 11999957, + "0xdCD90C7f6324cfa40d7169ef80b12031770B4325": 11720423, + "0x986b4AFF588a109c09B50A03f42E4110E29D353F": 11881934, + "0xcB550A6D4C8e3517A939BC79d0c7093eb7cF56B5": 11770630, + "0xa9fE4601811213c340e850ea305481afF02f5b28": 11927501, + "0xB8C3B7A2A618C552C23B1E4701109a9E756Bab67": 12019352, + "0xBFa4D8AA6d8a379aBFe7793399D3DdaCC5bBECBB": 11579535, + "0x19D3364A399d251E894aC732651be8B0E4e85001": 11682465, + "0xe11ba472F74869176652C35D30dB89854b5ae84D": 11631914, + "0xe2F6b9773BF3A015E2aA70741Bde1498bdB9425b": 11579535, + "0x5f18C75AbDAe578b483E5F43f12a39cF75b973a9": 11682465, + "0x27b7b1ad7288079A66d12350c828D3C00A6F07d7": 12089661, +} + + +CHAIN_VALUES = { + Network.Mainnet: { + "NETWORK_NAME": "Ethereum Mainnet", + "NETWORK_SYMBOL": "ETH", + "EMOJI": "🇪🇹", + "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), + "START_BLOCK": 11563389, + "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", + "REGISTRY_DEPLOY_BLOCK": 12045555, + "REGISTRY_HELPER_ADDRESS": "0x52CbF68959e082565e7fd4bBb23D9Ccfb8C8C057", + "LENS_ADDRESS": "0x5b4F3BE554a88Bd0f8d8769B9260be865ba03B4a", + "LENS_DEPLOY_BLOCK": 12707450, + "VAULT_ADDRESS030": "0x19D3364A399d251E894aC732651be8B0E4e85001", + "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", + "KEEPER_CALL_CONTRACT": "0x2150b45626199CFa5089368BDcA30cd0bfB152D6", + "KEEPER_TOKEN": "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", + "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", + "STRATEGIST_MULTISIG": "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", + "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", + "EXPLORER_URL": "https://etherscan.io/", + "TENDERLY_CHAIN_IDENTIFIER": "mainnet", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_1'), + }, + Network.Fantom: { + "NETWORK_NAME": "Fantom", + "NETWORK_SYMBOL": "FTM", + "EMOJI": "👻", + "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), + "START_BLOCK": 18450847, + "REGISTRY_ADDRESS": "0x727fe1759430df13655ddb0731dE0D0FDE929b04", + "REGISTRY_DEPLOY_BLOCK": 18455565, + "REGISTRY_HELPER_ADDRESS": "0x8CC45f739104b3Bdb98BFfFaF2423cC0f817ccc1", + "REGISTRY_HELPER_DEPLOY_BLOCK": 18456459, + "LENS_ADDRESS": "0x97D0bE2a72fc4Db90eD9Dbc2Ea7F03B4968f6938", + "LENS_DEPLOY_BLOCK": 18842673, + "VAULT_ADDRESS030": "0x637eC617c86D24E421328e6CAEa1d92114892439", + "VAULT_ADDRESS031": "0x637eC617c86D24E421328e6CAEa1d92114892439", + "KEEPER_CALL_CONTRACT": "0x57419fb50fa588fc165acc26449b2bf4c7731458", + "KEEPER_TOKEN": "", + "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", + "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", + "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", + "EXPLORER_URL": "https://ftmscan.com/", + "TENDERLY_CHAIN_IDENTIFIER": "fantom", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_250'), + }, + Network.Arbitrum: { + "NETWORK_NAME": "Arbitrum", + "NETWORK_SYMBOL": "ARRB", + "EMOJI": "🤠", + "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), + "START_BLOCK": 4841854, + "REGISTRY_ADDRESS": "0x3199437193625DCcD6F9C9e98BDf93582200Eb1f", + "REGISTRY_DEPLOY_BLOCK": 12045555, + "REGISTRY_HELPER_ADDRESS": "0x237C3623bed7D115Fc77fEB08Dd27E16982d972B", + "LENS_ADDRESS": "0xcAd10033C86B0C1ED6bfcCAa2FF6779938558E9f", + "VAULT_ADDRESS030": "0x239e14A19DFF93a17339DCC444f74406C17f8E67", + "VAULT_ADDRESS031": "0x239e14A19DFF93a17339DCC444f74406C17f8E67", + "KEEPER_CALL_CONTRACT": "", + "KEEPER_TOKEN": "", + "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", + "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", + "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", + "EXPLORER_URL": "https://arbiscan.io/", + "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_42161_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_42161'), + } +} + + +# Primary vault interface +vault = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS031"]) +vault = web3.eth.contract(str(vault), abi=vault.abi) +topics = construct_event_topic_set( + vault.events.StrategyReported().abi, web3.codec, {} +) +# Deprecated vault interface +if chain.id == 1: + vault_v030 = contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"]) + vault_v030 = web3.eth.contract(CHAIN_VALUES[chain.id]["VAULT_ADDRESS030"], abi=vault_v030.abi) + topics_v030 = construct_event_topic_set( + vault_v030.events.StrategyReported().abi, web3.codec, {} + ) + +def main(target_vault, dynamically_find_multi_harvest=False): + start_block = CHAIN_VALUES[chain.id]["START_BLOCK"] + print(f"dynamic multi_harvest detection is enabled: {dynamically_find_multi_harvest}") + interval_seconds = 25 + + last_reported_block, last_reported_block030 = last_harvest_block() + + print("latest block (v0.3.1+ API)",last_reported_block) + print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) + if chain.id == 1: + print("latest block (v0.3.0 API)",last_reported_block030) + print("blocks behind (v0.3.0 API)", chain.height - last_reported_block030) + event_filter = web3.eth.filter({'address':target_vault, 'topics': topics, "fromBlock": start_block + 1}) + if chain.id == 1: + event_filter_v030 = web3.eth.filter({'address':target_vault, 'topics': topics_v030, "fromBlock": start_block + 1}) + + while True: # Keep this as a long-running script + events_to_process = [] + transaction_hashes = [] + if dynamically_find_multi_harvest: + # The code below is used to populate the "multi_harvest" property # + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(0, len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + + if chain.id == 1: # No old vaults deployed anywhere other than mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(0, len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + + for e in events_to_process: + handle_event(e.event, e.multi_harvest) + time.sleep(interval_seconds) + else: + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + handle_event(e.event, e.multi_harvest) + + if chain.id == 1: # Old vault API exists only on Ethereum mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) + handle_event(e.event, e.multi_harvest) + + time.sleep(interval_seconds) + +def handle_event(event, multi_harvest): + # exception because skeletor didnt verify contract + endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) + txn_hash = event.transaction_hash.hex() + if event.address not in endorsed_vaults: + print("trying",event.address) + print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + if get_vault_endorsement_block(event.address) > event.block_number: + print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + + tx = web3.eth.getTransactionReceipt(txn_hash) + gas_price = web3.eth.getTransaction(txn_hash).gasPrice + ts = chain[event.block_number].timestamp + dt = datetime.utcfromtimestamp(ts).strftime("%m/%d/%Y, %H:%M:%S") + r = Reports() + r.multi_harvest = multi_harvest + r.chain_id = chain.id + r.vault_address = event.address + try: + vault = contract(r.vault_address) + except ValueError: + return + r.vault_decimals = vault.decimals() + r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) + + txn_record_exists = False + t = transaction_record_exists(txn_hash) + if not t: + t = Transactions() + t.chain_id = chain.id + t.txn_hash = txn_hash + t.block = event.block_number + t.txn_to = tx.to + t.txn_from = tx["from"] + t.txn_gas_used = tx.gasUsed + t.txn_gas_price = gas_price / 1e9 # Use gwei + t.eth_price_at_block = magic.get_price(constants.weth, t.block) + t.call_cost_eth = gas_price * tx.gasUsed / 1e18 + t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth + if chain.id == 1: + t.kp3r_price_at_block = magic.get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) + t.kp3r_paid = get_keeper_payment(tx) / 1e18 + t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block + t.keeper_called = t.kp3r_paid > 0 + else: + if t.txn_to == CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"]: + t.keeper_called = True + else: + t.keeper_called = False + t.date = datetime.utcfromtimestamp(ts) + t.date_string = dt + t.timestamp = ts + t.updated_timestamp = datetime.now() + else: + txn_record_exists = True + r.block = event.block_number + r.txn_hash = txn_hash + strategy = contract(r.strategy_address) + + + r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) + r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want + r.token_symbol = contract(strategy.want()).symbol() + r.want_token = strategy.want() + r.want_price_at_block = 0 + if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": + r.want_price_at_block = magic.get_price(constants.weth, r.block) * contract("0xae78736Cd615f374D3085123A210448E74Fc6393").getExchangeRate() / 1e18 + else: + r.want_price_at_block = magic.get_price(r.want_token, r.block) + r.vault_api = vault.apiVersion() + r.want_gain_usd = r.gain * r.want_price_at_block + r.vault_name = vault.name() + r.strategy_name = strategy.name() + r.strategy_api = strategy.apiVersion() + r.strategist = strategy.strategist() + r.vault_symbol = vault.symbol() + r.date = datetime.utcfromtimestamp(ts) + r.date_string = dt + r.timestamp = ts + r.updated_timestamp = datetime.now() + + with Session(engine) as session: + query = select(Reports).where( + Reports.chain_id == chain.id, Reports.strategy_address == r.strategy_address + ).order_by(desc(Reports.block)) + previous_report = session.exec(query).first() + if previous_report != None: + previous_report_id = previous_report.id + r.previous_report_id = previous_report_id + r.rough_apr_pre_fee, r.rough_apr_post_fee = compute_apr(r, previous_report) + # Insert to database + insert_success = False + try: + session.add(r) + if not txn_record_exists: + session.add(t) + session.commit() + print(f"report added. strategy {r.strategy_address} txn hash {r.txn_hash}. chain id {r.chain_id} sync {r.block} / {chain.height}.") + insert_success = True + except: + print(f"skipped duplicate record. strategy: {r.strategy_address} at tx hash: {r.txn_hash}") + pass + if insert_success: + prepare_alerts(r, t) + +def transaction_record_exists(txn_hash): + with Session(engine) as session: + query = select(Transactions).where( + Transactions.txn_hash == txn_hash + ) + result = session.exec(query).first() + if result == None: + return False + return result + +def last_harvest_block(): + with Session(engine) as session: + query = select(Reports.block).where( + Reports.chain_id == chain.id, Reports.vault_api != "0.3.0" + ).order_by(desc(Reports.block)) + result1 = session.exec(query).first() + if result1 == None: + result1 = CHAIN_VALUES[chain.id]["START_BLOCK"] + if chain.id == 1: + query = select(Reports.block).where( + Reports.chain_id == chain.id, Reports.vault_api == "0.3.0" + ).order_by(desc(Reports.block)) + result2 = session.exec(query).first() + if result2 == None: + result2 = CHAIN_VALUES[chain.id]["START_BLOCK"] + else: + result2 = 0 + + return result1, result2 + +def get_keeper_payment(tx): + kp3r_token = CHAIN_VALUES[chain.id]["KEEPER_TOKEN"] + token = contract(kp3r_token) + denominator = 10 ** token.decimals() + token = web3.eth.contract(str(kp3r_token), abi=token.abi) + decoded_events = token.events.Transfer().processReceipt(tx) + amount = 0 + for e in decoded_events: + if e.address == kp3r_token: + sender, receiver, token_amount = e.args.values() + token_amount = token_amount / denominator + if receiver == tx["from"]: + amount = token_amount + return amount + +def compute_apr(report, previous_report): + SECONDS_IN_A_YEAR = 31557600 + seconds_between_reports = report.timestamp - previous_report.timestamp + pre_fee_apr = 0 + post_fee_apr = 0 + if int(previous_report.total_debt) == 0 or seconds_between_reports == 0: + return 0, 0 + else: + pre_fee_apr = report.gain / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + if report.gain_post_fees != 0: + post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + return pre_fee_apr, post_fee_apr + +def parse_fees(tx, vault_address, strategy_address, decimals): + denominator = 10 ** decimals + treasury = CHAIN_VALUES[chain.id]["YEARN_TREASURY"] + token = contract(vault_address) + token = web3.eth.contract(str(vault_address), abi=token.abi) + decoded_events = token.events.Transfer().processReceipt(tx) + amount = 0 + gov_fee_in_underlying = 0 + strategist_fee_in_underlying = 0 + counter = 0 + """ + Using the counter, we will keep track to ensure the expected sequence of fee Transfer events is followed. + Fee transfers always follow this sequence: + 1. mint + 2. transfer to strategy + 3. transfer to treasury + """ + for e in decoded_events: + if e.address == vault_address: + sender, receiver, token_amount = e.args.values() + token_amount = token_amount / denominator + if sender == ZERO_ADDRESS: + counter = 1 + continue + if receiver == strategy_address and counter == 1: + counter = 2 + strategist_fee_in_underlying = ( + token_amount * ( + contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / + denominator + ) + ) + continue + if receiver == treasury and (counter == 1 or counter == 2): + counter = 0 + gov_fee_in_underlying = ( + token_amount * ( + contract(vault_address).pricePerShare(block_identifier=tx.blockNumber) / + denominator + ) + ) + continue + elif counter == 1 or counter == 2: + counter = 0 + return gov_fee_in_underlying, strategist_fee_in_underlying + +@memory.cache() +def get_vault_endorsement_block(vault_address): + token = contract(vault_address).token() + try: + block = OLD_REGISTRY_ENDORSEMENT_BLOCKS[vault_address] + return block + except KeyError: + pass + registry = contract(CHAIN_VALUES[chain.id]["REGISTRY_ADDRESS"]) + height = chain.height + lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height + while hi - lo > 1: + mid = lo + (hi - lo) // 2 + try: + num_vaults = registry.numVaults(token, block_identifier=mid) + if registry.vaults(token, num_vaults-1, block_identifier=mid) == vault_address: + hi = mid + else: + lo = mid + except: + lo = mid + return hi + +def normalize_event_values(vals, decimals): + denominator = 10**decimals + if len(vals) == 8: + strategy_address, gain, loss, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + debt_paid = 0 + if len(vals) == 9: + strategy_address, gain, loss, debt_paid, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + return ( + strategy_address, + gain/denominator, + loss/denominator, + debt_paid/denominator, + total_gain/denominator, + total_loss/denominator, + total_debt/denominator, + debt_added/denominator, + debt_ratio + ) + +def prepare_alerts(r, t): + if alerts_enabled: + m = format_public_telegram(r, t) + # Send to chain specific channels + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) + discord.post( + embeds=[{ + "title": "New harvest", + "description": m + }], + ) + + # Send to dev channel + m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + +def format_public_telegram(r, t): + explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] + sms = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] + gov = CHAIN_VALUES[chain.id]["GOVERNANCE_MULTISIG"] + keeper = CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"] + from_indicator = "" + + if t.txn_to == sms or t.txn_to == gov: + from_indicator = "✍ " + + elif t.txn_from == r.strategist and t.txn_to != sms: + from_indicator = "🧠 " + + elif t.keeper_called or t.txn_from == keeper or t.txn_to == keeper: + from_indicator = "🤖 " + + message = "" + message += from_indicator + message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' + message += f'📅 {r.date_string} UTC \n\n' + net_profit_want = "{:,.2f}".format(r.gain - r.loss) + net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) + message += f'💰 Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n\n' + txn_cost_str = "${:,.2f}".format(t.call_cost_usd) + message += f'💸 Transaction Cost: {txn_cost_str} \n\n' + message += f'🔗 [View on Explorer]({explorer}tx/{r.txn_hash})' + if r.multi_harvest: + message += "\n\n_part of a single txn with multiple harvests_" + return message + +def format_dev_telegram(r, t): + tenderly_str = CHAIN_VALUES[chain.id]["TENDERLY_CHAIN_IDENTIFIER"] + message = f' / [Tenderly](https://dashboard.tenderly.co/tx/{tenderly_str}/{r.txn_hash})\n\n' + df = pd.DataFrame(index=['']) + last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] + if last_harvest_ts == 0: + time_since_last_report = "n/a" + else: + seconds_since_report = int(time.time() - last_harvest_ts) + time_since_last_report = "%dd, %dhr, %dm" % dhms_from_seconds(seconds_since_report) + df[r.vault_name + " " + r.vault_api] = r.vault_address + df["Strategy Address"] = r.strategy_address + df["Last Report"] = time_since_last_report + df["Gain"] = "{:,.2f}".format(r.gain) + " (" + "${:,.2f}".format(r.gain * r.want_price_at_block) + ")" + df["Loss"] = "{:,.2f}".format(r.loss) + " (" + "${:,.2f}".format(r.loss * r.want_price_at_block) + ")" + df["Debt Paid"] = "{:,.2f}".format(r.debt_paid) + " (" + "${:,.2f}".format(r.debt_paid * r.want_price_at_block) + ")" + df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " (" + "${:,.2f}".format(r.debt_added * r.want_price_at_block) + ")" + df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " (" + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + ")" + df["Debt Ratio"] = r.debt_ratio + df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" + #df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" + prefee = "n/a" + postfee = "n/a" + if r.rough_apr_pre_fee is not None: + prefee = "{:.2%}".format(r.rough_apr_pre_fee) + if r.rough_apr_post_fee is not None: + postfee = "{:.2%}".format(r.rough_apr_post_fee) + df["Pre-fee APR"] = prefee + df["Post-fee APR"] = postfee + message2 = f"```{df.T.to_string()}\n```" + return message + message2 + +def dhms_from_seconds(seconds): + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + days, hours = divmod(hours, 24) + return (days, hours, minutes) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index dac5d7c33..f1ed6d3c3 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -257,7 +257,11 @@ def handle_event(event, multi_harvest): r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.token_symbol = contract(strategy.want()).symbol() r.want_token = strategy.want() - r.want_price_at_block = magic.get_price(r.want_token, r.block) + r.want_price_at_block = 0 + if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": + r.want_price_at_block = magic.get_price(constants.weth, r.block) * contract(strategy.want()).getExchangeRate() / 1e18 + else: + r.want_price_at_block = magic.get_price(r.want_token, r.block) r.vault_api = vault.apiVersion() r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() From ba3a27891d3877ca9561a3bae4f4ed5e0990e3e6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 22 Apr 2022 12:23:24 -0400 Subject: [PATCH 30/86] feat: add inverse alerts --- scripts/add_single_report.py | 54 +++++++++++++++++++++++------------- scripts/collect_reports.py | 54 +++++++++++++++++++++++------------- 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/scripts/add_single_report.py b/scripts/add_single_report.py index 18ab21d8b..1154517fd 100644 --- a/scripts/add_single_report.py +++ b/scripts/add_single_report.py @@ -45,6 +45,11 @@ "0x27b7b1ad7288079A66d12350c828D3C00A6F07d7": 12089661, } +INVERSE_PRIVATE_VAULTS = [ + "0xD4108Bb1185A5c30eA3f4264Fd7783473018Ce17", + "0x67B9F46BCbA2DF84ECd41cC6511ca33507c9f4E9", +] + CHAIN_VALUES = { Network.Mainnet: { @@ -68,6 +73,7 @@ "EXPLORER_URL": "https://etherscan.io/", "TENDERLY_CHAIN_IDENTIFIER": "mainnet", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC'), + "TELEGRAM_CHAT_ID_INVERSE_ALERTS": os.environ.get('TELEGRAM_CHAT_ID_INVERSE_ALERTS'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_1'), }, Network.Fantom: { @@ -197,12 +203,15 @@ def handle_event(event, multi_harvest): endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() if event.address not in endorsed_vaults: - print("trying",event.address) - print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") - return - if get_vault_endorsement_block(event.address) > event.block_number: - print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") - return + # check if a vault from inverse partnership + if event.address not in INVERSE_PRIVATE_VAULTS: + print("trying",event.address) + print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + if event.address not in INVERSE_PRIVATE_VAULTS: + if get_vault_endorsement_block(event.address) > event.block_number: + print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice @@ -447,20 +456,25 @@ def normalize_event_values(vals, decimals): def prepare_alerts(r, t): if alerts_enabled: - m = format_public_telegram(r, t) - # Send to chain specific channels - bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) - discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) - discord.post( - embeds=[{ - "title": "New harvest", - "description": m - }], - ) - - # Send to dev channel - m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) - bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + if r.vault_address not in INVERSE_PRIVATE_VAULTS: + m = format_public_telegram(r, t) + # Send to chain specific channels + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) + discord.post( + embeds=[{ + "title": "New harvest", + "description": m + }], + ) + + # Send to dev channel + m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + else: + m = format_public_telegram(r, t) + # Send to chain specific channels + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID_INVERSE_ALERTS"], m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index f1ed6d3c3..19b340506 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -27,6 +27,8 @@ telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') bot = telebot.TeleBot(telegram_key) +inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') +invbot = telebot.TeleBot(inv_telegram_key) alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') @@ -45,6 +47,10 @@ "0x27b7b1ad7288079A66d12350c828D3C00A6F07d7": 12089661, } +INVERSE_PRIVATE_VAULTS = [ + "0xD4108Bb1185A5c30eA3f4264Fd7783473018Ce17", + "0x67B9F46BCbA2DF84ECd41cC6511ca33507c9f4E9", +] CHAIN_VALUES = { Network.Mainnet: { @@ -92,6 +98,7 @@ "EXPLORER_URL": "https://ftmscan.com/", "TENDERLY_CHAIN_IDENTIFIER": "fantom", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC'), + "TELEGRAM_CHAT_ID_INVERSE_ALERTS": os.environ.get('TELEGRAM_CHAT_ID_INVERSE_ALERTS'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_250'), }, Network.Arbitrum: { @@ -196,12 +203,15 @@ def handle_event(event, multi_harvest): endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() if event.address not in endorsed_vaults: - print("trying",event.address) - print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") - return - if get_vault_endorsement_block(event.address) > event.block_number: - print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") - return + # check if a vault from inverse partnership + if event.address not in INVERSE_PRIVATE_VAULTS: + print("trying",event.address) + print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return + if event.address not in INVERSE_PRIVATE_VAULTS: + if get_vault_endorsement_block(event.address) > event.block_number: + print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") + return tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice @@ -446,20 +456,24 @@ def normalize_event_values(vals, decimals): def prepare_alerts(r, t): if alerts_enabled: - m = format_public_telegram(r, t) - # Send to chain specific channels - bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) - discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) - discord.post( - embeds=[{ - "title": "New harvest", - "description": m - }], - ) - - # Send to dev channel - m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) - bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + if r.vault_address not in INVERSE_PRIVATE_VAULTS: + m = format_public_telegram(r, t) + # Send to chain specific channels + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) + discord.post( + embeds=[{ + "title": "New harvest", + "description": m + }], + ) + + # Send to dev channel + m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) + else: + m = format_public_telegram(r, t) + invbot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID_INVERSE_ALERTS"], m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] From a84d301decf49a7a511111942080023648afb7af Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 22 Apr 2022 14:33:23 -0400 Subject: [PATCH 31/86] fix: add var to chain 1 --- scripts/collect_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 19b340506..c8353d558 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -74,6 +74,7 @@ "EXPLORER_URL": "https://etherscan.io/", "TENDERLY_CHAIN_IDENTIFIER": "mainnet", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC'), + "TELEGRAM_CHAT_ID_INVERSE_ALERTS": os.environ.get('TELEGRAM_CHAT_ID_INVERSE_ALERTS'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_1'), }, Network.Fantom: { From 799487128b2ab68d156b9b2296afcea1c9c700c6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 22 Apr 2022 15:05:30 -0400 Subject: [PATCH 32/86] feat: extra detail --- scripts/collect_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index c8353d558..4e2f56533 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -474,6 +474,7 @@ def prepare_alerts(r, t): bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) else: m = format_public_telegram(r, t) + m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) invbot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID_INVERSE_ALERTS"], m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): From b5e2bce3adc11ab48a8622907d91c957f4d02754 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 22 Apr 2022 21:11:48 -0400 Subject: [PATCH 33/86] feat: simplify inverse harvest alert formatting --- scripts/collect_reports.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 4e2f56533..1453181eb 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -473,8 +473,8 @@ def prepare_alerts(r, t): m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) bot.send_message(dev_channel, m, parse_mode="markdown", disable_web_page_preview = True) else: - m = format_public_telegram(r, t) - m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) + m = format_public_telegram_inv(r, t) + m = m + format_dev_telegram(r, t) invbot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID_INVERSE_ALERTS"], m, parse_mode="markdown", disable_web_page_preview = True) def format_public_telegram(r, t): @@ -507,9 +507,29 @@ def format_public_telegram(r, t): message += "\n\n_part of a single txn with multiple harvests_" return message +def format_public_telegram_inv(r, t): + explorer = CHAIN_VALUES[chain.id]["EXPLORER_URL"] + sms = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] + gov = CHAIN_VALUES[chain.id]["GOVERNANCE_MULTISIG"] + keeper = CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"] + from_indicator = "" + + message = f'👨‍🌾 New Harvest Detected!\n\n' + message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n' + message += f'{r.date_string} UTC \n' + net_profit_want = "{:,.2f}".format(r.gain - r.loss) + net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) + message += f'Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n' + txn_cost_str = "${:,.2f}".format(t.call_cost_usd) + message += f'Transaction Cost: {txn_cost_str} \n' + message += f'[View on Explorer]({explorer}tx/{r.txn_hash})' + if r.multi_harvest: + message += "\n\n_part of a single txn with multiple harvests_" + return message + def format_dev_telegram(r, t): tenderly_str = CHAIN_VALUES[chain.id]["TENDERLY_CHAIN_IDENTIFIER"] - message = f' / [Tenderly](https://dashboard.tenderly.co/tx/{tenderly_str}/{r.txn_hash})\n\n' + message = f' | [Tenderly](https://dashboard.tenderly.co/tx/{tenderly_str}/{r.txn_hash})\n\n' df = pd.DataFrame(index=['']) last_harvest_ts = contract(r.vault_address).strategies(r.strategy_address, block_identifier=r.block-1).dict()["lastReport"] if last_harvest_ts == 0: From 6e9925280c076e9df83b6f6861cb2d30987a95e2 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Sat, 14 May 2022 22:36:31 -0400 Subject: [PATCH 34/86] fix: replace underscores in symbol --- .gitignore | 3 +- scripts/bribe_income.py | 109 +++++++++++++++++++++++++++++++++++++ scripts/collect_reports.py | 6 +- 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 scripts/bribe_income.py diff --git a/.gitignore b/.gitignore index 8f2ac39ab..3e503120c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ hardhat.config.js .idea setup_db.py fees2.py -scripts/tracking \ No newline at end of file +scripts/tracking +yvboost_fees.py \ No newline at end of file diff --git a/scripts/bribe_income.py b/scripts/bribe_income.py new file mode 100644 index 000000000..dfc7ec064 --- /dev/null +++ b/scripts/bribe_income.py @@ -0,0 +1,109 @@ +from collections import defaultdict +from datetime import datetime + +from brownie import ZERO_ADDRESS, chain, web3 +from rich import print +from rich.progress import track +from rich.table import Table +from web3._utils.events import construct_event_topic_set +from yearn.prices.magic import get_price +from yearn.utils import contract, closest_block_after_timestamp +from brownie.exceptions import ContractNotFound + +def closest_block(): + ts = 1649304000 + ts = 1649289600 + print(closest_block_after_timestamp(ts)) + +def main(): + my_addresses = [ + '0xF147b8125d2ef93FB6965Db97D6746952a133934' + ] + + from_block = 12316532 + dai = contract("0x6B175474E89094C44Da98b954EedeAC495271d0F") + dai = web3.eth.contract(str(dai), abi=dai.abi) + print(f"Starting from block {from_block}") + + print(f"abi: {dai.events.Transfer().abi}") + + topics = construct_event_topic_set( + dai.events.Transfer().abi, + web3.codec, + {'dst': my_addresses, 'src': "0x7893bbb46613d7a4FbcC31Dab4C9b823FfeE1026"}, + ) + logs = web3.eth.get_logs( + {'topics': topics, 'fromBlock': from_block, 'toBlock': chain.height} + ) + + events = dai.events.Transfer().processReceipt({'logs': logs}) + income_by_month = defaultdict(float) + tokens_by_month = defaultdict(str) + grand_total = 0 + + for event in track(events): + ts = chain[event.blockNumber].timestamp + token = event.address + txn_hash = event.transactionHash.hex() + tx = web3.eth.getTransaction(txn_hash) + + token_contract = contract(token) + + src, dst, amount = event.args.values() + + if src in my_addresses: + print("Sent to self") + continue + + try: + price = get_price(event.address, block=event.blockNumber) + except: + print( + "Pricing error for", + token_contract.symbol(), + "on", + token_contract.address, + "****************************************", + ) + print(f"Amount: {amount}") + continue + print("\nDate:", datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d')) + + try: + amount /= 10 ** contract(token).decimals() + except (ValueError, ContractNotFound, AttributeError): + continue + + try: + price = get_price(event.address, block=event.blockNumber) + except: + print("\nPricing error for", token_contract.symbol(), "on", token_contract.address, "****************************************") + print(f"Amount: {amount}") + continue + symbol = token_contract.symbol() + print("\nToken Symbol:", symbol) + print(f"Amount: {amount}") + print(f"Price: {price}") + print(txn_hash) + month = datetime.utcfromtimestamp(ts).strftime('%Y-%m') + grand_total += amount * price + income_by_month[month] += amount * price + if tokens_by_month[month]: + if symbol not in tokens_by_month[month]: + tokens_by_month[month] = tokens_by_month[month] + ", " + symbol + else: + tokens_by_month[month] = symbol + + + + table = Table() + table.add_column('month') + table.add_column('value claimed') + table.add_column('tokens claimed') + for month in sorted(income_by_month): + table.add_row(month, f'{"${:,.0f}".format(income_by_month[month])}',f'{tokens_by_month[month]}') + # table.add_row(month, f'{tokens_by_month[month]}') + + print(table) + print(sum(income_by_month.values())) + print("${:,.2f}".format(grand_total)) \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 1453181eb..522d7064f 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -499,7 +499,8 @@ def format_public_telegram(r, t): message += f'📅 {r.date_string} UTC \n\n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) - message += f'💰 Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n\n' + sym = r.token_symbol.replace('_','-') + message += f'💰 Net profit: {net_profit_want} {sym} (${net_profit_usd})\n\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) message += f'💸 Transaction Cost: {txn_cost_str} \n\n' message += f'🔗 [View on Explorer]({explorer}tx/{r.txn_hash})' @@ -519,7 +520,8 @@ def format_public_telegram_inv(r, t): message += f'{r.date_string} UTC \n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) - message += f'Net profit: {net_profit_want} {r.token_symbol} (${net_profit_usd})\n' + sym = r.token_symbol.replace('_','-') + message += f'Net profit: {net_profit_want} {sym} (${net_profit_usd})\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) message += f'Transaction Cost: {txn_cost_str} \n' message += f'[View on Explorer]({explorer}tx/{r.txn_hash})' From 4f42d283d77f7dbd0532d3d1340f1b4809db8bb5 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Sun, 5 Jun 2022 13:50:54 -0400 Subject: [PATCH 35/86] feat: custom inverse tg post --- scripts/collect_reports.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 522d7064f..882262afc 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -230,7 +230,7 @@ def handle_event(event, multi_harvest): r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) txn_record_exists = False - t = transaction_record_exists(txn_hash) + t = transaction_record_exists(txn_hash, r.vault_address) if not t: t = Transactions() t.chain_id = chain.id @@ -309,12 +309,16 @@ def handle_event(event, multi_harvest): if insert_success: prepare_alerts(r, t) -def transaction_record_exists(txn_hash): +def transaction_record_exists(txn_hash, vault_address): with Session(engine) as session: query = select(Transactions).where( Transactions.txn_hash == txn_hash ) - result = session.exec(query).first() + result1 = session.exec(query).first() + query = select(Reports).where( + Reports.txn_hash == txn_hash and Reports.vault_address == vault_address + ) + result2 = session.exec(query) if result == None: return False return result @@ -548,8 +552,15 @@ def format_dev_telegram(r, t): df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " (" + "${:,.2f}".format(r.debt_added * r.want_price_at_block) + ")" df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " (" + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + ")" df["Debt Ratio"] = r.debt_ratio - df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" - #df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" + if r.vault_address in INVERSE_PRIVATE_VAULTS: + fees = r.gov_fee_in_want + r.strategist_fee_in_want + inverse_profit = r.gain - fees + df["Yearn Treasury Profit"] = "{:,.2f}".format(fees) + " (" + "${:,.2f}".format(fees * r.want_price_at_block) + ")" + df["Inverse Profit"] = "{:,.2f}".format(inverse_profit) + " (" + "${:,.2f}".format(inverse_profit * r.want_price_at_block) + ")" + else: + df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" + if r.strategy_address == "0xd025b85db175EF1b175Af223BD37f330dB277786": + df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" prefee = "n/a" postfee = "n/a" if r.rough_apr_pre_fee is not None: From 6ad2c98cb290d35db9eef828a0dd1a37d6c9252e Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 6 Jun 2022 20:21:08 -0400 Subject: [PATCH 36/86] fix: revert txn check check logic --- scripts/collect_reports.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 882262afc..70f6623c2 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -309,16 +309,12 @@ def handle_event(event, multi_harvest): if insert_success: prepare_alerts(r, t) -def transaction_record_exists(txn_hash, vault_address): +def transaction_record_exists(txn_hash): with Session(engine) as session: query = select(Transactions).where( Transactions.txn_hash == txn_hash ) - result1 = session.exec(query).first() - query = select(Reports).where( - Reports.txn_hash == txn_hash and Reports.vault_address == vault_address - ) - result2 = session.exec(query) + result = session.exec(query).first() if result == None: return False return result From b73369ae365f88a572c2cecc5dde35510cc6617d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 6 Jun 2022 20:22:15 -0400 Subject: [PATCH 37/86] fix: revert txn check check logic --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 70f6623c2..1ad6d4df8 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -230,7 +230,7 @@ def handle_event(event, multi_harvest): r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) txn_record_exists = False - t = transaction_record_exists(txn_hash, r.vault_address) + t = transaction_record_exists(txn_hash) if not t: t = Transactions() t.chain_id = chain.id From 801ce1600788b8a0a2141e4cea4e00785f26e1b6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 6 Jun 2022 21:00:18 -0400 Subject: [PATCH 38/86] feat: eliminate txn fee double counting --- scripts/daily_harvest_report.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/daily_harvest_report.py b/scripts/daily_harvest_report.py index 9acd09f12..4ecae8ef7 100644 --- a/scripts/daily_harvest_report.py +++ b/scripts/daily_harvest_report.py @@ -53,16 +53,19 @@ def main(): DAY_IN_SECONDS = 60 * 60 * 24 current_time = int(time.time()) yesterday = current_time - DAY_IN_SECONDS + txn_list = [] with Session(engine) as session: query = select(Reports, Transactions).join(Transactions).where( Reports.timestamp > yesterday ).order_by(desc(Reports.block)) results = session.exec(query) - for report, txn in results: + for report, txn in results: RESULTS[txn.chain_id]["profit_usd"] = RESULTS[txn.chain_id]["profit_usd"] + report.want_gain_usd RESULTS[txn.chain_id]["num_harvests"] = RESULTS[txn.chain_id]["num_harvests"] + 1 - RESULTS[txn.chain_id]["txn_cost_eth"] = RESULTS[txn.chain_id]["txn_cost_eth"] + txn.call_cost_eth - RESULTS[txn.chain_id]["txn_cost_usd"] = RESULTS[txn.chain_id]["txn_cost_usd"] + txn.call_cost_usd + if txn.txn_hash not in txn_list: + txn_list.append(txn.txn_hash) + RESULTS[txn.chain_id]["txn_cost_eth"] = RESULTS[txn.chain_id]["txn_cost_eth"] + txn.call_cost_eth + RESULTS[txn.chain_id]["txn_cost_usd"] = RESULTS[txn.chain_id]["txn_cost_usd"] + txn.call_cost_usd # Build Messages cumulative_message = "" for chain in RESULTS.keys(): From 6b635f7f8cc817b3b280da3ccf4b125af8be6b43 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 28 Jul 2022 21:41:49 -0400 Subject: [PATCH 39/86] fix: reth/wsteth price --- scripts/collect_reports.py | 18 +++-- scripts/daily_harvest_report.py | 2 +- scripts/event_parser.py | 117 ++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 8 deletions(-) create mode 100644 scripts/event_parser.py diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 1ad6d4df8..fd94588f0 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -263,17 +263,17 @@ def handle_event(event, multi_harvest): r.txn_hash = txn_hash strategy = contract(r.strategy_address) - - r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals) + r.vault_api = vault.apiVersion() + r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want r.token_symbol = contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = 0 if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": - r.want_price_at_block = magic.get_price(constants.weth, r.block) * contract(strategy.want()).getExchangeRate() / 1e18 + r.want_price_at_block = magic.get_price(constants.weth, r.block) * contract(contract(r.want_token).coins(0)).getExchangeRate() / 1e18 else: r.want_price_at_block = magic.get_price(r.want_token, r.block) - r.vault_api = vault.apiVersion() + r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() r.strategy_name = strategy.name() @@ -367,12 +367,16 @@ def compute_apr(report, previous_report): post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) return pre_fee_apr, post_fee_apr -def parse_fees(tx, vault_address, strategy_address, decimals): +def parse_fees(tx, vault_address, strategy_address, decimals, gain, vault_version): + v = int(''.join(x for x in vault_version.split('.'))) + if v < 35 and gain == 0: + return 0, 0 denominator = 10 ** decimals treasury = CHAIN_VALUES[chain.id]["YEARN_TREASURY"] token = contract(vault_address) token = web3.eth.contract(str(vault_address), abi=token.abi) - decoded_events = token.events.Transfer().processReceipt(tx) + transfers = token.events.Transfer().processReceipt(tx) + amount = 0 gov_fee_in_underlying = 0 strategist_fee_in_underlying = 0 @@ -384,7 +388,7 @@ def parse_fees(tx, vault_address, strategy_address, decimals): 2. transfer to strategy 3. transfer to treasury """ - for e in decoded_events: + for e in transfers: if e.address == vault_address: sender, receiver, token_amount = e.args.values() token_amount = token_amount / denominator diff --git a/scripts/daily_harvest_report.py b/scripts/daily_harvest_report.py index 4ecae8ef7..658bc5a22 100644 --- a/scripts/daily_harvest_report.py +++ b/scripts/daily_harvest_report.py @@ -83,7 +83,7 @@ def main(): RESULTS[chain]["message"] = message channel = RESULTS[chain]["telegram_channel"] print() - if channel != 0: + if channel != 0 and alerts_enabled: bot.send_message(channel, message, parse_mode="markdown", disable_web_page_preview = True) print(message) date_banner = f'📃 End of Day Report --- {datetime.utcfromtimestamp(current_time - 1000).strftime("%m-%d-%Y")} \n\n' diff --git a/scripts/event_parser.py b/scripts/event_parser.py new file mode 100644 index 000000000..83daeab75 --- /dev/null +++ b/scripts/event_parser.py @@ -0,0 +1,117 @@ +import logging +import time, os +import telebot +from discordwebhook import Discord +from dotenv import load_dotenv +from yearn.cache import memory +import pandas as pd +from datetime import datetime, timezone +from brownie import chain, web3, Contract, ZERO_ADDRESS +from web3._utils.events import construct_event_topic_set +from yearn.utils import contract, contract_creation_block +from yearn.events import decode_logs +import warnings +warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") +warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") +warnings.filterwarnings("ignore", ".*It has been discarded*") + +def main(): + vault_address = "0xdA816459F1AB5631232FE5e97a05BBBb94970c95" + txn_hash = "0xba5d72cf3052869c2259470b8f45f2131acd5c00daea0cac6580cd851e9774ea" + do_work(txn_hash, vault_address) + +def do_work(txn_hash, vault_address): + vault = Contract(vault_address) + # Set of complex txns to work with + txn = web3.eth.getTransactionReceipt(txn_hash) + abi = vault.abi + contract = web3.eth.contract(vault_address, abi=vault.abi) + transfers = contract.events["Transfer"]().processReceipt(txn) + reports = contract.events["StrategyReported"]().processReceipt(txn) + num_reports_for_vault = 0 + num_fees_in_tx = 0 + for r in reports: + strategy_address, gain, loss, debt_paid, total_gain, total_loss, total_debt, debt_added, debt_ratio = normalize_event_values(r.args.values(),vault.decimals()) + print(gain) + if r.address == vault_address and gain > 0: + num_reports_for_vault = num_reports_for_vault + 1 + for t in transfers: + if t.address != vault_address: + continue + sender, receiver, value = t.args.values() + if sender == ZERO_ADDRESS and receiver == vault: + num_fees_in_tx = num_fees_in_tx + 1 + assert False + + # vault_decimals = vault.decimals() + # treasury_fee_values = [] + # reports_list = [] + + # obj = { + # "gain": 0, + # "treasury_fee": treasury_fee_values[counter], + # "has_fees": False, + # } + # # this line checks if any fees at all were collected + # for e in transfers: + # if e.address != vault_address: + # continue + # sender, receiver, value = e.args.values() + # if sender == ZERO_ADDRESS and receiver == vault: + # # Fees collected! + # obj["has_fees"] = True + + + # if obj["has_fees"]: + # for e in transfers: + # if e.address != vault_address: + # continue + # sender, receiver, value = e.args.values() + # # Here we count number of times we mint vault tokens + # # Or should we count sending them from vault to treasury? + # if sender == vault_address and receiver == vault.rewards(): + # treasury_fee_values.append(value) + + # # ZIP + # counter = 0 + # for r in reports: + # if r.address != vault_address: + # continue + # strategy_address = r.args.get("strategy") + # gain = r.args.get("gain") + # if gain == 0: + # continue + + # counter += 1 + # reports_list.append(obj) + + # print(strategy_address) + # print(obj) + # assert len(reports_list) == len(treasury_fee_values) + + + # object = { + # "strategy": ZERO_ADDRESS, + # "gain": 0, + # "fee": 0, + # } + + +def normalize_event_values(vals, decimals): + denominator = 10**decimals + if len(vals) == 8: + strategy_address, gain, loss, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + debt_paid = 0 + if len(vals) == 9: + strategy_address, gain, loss, debt_paid, total_gain, total_loss, total_debt, debt_added, debt_ratio = vals + return ( + strategy_address, + gain/denominator, + loss/denominator, + debt_paid/denominator, + total_gain/denominator, + total_loss/denominator, + total_debt/denominator, + debt_added/denominator, + debt_ratio + ) \ No newline at end of file From 57cb488948baf5ec4eed67dfeb0745559446a9d9 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 8 Aug 2022 09:54:29 -0400 Subject: [PATCH 40/86] feat: support CRV lock stats on inverse harvests --- scripts/collect_reports.py | 55 +++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index fd94588f0..d14f8148b 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -25,12 +25,21 @@ # discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') # discord_ftm = os.environ.get('DISCORD_CHANNEL_250') -telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') -bot = telebot.TeleBot(telegram_key) + inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') invbot = telebot.TeleBot(inv_telegram_key) -alerts_enabled = True if os.environ.get('ENVIRONMENT') == "PROD" else False -dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') +env = os.environ.get('ENVIRONMENT') +alerts_enabled = True if env == "PROD" or env == "TEST" else False + +test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') +if env == "TEST": + telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') + dev_channel = test_channel + bot = telebot.TeleBot(telegram_key) +else: + telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') + bot = telebot.TeleBot(telegram_key) + dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') OLD_REGISTRY_ENDORSEMENT_BLOCKS = { "0xE14d13d8B3b85aF791b2AADD661cDBd5E6097Db1": 11999957, @@ -73,8 +82,8 @@ "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", "EXPLORER_URL": "https://etherscan.io/", "TENDERLY_CHAIN_IDENTIFIER": "mainnet", - "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC'), - "TELEGRAM_CHAT_ID_INVERSE_ALERTS": os.environ.get('TELEGRAM_CHAT_ID_INVERSE_ALERTS'), + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') if env == "PROD" else test_channel, + "TELEGRAM_CHAT_ID_INVERSE_ALERTS": os.environ.get('TELEGRAM_CHAT_ID_INVERSE_ALERTS') if env == "PROD" else test_channel, "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_1'), }, Network.Fantom: { @@ -546,23 +555,37 @@ def format_dev_telegram(r, t): df[r.vault_name + " " + r.vault_api] = r.vault_address df["Strategy Address"] = r.strategy_address df["Last Report"] = time_since_last_report - df["Gain"] = "{:,.2f}".format(r.gain) + " (" + "${:,.2f}".format(r.gain * r.want_price_at_block) + ")" - df["Loss"] = "{:,.2f}".format(r.loss) + " (" + "${:,.2f}".format(r.loss * r.want_price_at_block) + ")" - df["Debt Paid"] = "{:,.2f}".format(r.debt_paid) + " (" + "${:,.2f}".format(r.debt_paid * r.want_price_at_block) + ")" - df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " (" + "${:,.2f}".format(r.debt_added * r.want_price_at_block) + ")" - df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " (" + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + ")" - df["Debt Ratio"] = r.debt_ratio + df["Gain"] = "{:,.2f}".format(r.gain) + " | " + "${:,.2f}".format(r.gain * r.want_price_at_block) + df["Loss"] = "{:,.2f}".format(r.loss) + " | " + "${:,.2f}".format(r.loss * r.want_price_at_block) if r.vault_address in INVERSE_PRIVATE_VAULTS: + tx = web3.eth.getTransactionReceipt(r.txn_hash) + crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' + voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' fees = r.gov_fee_in_want + r.strategist_fee_in_want inverse_profit = r.gain - fees - df["Yearn Treasury Profit"] = "{:,.2f}".format(fees) + " (" + "${:,.2f}".format(fees * r.want_price_at_block) + ")" - df["Inverse Profit"] = "{:,.2f}".format(inverse_profit) + " (" + "${:,.2f}".format(inverse_profit * r.want_price_at_block) + ")" + df["Yearn Treasury Profit"] = "{:,.2f}".format(fees) + " | " + "${:,.2f}".format(fees * r.want_price_at_block) + df["Inverse Profit"] = "{:,.2f}".format(inverse_profit) + " | " + "${:,.2f}".format(inverse_profit * r.want_price_at_block) + token = web3.eth.contract(crv, abi=Contract(crv).abi) + decoded_events = token.events.Transfer().processReceipt(tx) + for t in decoded_events: + _from, _to, _val = t.args.values() + if t.address == crv and _from == r.strategy_address and _to == voter: + crv_amount = _val/1e18 + crv_value = magic.get_price(crv, r.block) * crv_amount + df["CRV Locked"] = "{:,.2f}".format(crv_amount) + " | " + "${:,.2f}".format(crv_value) + else: - df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " (" + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) + ")" + df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " | " + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) if r.strategy_address == "0xd025b85db175EF1b175Af223BD37f330dB277786": - df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " (" + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) + ")" + df["Strategist Fee"] = "{:,.2f}".format(r.strategist_fee_in_want) + " | " + "${:,.2f}".format(r.strategist_fee_in_want * r.want_price_at_block) prefee = "n/a" postfee = "n/a" + df["Debt Paid"] = "{:,.2f}".format(r.debt_paid) + " | " + "${:,.2f}".format(r.debt_paid * r.want_price_at_block) + df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " | " + "${:,.2f}".format(r.debt_added * r.want_price_at_block) + df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " | " + "${:,.2f}".format(r.total_debt * r.want_price_at_block) + df["Debt Ratio"] = r.debt_ratio + + if r.rough_apr_pre_fee is not None: prefee = "{:.2%}".format(r.rough_apr_pre_fee) if r.rough_apr_post_fee is not None: From cb4d014943cee98cdb30c544c39d6a23097bc988 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 10 Aug 2022 10:57:38 -0400 Subject: [PATCH 41/86] feat: add keepcrv data --- .gitignore | 1 + scripts/collect_reports.py | 31 ++++++++++++++++++++++++++++++- yearn/db/models.py | 6 ++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3e503120c..758c9574e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ hardhat.config.js setup_db.py fees2.py scripts/tracking +scripts/update_crv.py yvboost_fees.py \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index d14f8148b..be5859c8c 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -1,3 +1,4 @@ +from lib2to3.pgen2 import token import logging import time, os import telebot @@ -294,6 +295,33 @@ def handle_event(event, multi_harvest): r.timestamp = ts r.updated_timestamp = datetime.now() + # KeepCRV stuff + crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' + yvecrv = '0xc5bDdf9843308380375a611c18B50Fb9341f502A' + voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' + token_abi = Contract(crv).abi + crv_token = web3.eth.contract(crv, abi=token_abi) + decoded_events = crv_token.events.Transfer().processReceipt(tx) + r.keep_crv = 0 + for tfr in decoded_events: + _from, _to, _val = tfr.args.values() + if tfr.address == crv and _from == r.strategy_address and _to == voter: + r.keep_crv = _val / 1e18 + r.crv_price_usd = magic.get_price(crv, r.block) + r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd + + if r.keep_crv > 0: + yvecrv_token = web3.eth.contract(yvecrv, abi=token_abi) + decoded_events = yvecrv_token.events.Transfer().processReceipt(tx) + try: + r.keep_crv_percent = strategy.keepCRV() + except: + pass + for tfr in decoded_events: + _from, _to, _val = tfr.args.values() + if tfr.address == yvecrv and _from == ZERO_ADDRESS: + r.yvecrv_minted = _val/1e18 + with Session(engine) as session: query = select(Reports).where( Reports.chain_id == chain.id, Reports.strategy_address == r.strategy_address @@ -584,7 +612,8 @@ def format_dev_telegram(r, t): df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " | " + "${:,.2f}".format(r.debt_added * r.want_price_at_block) df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " | " + "${:,.2f}".format(r.total_debt * r.want_price_at_block) df["Debt Ratio"] = r.debt_ratio - + if r.keep_crv > 0: + df["CRV Locked"] = "{:,.2f}".format(r.keep_crv) + " | " + "${:,.2f}".format(r.keep_crv_value_usd) if r.rough_apr_pre_fee is not None: prefee = "{:.2%}".format(r.rough_apr_pre_fee) diff --git a/yearn/db/models.py b/yearn/db/models.py index 4e772a94d..47ed4d703 100644 --- a/yearn/db/models.py +++ b/yearn/db/models.py @@ -112,6 +112,12 @@ class Reports(SQLModel, table=True): date_string: str timestamp: str updated_timestamp: datetime + # KeepCRV + keep_crv: int + keep_crv_percent: int + crv_price_usd: int + keep_crv_value_usd: int + yvecrv_minted: int From ec83d7acc150eb487ca1900ea7d7f5c4b1cd28ac Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 10 Aug 2022 18:11:26 -0400 Subject: [PATCH 42/86] fix: only do keepcrv on chainid=1 --- scripts/collect_reports.py | 58 ++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index be5859c8c..3c719c8c0 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -296,31 +296,32 @@ def handle_event(event, multi_harvest): r.updated_timestamp = datetime.now() # KeepCRV stuff - crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' - yvecrv = '0xc5bDdf9843308380375a611c18B50Fb9341f502A' - voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' - token_abi = Contract(crv).abi - crv_token = web3.eth.contract(crv, abi=token_abi) - decoded_events = crv_token.events.Transfer().processReceipt(tx) - r.keep_crv = 0 - for tfr in decoded_events: - _from, _to, _val = tfr.args.values() - if tfr.address == crv and _from == r.strategy_address and _to == voter: - r.keep_crv = _val / 1e18 - r.crv_price_usd = magic.get_price(crv, r.block) - r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd - - if r.keep_crv > 0: - yvecrv_token = web3.eth.contract(yvecrv, abi=token_abi) - decoded_events = yvecrv_token.events.Transfer().processReceipt(tx) - try: - r.keep_crv_percent = strategy.keepCRV() - except: - pass + if chain.id == 1: + crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' + yvecrv = '0xc5bDdf9843308380375a611c18B50Fb9341f502A' + voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' + token_abi = Contract(crv).abi + crv_token = web3.eth.contract(crv, abi=token_abi) + decoded_events = crv_token.events.Transfer().processReceipt(tx) + r.keep_crv = 0 for tfr in decoded_events: _from, _to, _val = tfr.args.values() - if tfr.address == yvecrv and _from == ZERO_ADDRESS: - r.yvecrv_minted = _val/1e18 + if tfr.address == crv and _from == r.strategy_address and _to == voter: + r.keep_crv = _val / 1e18 + r.crv_price_usd = magic.get_price(crv, r.block) + r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd + + if r.keep_crv > 0: + yvecrv_token = web3.eth.contract(yvecrv, abi=token_abi) + decoded_events = yvecrv_token.events.Transfer().processReceipt(tx) + try: + r.keep_crv_percent = strategy.keepCRV() + except: + pass + for tfr in decoded_events: + _from, _to, _val = tfr.args.values() + if tfr.address == yvecrv and _from == ZERO_ADDRESS: + r.yvecrv_minted = _val/1e18 with Session(engine) as session: query = select(Reports).where( @@ -586,21 +587,10 @@ def format_dev_telegram(r, t): df["Gain"] = "{:,.2f}".format(r.gain) + " | " + "${:,.2f}".format(r.gain * r.want_price_at_block) df["Loss"] = "{:,.2f}".format(r.loss) + " | " + "${:,.2f}".format(r.loss * r.want_price_at_block) if r.vault_address in INVERSE_PRIVATE_VAULTS: - tx = web3.eth.getTransactionReceipt(r.txn_hash) - crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' - voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' fees = r.gov_fee_in_want + r.strategist_fee_in_want inverse_profit = r.gain - fees df["Yearn Treasury Profit"] = "{:,.2f}".format(fees) + " | " + "${:,.2f}".format(fees * r.want_price_at_block) df["Inverse Profit"] = "{:,.2f}".format(inverse_profit) + " | " + "${:,.2f}".format(inverse_profit * r.want_price_at_block) - token = web3.eth.contract(crv, abi=Contract(crv).abi) - decoded_events = token.events.Transfer().processReceipt(tx) - for t in decoded_events: - _from, _to, _val = t.args.values() - if t.address == crv and _from == r.strategy_address and _to == voter: - crv_amount = _val/1e18 - crv_value = magic.get_price(crv, r.block) * crv_amount - df["CRV Locked"] = "{:,.2f}".format(crv_amount) + " | " + "${:,.2f}".format(crv_value) else: df["Treasury Fee"] = "{:,.2f}".format(r.gov_fee_in_want) + " | " + "${:,.2f}".format(r.gov_fee_in_want * r.want_price_at_block) From 221d00abb866670d6fa8ec638f215d8e848301d5 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 12 Aug 2022 12:40:38 -0400 Subject: [PATCH 43/86] feat: arbi support --- yearn/abis.py | 90 ++++++ yearn/apy/curve/rewards.py | 15 +- yearn/apy/curve/simple.py | 269 +++++++++++++----- yearn/constants.py | 187 ++++--------- yearn/exceptions.py | 12 + yearn/networks.py | 9 +- yearn/prices/__init__.py | 4 + yearn/prices/aave.py | 34 ++- yearn/prices/balancer/__init__.py | 0 yearn/prices/balancer/balancer.py | 26 ++ yearn/prices/{balancer.py => balancer/v1.py} | 31 ++- yearn/prices/balancer/v2.py | 56 ++++ yearn/prices/band.py | 21 +- yearn/prices/chainlink.py | 56 +++- yearn/prices/compound.py | 70 +++-- yearn/prices/constants.py | 31 ++- yearn/prices/curve.py | 109 +++++--- yearn/prices/fixed_forex.py | 18 +- yearn/prices/generic_amm.py | 46 ++++ yearn/prices/incidents.py | 49 ++++ yearn/prices/magic.py | 92 +++++-- yearn/prices/synthetix.py | 15 +- yearn/prices/uniswap/uniswap.py | 69 +++++ yearn/prices/uniswap/v1.py | 39 ++- yearn/prices/uniswap/v2.py | 276 +++++++++++++++++-- yearn/prices/uniswap/v3.py | 134 +++++++-- yearn/prices/yearn.py | 34 ++- yearn/typing.py | 16 ++ yearn/utils.py | 112 +++++++- 29 files changed, 1475 insertions(+), 445 deletions(-) create mode 100644 yearn/abis.py create mode 100644 yearn/prices/balancer/__init__.py create mode 100644 yearn/prices/balancer/balancer.py rename yearn/prices/{balancer.py => balancer/v1.py} (62%) create mode 100644 yearn/prices/balancer/v2.py create mode 100644 yearn/prices/generic_amm.py create mode 100644 yearn/prices/incidents.py create mode 100644 yearn/prices/uniswap/uniswap.py create mode 100644 yearn/typing.py diff --git a/yearn/abis.py b/yearn/abis.py new file mode 100644 index 000000000..72621628b --- /dev/null +++ b/yearn/abis.py @@ -0,0 +1,90 @@ + +# This module is used to ensure necessary contracts with frequent issues are correctly defined in brownie's deployments.db + +from typing import Dict, List + +from brownie import Contract, chain + +from yearn.networks import Network +from yearn.utils import contract + + +class IncorrectABI(Exception): + pass + +# {non_verified_contract_address: verified_contract_address} +non_verified_contracts: Dict[str,str] = { + Network.Fantom: { + "0x154eA0E896695824C87985a52230674C2BE7731b": "0xbcab7d083Cf6a01e0DdA9ed7F8a02b47d125e682", + }, +}.get(chain.id,{}) + +def _fix_problematic_abis() -> None: + __force_non_verified_contracts() + __validate_unitroller_abis() + +def __force_non_verified_contracts(): + for non_verified_contract, verified_contract in non_verified_contracts.items(): + try: + contract(non_verified_contract) + except: + verified_contract = contract(verified_contract) + build_name = verified_contract._build['contractName'] + abi = verified_contract.abi + Contract.from_abi(build_name,non_verified_contract,abi) + +def __validate_unitroller_abis() -> None: + ''' + Ensure correct abi for comptrollers. + This might not always work. If it fails and your script doesn't require the `price` module, you should be fine. + If this fails and your script does require the `price` module, you will need to manually cache the correct abi in `deployments.db`. + ''' + # we can't just import the list of unitrollers, the import will fail if one of their abi definitions are messed up + unitrollers: List[str] = { + Network.Mainnet: [ + "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B", + "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258", + "0xAB1c342C7bf5Ec5F02ADEA1c2270670bCa144CbB", + ], + }.get(chain.id, []) + + good: List[Contract] = [] + bad: List[Contract] = [] + for address in unitrollers: + unitroller = contract(address) + if hasattr(unitroller,'getAllMarkets'): + good.append(unitroller) + else: + bad.append(unitroller) + + if not bad: + return + + if not good: + fixed: List[int] = [] + for i, unitroller in enumerate(bad): + unitroller = Contract.from_explorer(unitroller.address) + if hasattr(unitroller,'getAllMarkets'): + good.append(unitroller) + fixed.append(i) + fixed.sort(reverse=True) + for i in fixed: + bad.pop(i) + + if not good: + raise IncorrectABI(''' + Somehow, none of your unitrollers have a correct abi. + You will need to manually cache one or more abi into brownie + using `brownie.Contract.from_abi` in order to use this module.''') + + for unitroller in bad: + Contract.from_abi( + unitroller._build['contractName'], + unitroller.address, + good[0].abi) + + raise IncorrectABI(f""" + Re-cached Comptroller {address} that was misdefined in brownie's db. + Restarting to ensure the in-memory `contract('{address}')` cache is correct. + If you were running your script manually, please restart. + Everything will run fine upon restart.""") \ No newline at end of file diff --git a/yearn/apy/curve/rewards.py b/yearn/apy/curve/rewards.py index 1cc5923e5..945f3f9cc 100644 --- a/yearn/apy/curve/rewards.py +++ b/yearn/apy/curve/rewards.py @@ -1,11 +1,10 @@ from time import time from typing import Optional -from brownie import Contract, ZERO_ADDRESS +from brownie import ZERO_ADDRESS from yearn.apy.common import SECONDS_PER_YEAR -from yearn.utils import get_block_timestamp, contract - -from yearn.prices.magic import get_price +from yearn.prices import magic +from yearn.utils import contract, get_block_timestamp def rewards(address: str, pool_price: int, base_asset_price: int, block: Optional[int]=None) -> float: @@ -35,7 +34,7 @@ def staking(address: str, pool_price: int, base_asset_price: int, block: Optiona if token and rate: # Single reward token - token_price = get_price(token, block=block) + token_price = magic.get_price(token, block=block) return (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / ( (pool_price / 1e18) * (total_supply / 1e18) * base_asset_price ) @@ -56,7 +55,7 @@ def staking(address: str, pool_price: int, base_asset_price: int, block: Optiona except ValueError: token = None rate = data.rewardRate / 1e18 if data else 0 - token_price = get_price(token, block=block) or 0 + token_price = magic.get_price(token, block=block) or 0 apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price) queue += 1 try: @@ -84,11 +83,11 @@ def multi(address: str, pool_price: int, base_asset_price: int, block: Optional[ token = None if data.periodFinish >= time(): rate = data.rewardRate / 1e18 if data else 0 - token_price = get_price(token, block=block) or 0 + token_price = magic.get_price(token, block=block) or 0 apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price) queue += 1 try: token = multi_rewards.rewardTokens(queue, block_identifier=block) except ValueError: token = None - return apr + return apr \ No newline at end of file diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py index 89bc456d4..8fa08523f 100644 --- a/yearn/apy/curve/simple.py +++ b/yearn/apy/curve/simple.py @@ -1,15 +1,26 @@ +from dataclasses import dataclass import logging from time import time -from brownie import ZERO_ADDRESS, Contract, chain, interface +from brownie import ZERO_ADDRESS, chain, interface from semantic_version import Version from yearn.apy.common import (SECONDS_PER_YEAR, Apy, ApyError, ApyFees, ApySamples, SharePricePoint, calculate_roi) from yearn.apy.curve.rewards import rewards from yearn.networks import Network +from yearn.prices import magic from yearn.prices.curve import curve -from yearn.prices.magic import get_price -from yearn.utils import get_block_timestamp, contract +from yearn.utils import contract, get_block_timestamp + + +@dataclass +class ConvexDetailedApyData: + cvx_apr: float = 0 + cvx_apr_minus_keep_crv: float = 0 + cvx_keep_crv: float = 0 + cvx_debt_ratio: float = 0 + convex_reward_apr: float = 0 + logger = logging.getLogger(__name__) @@ -30,9 +41,9 @@ def simple(vault, samples: ApySamples) -> Apy: - if chain.id == Network.Arbitrum: - raise ApyError("crv", "not yet implemented") - + if chain.id != Network.Mainnet: + raise ApyError("crv", "chain not supported") + lp_token = vault.token.address pool_address = curve.get_pool(lp_token) @@ -52,7 +63,7 @@ def simple(vault, samples: ApySamples) -> Apy: controller = curve.gauge_controller block = samples.now - gauge_weight = controller.gauge_relative_weight.call(gauge_address, block_identifier=block) + gauge_weight = controller.gauge_relative_weight.call(gauge_address, block_identifier=block) gauge_working_supply = gauge.working_supply(block_identifier=block) if gauge_working_supply == 0: raise ApyError("crv", "gauge working supply is zero") @@ -61,9 +72,9 @@ def simple(vault, samples: ApySamples) -> Apy: pool = contract(pool_address) pool_price = pool.get_virtual_price(block_identifier=block) - base_asset_price = get_price(lp_token, block=block) or 1 + base_asset_price = magic.get_price(lp_token, block=block) or 1 - crv_price = get_price(curve.crv, block=block) + crv_price = magic.get_price(curve.crv, block=block) yearn_voter = addresses[chain.id]['yearn_voter_proxy'] y_working_balance = gauge.working_balances(yearn_voter, block_identifier=block) @@ -103,13 +114,7 @@ def simple(vault, samples: ApySamples) -> Apy: rate = reward_data['rate'] period_finish = reward_data['period_finish'] total_supply = gauge.totalSupply() - token_price = 0 - if gauge_reward_token == addresses[chain.id]['rkp3r_rewards']: - rKP3R_contract = interface.rKP3R(gauge_reward_token) - discount = rKP3R_contract.discount(block_identifier=block) - token_price = get_price(addresses[chain.id]['kp3r'], block=block) * (100 - discount) / 100 - else: - token_price = get_price(gauge_reward_token, block=block) + token_price = _get_reward_token_price(gauge_reward_token) current_time = time() if block is None else get_block_timestamp(block) if period_finish < current_time: reward_apr = 0 @@ -146,7 +151,7 @@ def simple(vault, samples: ApySamples) -> Apy: crv_keep_crv = vault.strategies[0].strategy.keepCrvPercent(block_identifier=block) / 1e4 else: crv_keep_crv = 0 - performance = (vault_contract.performanceFee(block_identifier=block) * 2) / 1e4 if hasattr(vault_contract, "performanceFee") else 0 + performance = vault_contract.performanceFee(block_identifier=block) / 1e4 if hasattr(vault_contract, "performanceFee") else 0 management = vault_contract.managementFee(block_identifier=block) / 1e4 if hasattr(vault_contract, "managementFee") else 0 else: strategy = vault.strategy @@ -157,59 +162,43 @@ def simple(vault, samples: ApySamples) -> Apy: performance = (strategist_reward + strategist_performance + treasury) / 1e4 management = 0 - - if isinstance(vault, VaultV2) and len(vault.strategies) == 2: - crv_strategy = vault.strategies[0].strategy - cvx_strategy = vault.strategies[1].strategy - convex_voter = addresses[chain.id]['convex_voter_proxy'] - cvx_working_balance = gauge.working_balances(convex_voter, block_identifier=block) - cvx_gauge_balance = gauge.balanceOf(convex_voter, block_identifier=block) - - if cvx_gauge_balance > 0: - cvx_boost = cvx_working_balance / (PER_MAX_BOOST * cvx_gauge_balance) or 1 - else: - cvx_boost = MAX_BOOST - - cvx_booster = contract(addresses[chain.id]['convex_booster']) - cvx_lock_incentive = cvx_booster.lockIncentive(block_identifier=block) - cvx_staker_incentive = cvx_booster.stakerIncentive(block_identifier=block) - cvx_earmark_incentive = cvx_booster.earmarkIncentive(block_identifier=block) - cvx_fee = (cvx_lock_incentive + cvx_staker_incentive + cvx_earmark_incentive) / 1e4 - cvx_keep_crv = cvx_strategy.keepCRV(block_identifier=block) / 1e4 - - total_cliff = 1e3 - max_supply = 1e2 * 1e6 * 1e18 # ? - reduction_per_cliff = 1e23 - cvx = contract(addresses[chain.id]['cvx']) - supply = cvx.totalSupply(block_identifier=block) - cliff = supply / reduction_per_cliff - if supply <= max_supply: - reduction = total_cliff - cliff - cvx_minted_as_crv = reduction / total_cliff - cvx_price = get_price(cvx, block=block) - converted_cvx = cvx_price / crv_price - cvx_printed_as_crv = cvx_minted_as_crv * converted_cvx - else: - cvx_printed_as_crv = 0 - - cvx_apr = ((1 - cvx_fee) * cvx_boost * base_apr) * (1 + cvx_printed_as_crv) + reward_apr - cvx_apr_minus_keep_crv = ((1 - cvx_fee) * cvx_boost * base_apr) * ((1 - cvx_keep_crv) + cvx_printed_as_crv) - + + # if the vault consists of only a convex strategy then return + # specialized apy calculations for convex + if _ConvexVault.is_convex_vault(vault): + cvx_strategy = vault.strategies[0].strategy + cvx_vault = _ConvexVault(cvx_strategy, vault, gauge) + return cvx_vault.apy(base_asset_price, pool_price, base_apr, pool_apy, management, performance) + + # if the vault has two strategies then the first is curve and the second is convex + if isinstance(vault, VaultV2) and len(vault.strategies) == 2: # this vault has curve and convex + + # The first strategy should be curve, the second should be convex. + # However the order on the vault object here does not correspond + # to the order on the withdrawal queue on chain, therefore we need to + # re-order so convex is always second if necessary + first_strategy = vault.strategies[0].strategy + second_strategy = vault.strategies[1].strategy + + crv_strategy = first_strategy + cvx_strategy = second_strategy + if not _ConvexVault.is_convex_strategy(vault.strategies[1]): + cvx_strategy = first_strategy + crv_strategy = second_strategy + + cvx_vault = _ConvexVault(cvx_strategy, vault, gauge) crv_debt_ratio = vault.vault.strategies(crv_strategy)[2] / 1e4 - cvx_debt_ratio = vault.vault.strategies(cvx_strategy)[2] / 1e4 + cvx_apy_data = cvx_vault.get_detailed_apy_data(base_asset_price, pool_price, base_apr) else: - cvx_apr = 0 - cvx_apr_minus_keep_crv = 0 - cvx_keep_crv = 0 + cvx_apy_data = ConvexDetailedApyData() crv_debt_ratio = 1 - cvx_debt_ratio = 0 crv_apr = base_apr * boost + reward_apr crv_apr_minus_keep_crv = base_apr * boost * (1 - crv_keep_crv) - gross_apr = (1 + (crv_apr * crv_debt_ratio + cvx_apr * cvx_debt_ratio)) * (1 + pool_apy) - 1 + gross_apr = (1 + (crv_apr * crv_debt_ratio + cvx_apy_data.cvx_apr * cvx_apy_data.cvx_debt_ratio)) * (1 + pool_apy) - 1 - cvx_net_apr = (cvx_apr_minus_keep_crv + reward_apr) * (1 - performance) - management + cvx_net_apr = (cvx_apy_data.cvx_apr_minus_keep_crv + cvx_apy_data.convex_reward_apr) * (1 - performance) - management cvx_net_farmed_apy = (1 + (cvx_net_apr / COMPOUNDING)) ** COMPOUNDING - 1 cvx_net_apy = ((1 + cvx_net_farmed_apy) * (1 + pool_apy)) - 1 @@ -217,20 +206,170 @@ def simple(vault, samples: ApySamples) -> Apy: crv_net_farmed_apy = (1 + (crv_net_apr / COMPOUNDING)) ** COMPOUNDING - 1 crv_net_apy = ((1 + crv_net_farmed_apy) * (1 + pool_apy)) - 1 - net_apy = crv_net_apy * crv_debt_ratio + cvx_net_apy * cvx_debt_ratio + net_apy = crv_net_apy * crv_debt_ratio + cvx_net_apy * cvx_apy_data.cvx_debt_ratio # 0.3.5+ should never be < 0% because of management if isinstance(vault, VaultV2) and net_apy < 0 and Version(vault.api_version) >= Version("0.3.5"): net_apy = 0 - fees = ApyFees(performance=performance, management=management, keep_crv=crv_keep_crv, cvx_keep_crv=cvx_keep_crv) + fees = ApyFees(performance=performance, management=management, keep_crv=crv_keep_crv, cvx_keep_crv=cvx_apy_data.cvx_keep_crv) composite = { "boost": boost, "pool_apy": pool_apy, "boosted_apr": crv_apr, "base_apr": base_apr, - "cvx_apr": cvx_apr, + "cvx_apr": cvx_apy_data.cvx_apr, "rewards_apr": reward_apr, } return Apy("crv", gross_apr, net_apy, fees, composite=composite) + +class _ConvexVault: + def __init__(self, cvx_strategy, vault, gauge, block=None) -> None: + self._cvx_strategy = cvx_strategy + self.block = block + self.vault = vault + self.gauge = gauge + + @staticmethod + def is_convex_vault(vault) -> bool: + """Determines whether the passed in vault is a Convex vault + i.e. it only has one strategy that's based on farming Convex. + """ + # prevent circular import for partners calculations + from yearn.v2.vaults import Vault as VaultV2 + if not isinstance(vault, VaultV2): + return False + + return len(vault.strategies) == 1 and _ConvexVault.is_convex_strategy(vault.strategies[0]) + + @staticmethod + def is_convex_strategy(strategy) -> bool: + return "convex" in strategy.name.lower() + + def apy(self, base_asset_price, pool_price, base_apr, pool_apy: float, management_fee: float, performance_fee: float) -> Apy: + """The standard APY data.""" + apy_data = self.get_detailed_apy_data(base_asset_price, pool_price, base_apr) + gross_apr = (1 + (apy_data.cvx_apr * apy_data.cvx_debt_ratio)) * (1 + pool_apy) - 1 + + cvx_net_apr = (apy_data.cvx_apr_minus_keep_crv + apy_data.convex_reward_apr) * (1 - performance_fee) - management_fee + cvx_net_farmed_apy = (1 + (cvx_net_apr / COMPOUNDING)) ** COMPOUNDING - 1 + cvx_net_apy = ((1 + cvx_net_farmed_apy) * (1 + pool_apy)) - 1 + + # 0.3.5+ should never be < 0% because of management + if cvx_net_apy < 0 and Version(self.vault.api_version) >= Version("0.3.5"): + cvx_net_apy = 0 + + fees = ApyFees(performance=performance_fee, management=management_fee, cvx_keep_crv=apy_data.cvx_keep_crv) + return Apy("convex", gross_apr, cvx_net_apy, fees) + + def get_detailed_apy_data(self, base_asset_price, pool_price, base_apr) -> ConvexDetailedApyData: + """Detailed data about the apy.""" + # some strategies have a localCRV property which is used based on a flag, otherwise + # falling back to the global curve config contract. + # note the spelling mistake in the variable name uselLocalCRV + if hasattr(self._cvx_strategy, "uselLocalCRV"): + use_local_crv = self._cvx_strategy.uselLocalCRV(block_identifier=self.block) + if use_local_crv: + cvx_keep_crv = self._cvx_strategy.localCRV(block_identifier=self.block) / 1e4 + else: + curve_global = contract(self._cvx_strategy.curveGlobal(block_identifier=self.block)) + cvx_keep_crv = curve_global.keepCRV(block_identifier=self.block) / 1e4 + else: + cvx_keep_crv = self._cvx_strategy.keepCRV(block_identifier=self.block) / 1e4 + + cvx_booster = contract(addresses[chain.id]['convex_booster']) + cvx_fee = self._get_convex_fee(cvx_booster, self.block) + convex_reward_apr = self._get_reward_apr(self._cvx_strategy, cvx_booster, base_asset_price, pool_price, self.block) + + cvx_boost = self._get_cvx_boost() + cvx_printed_as_crv = self._get_cvx_emissions_converted_to_crv() + cvx_apr = ((1 - cvx_fee) * cvx_boost * base_apr) * (1 + cvx_printed_as_crv) + convex_reward_apr + cvx_apr_minus_keep_crv = ((1 - cvx_fee) * cvx_boost * base_apr) * ((1 - cvx_keep_crv) + cvx_printed_as_crv) + + return ConvexDetailedApyData(cvx_apr, cvx_apr_minus_keep_crv, cvx_keep_crv, self._debt_ratio, convex_reward_apr) + + def _get_cvx_emissions_converted_to_crv(self) -> float: + """The amount of CVX emissions at the current block for a given pool, converted to CRV (from a pricing standpoint) to ease calculation of total APY.""" + crv_price = magic.get_price(curve.crv, block=self.block) + total_cliff = 1e3 # the total number of cliffs to happen + max_supply = 1e2 * 1e6 * 1e18 # the maximum amount of CVX that will be minted + reduction_per_cliff = 1e23 # the reduction in emission per cliff + cvx = contract(addresses[chain.id]['cvx']) + supply = cvx.totalSupply(block_identifier=self.block) # current supply of CVX + cliff = supply / reduction_per_cliff # the current cliff we're on + if supply <= max_supply: + reduction = total_cliff - cliff + cvx_minted = reduction / total_cliff + cvx_price = magic.get_price(cvx, block=self.block) + converted_cvx = cvx_price / crv_price + return cvx_minted * converted_cvx + else: + return 0 + + def _get_cvx_boost(self) -> float: + """The Curve boost (1-2.5x) being applied to this pool thanks to veCRV locked in Convex's voter proxy.""" + convex_voter = addresses[chain.id]['convex_voter_proxy'] + cvx_working_balance = self.gauge.working_balances(convex_voter, block_identifier=self.block) + cvx_gauge_balance = self.gauge.balanceOf(convex_voter, block_identifier=self.block) + + if cvx_gauge_balance > 0: + return cvx_working_balance / (PER_MAX_BOOST * cvx_gauge_balance) or 1 + else: + return MAX_BOOST + + def _get_reward_apr(self, cvx_strategy, cvx_booster, base_asset_price, pool_price, block=None) -> float: + """The cumulative apr of all extra tokens that are emitted by depositing + to Convex, assuming that they will be sold for profit. + """ + if hasattr(cvx_strategy, "id"): + # Convex hBTC strategy uses id rather than pid - 0x7Ed0d52C5944C7BF92feDC87FEC49D474ee133ce + pid = cvx_strategy.id() + else: + pid = cvx_strategy.pid() + + # pull data from convex's virtual rewards contracts to get bonus rewards + rewards_contract = contract(cvx_booster.poolInfo(pid)["crvRewards"]) + rewards_length = rewards_contract.extraRewardsLength() + current_time = time() if block is None else get_block_timestamp(block) + if rewards_length == 0: + return 0 + + convex_reward_apr = 0 # reset our rewards apr if we're calculating it via convex + + for x in range(rewards_length): + virtual_rewards_pool = contract(rewards_contract.extraRewards(x)) + # do this for all assets, which will duplicate much of the curve info but we don't want to miss anything + if virtual_rewards_pool.periodFinish() > current_time: + reward_token = virtual_rewards_pool.rewardToken() + reward_token_price = _get_reward_token_price(reward_token, block) + + reward_apr = (virtual_rewards_pool.rewardRate() * SECONDS_PER_YEAR * reward_token_price) / (base_asset_price * (pool_price / 1e18) * virtual_rewards_pool.totalSupply()) + convex_reward_apr += reward_apr + + return convex_reward_apr + + def _get_convex_fee(self, cvx_booster, block=None) -> float: + """The fee % that Convex charges on all CRV yield.""" + cvx_lock_incentive = cvx_booster.lockIncentive(block_identifier=block) + cvx_staker_incentive = cvx_booster.stakerIncentive(block_identifier=block) + cvx_earmark_incentive = cvx_booster.earmarkIncentive(block_identifier=block) + cvx_platform_fee = cvx_booster.platformFee(block_identifier=block) + return (cvx_lock_incentive + cvx_staker_incentive + cvx_earmark_incentive + cvx_platform_fee) / 1e4 + + @property + def _debt_ratio(self) -> float: + """The debt ratio of the Convex strategy.""" + return self.vault.vault.strategies(self._cvx_strategy)[2] / 1e4 + + +def _get_reward_token_price(reward_token, block=None): + # if the reward token is rKP3R we need to calculate it's price in + # terms of KP3R after the discount + contract_addresses = addresses[chain.id] + if reward_token == contract_addresses['rkp3r_rewards']: + rKP3R_contract = interface.rKP3R(reward_token) + discount = rKP3R_contract.discount(block_identifier=block) + return magic.get_price(contract_addresses['kp3r'], block=block) * (100 - discount) / 100 + else: + return magic.get_price(reward_token, block=block) \ No newline at end of file diff --git a/yearn/constants.py b/yearn/constants.py index 050f81a5b..e7bb181a6 100644 --- a/yearn/constants.py +++ b/yearn/constants.py @@ -1,146 +1,79 @@ -from brownie import interface, chain +from brownie import chain, convert + from yearn.networks import Network -CONTROLLER_INTERFACES = { - "0x2be5D998C95DE70D9A38b3d78e49751F10F9E88b": interface.ControllerV1, - "0x9E65Ad11b299CA0Abefc2799dDB6314Ef2d91080": interface.ControllerV2, -} +WRAPPED_GAS_COIN = { + Network.Mainnet: "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", + Network.Fantom: "0x21be370D5312f44cB42ce377BC9b8a0cEF1A4C83", + Network.Arbitrum: "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1", + Network.Gnosis: "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d", +}.get(chain.id, None) -VAULT_INTERFACES = { - "0x29E240CFD7946BA20895a7a02eDb25C210f9f324": interface.yDelegatedVault, - "0x881b06da56BB5675c54E4Ed311c21E54C5025298": interface.yWrappedVault, - "0xc5bDdf9843308380375a611c18B50Fb9341f502A": interface.yveCurveVault, -} +YEARN_ADDRESSES_PROVIDER = "0x9be19Ee7Bc4099D62737a7255f5c227fBcd6dB93" +CURVE_ADDRESSES_PROVIDER = "0x0000000022D53366457F9d5E68Ec105046FC4383" -STRATEGY_INTERFACES = { - "0x25fAcA21dd2Ad7eDB3a027d543e617496820d8d6": interface.StrategyVaultUSDC, - "0xA30d1D98C502378ad61Fe71BcDc3a808CF60b897": interface.StrategyDForceUSDC, - "0x1d91E3F77271ed069618b4BA06d19821BC2ed8b0": interface.StrategyTUSDCurve, - "0xAa880345A3147a1fC6889080401C791813ed08Dc": interface.StrategyDAICurve, - "0x787C771035bDE631391ced5C083db424A4A64bD8": interface.StrategyDForceUSDT, - "0x932fc4fd0eEe66F22f1E23fBA74D7058391c0b15": interface.StrategyMKRVaultDAIDelegate, - "0xF147b8125d2ef93FB6965Db97D6746952a133934": interface.CurveYCRVVoter, - "0x112570655b32A8c747845E0215ad139661e66E7F": interface.StrategyCurveBUSDVoterProxy, - "0x6D6c1AD13A5000148Aa087E7CbFb53D402c81341": interface.StrategyCurveBTCVoterProxy, - "0x07DB4B9b3951094B9E278D336aDf46a036295DE7": interface.StrategyCurveYVoterProxy, - "0xC59601F0CC49baa266891b7fc63d2D5FE097A79D": interface.StrategyCurve3CrvVoterProxy, - "0x395F93350D5102B6139Abfc84a7D6ee70488797C": interface.StrategyYFIGovernance, - "0xc8327D8E1094a94466e05a2CC1f10fA70a1dF119": interface.StrategyCurveGUSDProxy, - "0x530da5aeF3c8f9CCbc75C97C182D6ee2284B643F": interface.StrategyCurveCompoundVoterProxy, - "0x4720515963A9d40ca10B1aDE806C1291E6c9A86d": interface.StrategyUSDC3pool, - "0xe3a711987612BFD1DAFa076506f3793c78D81558": interface.StrategyTUSDypool, - "0xc7e437033D849474074429Cbe8077c971Ea2a852": interface.StrategyUSDT3pool, - "0xBA0c07BBE9C22a1ee33FE988Ea3763f21D0909a0": interface.StrategyCurvemUSDVoterProxy, - "0xD42eC70A590C6bc11e9995314fdbA45B4f74FABb": interface.StrategyCurveGUSDVoterProxy, - "0xF4Fd9B4dAb557DD4C9cf386634d61231D54d03d6": interface.StrategyGUSDRescue, - "0x9c211BFa6DC329C5E757A223Fb72F5481D676DC1": interface.StrategyDAI3pool, - "0x39AFF7827B9D0de80D86De295FE62F7818320b76": interface.StrategyMKRVaultDAIDelegate, - "0x22422825e2dFf23f645b04A3f89190B69f174659": interface.StrategyCurveEURVoterProxy, - "0x6f1EbF5BBc5e32fffB6B3d237C3564C15134B8cF": interface.StrategymUSDCurve, - "0x76B29E824C183dBbE4b27fe5D8EdF0f926340750": interface.StrategyCurveRENVoterProxy, - "0x406813fF2143d178d1Ebccd2357C20A424208912": interface.StrategyCurveUSDNVoterProxy, - "0x3be2717DA725f43b7d6C598D8f76AeC43e231B99": interface.StrategyCurveUSTVoterProxy, - "0x15CfA851403aBFbbD6fDB1f6fe0d32F22ddc846a": interface.StrategyCurveOBTCVoterProxy, - "0xD96041c5EC05735D965966bF51faEC40F3888f6e": interface.StrategyCurvePBTCVoterProxy, - "0x61A01a704665b3C0E6898C1B4dA54447f561889d": interface.StrategyCurveTBTCVoterProxy, - "0x551F41aD4ebeCa4F5d025D2B3082b7AB2383B768": interface.StrategyCurveBBTCVoterProxy, - "0xE02363cB1e4E1B77a74fAf38F3Dbb7d0B70F26D7": interface.StrategyCurveHBTCVoterProxy, - "0xd7F641697ca4e0e19F6C9cF84989ABc293D24f84": interface.StrategyCurvesUSDVoterProxy, - "0xb21C4d2f7b2F29109FF6243309647A01bEB9950a": interface.StrategyCurveHUSDVoterProxy, - "0x33F3f002b8f812f3E087E9245921C8355E777231": interface.StrategyCurveDUSDVoterProxy, - "0x7A10bE29c4d9073E6B3B6b7D1fB5bCDBecA2AA1F": interface.StrategyCurvea3CRVVoterProxy, - "0xBdCeae91e10A80dbD7ad5e884c86EAe56b075Caa": interface.StrategyCurveAnkrVoterProxy, - "0x2F90c531857a2086669520e772E9d433BbfD5496": interface.StrategyDAI3pool, - "0xBcC6abd115a32fC27f7B49F9e17D0BcefDd278aC": interface.StrategyCurvemUSDVoterProxy, - "0x83e7399113561ae691c413ed334137D3839e2302": interface.StrategyCurveEURVoterProxy, - "0x4f2fdebE0dF5C92EEe77Ff902512d725F6dfE65c": interface.StrategyUSDC3pool, - "0xAa12d6c9d680EAfA48D8c1ECba3FCF1753940A12": interface.StrategyUSDT3pool, - "0x4BA03330338172fEbEb0050Be6940c6e7f9c91b0": interface.StrategyTUSDypool, - "0x8e2057b8fe8e680B48858cDD525EBc9510620621": interface.StrategyCurvesaCRVVoterProxy, -} +# EVENTS +ERC20_TRANSFER_EVENT_HASH = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' +ERC677_TRANSFER_EVENT_HASH = '0xe19260aff97b920c7df27010903aeb9c8d2be5d310a2c67824cf3f15396e4c16' -VAULT_ALIASES = { - "0x29E240CFD7946BA20895a7a02eDb25C210f9f324": "aLINK", - "0x881b06da56BB5675c54E4Ed311c21E54C5025298": "LINK", - "0x597aD1e0c13Bfe8025993D9e79C69E1c0233522e": "USDC", - "0x5dbcF33D8c2E976c6b560249878e6F1491Bca25c": "curve.fi/y", - "0x37d19d1c4E1fa9DC47bD1eA12f742a0887eDa74a": "TUSD", - "0xACd43E627e64355f1861cEC6d3a6688B31a6F952": "DAI", - "0x2f08119C6f07c006695E079AAFc638b8789FAf18": "USDT", - "0xBA2E7Fed597fd0E3e70f5130BcDbbFE06bB94fe1": "YFI", - "0x2994529C0652D127b7842094103715ec5299bBed": "curve.fi/busd", - "0x7Ff566E1d69DEfF32a7b244aE7276b9f90e9D0f6": "curve.fi/sbtc", - "0xe1237aA7f535b0CC33Fd973D66cBf830354D16c7": "WETH", - "0x9cA85572E6A3EbF24dEDd195623F188735A5179f": "curve.fi/3pool", - "0xec0d8D3ED5477106c6D4ea27D90a60e594693C90": "GUSD", - "0x629c759D1E83eFbF63d84eb3868B564d9521C129": "curve.fi/compound", - "0xcC7E70A958917cCe67B4B87a8C30E6297451aE98": "curve.fi/gusd", - "0x0FCDAeDFb8A7DfDa2e9838564c5A1665d856AFDF": "curve.fi/musd", - "0x98B058b2CBacF5E99bC7012DF757ea7CFEbd35BC": "curve.fi/eurs", - "0xE0db48B4F71752C4bEf16De1DBD042B82976b8C7": "mUSD", - "0x5334e150B938dd2b6bd040D9c4a03Cff0cED3765": "curve.fi/renbtc", - "0xFe39Ce91437C76178665D64d7a2694B0f6f17fE3": "curve.fi/usdn", - "0xF6C9E9AF314982A4b38366f4AbfAa00595C5A6fC": "curve.fi/ust", - "0x7F83935EcFe4729c4Ea592Ab2bC1A32588409797": "curve.fi/obtc", - "0x123964EbE096A920dae00Fb795FFBfA0c9Ff4675": "curve.fi/pbtc", - "0x07FB4756f67bD46B748b16119E802F1f880fb2CC": "curve.fi/tbtc", - "0xA8B1Cb4ed612ee179BDeA16CCa6Ba596321AE52D": "curve.fi/bbtc", - "0x46AFc2dfBd1ea0c0760CAD8262A5838e803A37e5": "curve.fi/hbtc", - "0x39546945695DCb1c037C836925B355262f551f55": "curve.fi/husd", - "0x8e6741b456a074F0Bc45B8b82A755d4aF7E965dF": "curve.fi/dusd", - "0x5533ed0a3b83F70c3c4a1f69Ef5546D3D4713E44": "curve.fi/susd", - "0x03403154afc09Ce8e44C3B185C82C6aD5f86b9ab": "curve.fi/aave", - "0xE625F5923303f1CE7A43ACFEFd11fd12f30DbcA4": "curve.fi/ankreth", - "0xBacB69571323575C6a5A3b4F9EEde1DC7D31FBc1": "curve.fi/saave", - "0x1B5eb1173D2Bf770e50F10410C9a96F7a8eB6e75": "curve.fi/usdp", - "0x96Ea6AF74Af09522fCB4c28C269C26F59a31ced6": "curve.fi/link", -} +# ADDRESSES +STRATEGIST_MULTISIG = { + Network.Mainnet: { + "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", + }, + Network.Fantom: { + "0x72a34AbafAB09b15E7191822A679f28E067C4a16", + }, + Network.Gnosis: { + "0xFB4464a18d18f3FF439680BBbCE659dB2806A187", + }, + Network.Arbitrum: { + "0x6346282db8323a54e840c6c772b4399c9c655c0d", + } +}.get(chain.id,set()) -BTC_LIKE = { - "0xEB4C2781e4ebA804CE9a9803C67d0893436bB27D", # renbtc - "0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599", # wbtc - "0xfE18be6b3Bd88A2D2A7f928d00292E7a9963CfC6", # sbtc - "0x8064d9Ae6cDf087b1bcd5BDf3531bD5d8C537a68", # obtc - "0x9BE89D2a4cd102D8Fecc6BF9dA793be995C22541", # bbtc - "0x0316EB71485b0Ab14103307bf65a021042c6d380", # hbtc - "0x5228a22e72ccC52d415EcFd199F99D0665E7733b", # pbtc - "0x8dAEBADE922dF735c38C80C7eBD708Af50815fAa", # tbtc -} +STRATEGIST_MULTISIG = {convert.to_address(address) for address in STRATEGIST_MULTISIG} -ETH_LIKE = { - "0x5e74C9036fb86BD7eCdcb084a0673EFc32eA31cb", # seth - "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE", # eth - "0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84", # steth - "0x9559Aaa82d9649C7A7b220E7c461d2E74c9a3593", # reth - "0xE95A203B1a91a908F9B9CE46459d101078c2c3cb", # ankreth -} +YCHAD_MULTISIG = { + Network.Mainnet: "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", + Network.Fantom: "0xC0E2830724C946a6748dDFE09753613cd38f6767", + Network.Gnosis: "0x22eAe41c7Da367b9a15e942EB6227DF849Bb498C", + Network.Arbitrum: "0xb6bc033d34733329971b938fef32fad7e98e56ad", +}.get(chain.id, None) -YEARN_ADDRESSES_PROVIDER = "0x9be19Ee7Bc4099D62737a7255f5c227fBcd6dB93" -CURVE_ADDRESSES_PROVIDER = "0x0000000022D53366457F9d5E68Ec105046FC4383" +if YCHAD_MULTISIG: + YCHAD_MULTISIG = convert.to_address(YCHAD_MULTISIG) + +TREASURY_MULTISIG = { + Network.Mainnet: "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", + Network.Fantom: "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", + Network.Arbitrum: "0x1deb47dcc9a35ad454bf7f0fcdb03c09792c08c1", +}.get(chain.id, None) + +if TREASURY_MULTISIG: + TREASURY_MULTISIG = convert.to_address(TREASURY_MULTISIG) TREASURY_WALLETS = { Network.Mainnet: { - "0x5f0845101857d2A91627478e302357860b1598a1", # Yearn KP3R Wallet - "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", # Yearn Treasury + TREASURY_MULTISIG, + YCHAD_MULTISIG, "0xb99a40fcE04cb740EB79fC04976CA15aF69AaaaE", # Yearn Treasury V1 - "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", # Yearn MultiSig + "0x5f0845101857d2A91627478e302357860b1598a1", # Yearn KP3R Wallet "0x7d2aB9CA511EBD6F03971Fb417d3492aA82513f0", # ySwap Multisig "0x2C01B4AD51a67E2d8F02208F54dF9aC4c0B778B6", # yMechs Multisig }, Network.Fantom: { - "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", # Yearn Multisig - } -}.get(chain.id,set()) - -# EVENTS -ERC20_TRANSFER_EVENT_HASH = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' -ERC677_TRANSFER_EVENT_HASH = '0xe19260aff97b920c7df27010903aeb9c8d2be5d310a2c67824cf3f15396e4c16' - -STRATEGIST_MULTISIG = { - Network.Mainnet: { - "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", + TREASURY_MULTISIG, + YCHAD_MULTISIG, }, - Network.Fantom: { - "0x72a34AbafAB09b15E7191822A679f28E067C4a16", + Network.Gnosis: { + YCHAD_MULTISIG, + "0x5FcdC32DfC361a32e9d5AB9A384b890C62D0b8AC", # Yearn Treasury (EOA?) + }, + Network.Arbitrum: { + YCHAD_MULTISIG, + TREASURY_MULTISIG, } }.get(chain.id,set()) + +TREASURY_WALLETS = {convert.to_address(address) for address in TREASURY_WALLETS} \ No newline at end of file diff --git a/yearn/exceptions.py b/yearn/exceptions.py index 31a59e3f0..2de77acb4 100644 --- a/yearn/exceptions.py +++ b/yearn/exceptions.py @@ -12,3 +12,15 @@ class ArchiveNodeRequired(Exception): class MulticallError(Exception): pass + + +class EmptyS3Export(Exception): + pass + + +class NodeNotSynced(Exception): + pass + + +class BatchSizeError(Exception): + pass \ No newline at end of file diff --git a/yearn/networks.py b/yearn/networks.py index 9a63b1406..ce5164533 100644 --- a/yearn/networks.py +++ b/yearn/networks.py @@ -1,3 +1,5 @@ + + from enum import IntEnum from brownie import chain @@ -7,6 +9,7 @@ class Network(IntEnum): Mainnet = 1 + Gnosis = 100 Fantom = 250 Arbitrum = 42161 @@ -17,11 +20,13 @@ def label(chain_id: int = None): if chain_id == Network.Mainnet: return "ETH" + elif chain_id == Network.Gnosis: + return "GNO" elif chain_id == Network.Fantom: return "FTM" elif chain_id == Network.Arbitrum: - return "ARRB" + return "ARBB" else: raise UnsupportedNetwork( f'chainid {chain_id} is not currently supported. Please add network details to yearn-exporter/yearn/networks.py' - ) + ) \ No newline at end of file diff --git a/yearn/prices/__init__.py b/yearn/prices/__init__.py index e69de29bb..ada4ec154 100644 --- a/yearn/prices/__init__.py +++ b/yearn/prices/__init__.py @@ -0,0 +1,4 @@ + +from yearn.abis import _fix_problematic_abis + +_fix_problematic_abis() diff --git a/yearn/prices/aave.py b/yearn/prices/aave.py index 0e3f667f9..8f2c7251b 100644 --- a/yearn/prices/aave.py +++ b/yearn/prices/aave.py @@ -1,12 +1,14 @@ -from typing import Optional +from typing import Dict, List, Literal, Optional -from brownie import chain +from brownie import Contract, chain, web3 +from brownie.convert.datatypes import EthAddress from cachetools.func import ttl_cache from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network -from yearn.utils import Singleton, contract +from yearn.typing import Address, AddressOrContract +from yearn.utils import Singleton, contract, _resolve_proxy address_providers = { Network.Mainnet: { @@ -23,28 +25,22 @@ class Aave(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in address_providers: raise UnsupportedNetwork("aave is not supported on this network") - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: return token in self.markets - def atoken_underlying(self, atoken: str) -> Optional[str]: + def atoken_underlying(self, atoken: AddressOrContract) -> Optional[EthAddress]: return self.markets.get(atoken) @property @ttl_cache(ttl=3600) - def markets(self): + def markets(self) -> Dict[EthAddress,EthAddress]: atoken_to_token = {} for version, provider in address_providers[chain.id].items(): - lending_pool = contract(contract(provider).getLendingPool()) - if version == 'v1': - tokens = lending_pool.getReserves() - elif version == 'v2': - tokens = lending_pool.getReservesList() - else: - raise ValueError(f'unsupported aave version {version}') + lending_pool, tokens = self.get_tokens(contract(contract(provider).getLendingPool()), version) reserves = fetch_multicall( *[[lending_pool, 'getReserveData', token] for token in tokens] @@ -57,6 +53,16 @@ def markets(self): return atoken_to_token + def get_tokens(self, lending_pool: Contract, version: Literal['v1','v2']) -> List[Address]: + fns_by_version = {"v1": "getReserves", "v2": "getReservesList"} + if version not in fns_by_version: + raise ValueError(f'unsupported aave version {version}') + fn = fns_by_version[version] + if not hasattr(lending_pool, fn): + lending_pool = _resolve_proxy(str(lending_pool)) + tokens = getattr(lending_pool, fn)() + return lending_pool, tokens + aave = None try: aave = Aave() diff --git a/yearn/prices/balancer/__init__.py b/yearn/prices/balancer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/yearn/prices/balancer/balancer.py b/yearn/prices/balancer/balancer.py new file mode 100644 index 000000000..dbc884867 --- /dev/null +++ b/yearn/prices/balancer/balancer.py @@ -0,0 +1,26 @@ +import logging +from typing import Optional, Union +from yearn.prices.balancer.v1 import BalancerV1, balancer_v1 +from yearn.prices.balancer.v2 import BalancerV2, balancer_v2 +from yearn.typing import Address + +logger = logging.getLogger(__name__) + +BALANCERS = { + "v1": balancer_v1, + "v2": balancer_v2 +} +class BalancerSelector: + def __init__(self) -> None: + self.balancers = { + version: balancer for version, balancer in BALANCERS.items() if balancer + } + + def get_balancer_for_pool(self, address: Address) -> Optional[Union[BalancerV1, BalancerV2]]: + for b in BALANCERS.values(): + if b and b.is_balancer_pool(address): + return b + + return None + +selector = BalancerSelector() diff --git a/yearn/prices/balancer.py b/yearn/prices/balancer/v1.py similarity index 62% rename from yearn/prices/balancer.py rename to yearn/prices/balancer/v1.py index 38f4218b8..481827cc3 100644 --- a/yearn/prices/balancer.py +++ b/yearn/prices/balancer/v1.py @@ -1,4 +1,5 @@ -from brownie import Contract, chain +from typing import Any, Literal, Optional, List +from brownie import chain from cachetools.func import ttl_cache from yearn.cache import memory @@ -6,31 +7,37 @@ from yearn.prices import magic from yearn.utils import contract, Singleton from yearn.networks import Network +from yearn.typing import Address, Block from yearn.exceptions import UnsupportedNetwork networks = [ Network.Mainnet ] @memory.cache() -def is_balancer_pool_cached(address): +def is_balancer_pool_cached(address: Address) -> bool: pool = contract(address) required = {"getCurrentTokens", "getBalance", "totalSupply"} - if set(pool.__dict__) & required == required: - return True - return False + return required.issubset(set(pool.__dict__)) -class Balancer(metaclass=Singleton): - def __init__(self): +class BalancerV1(metaclass=Singleton): + def __init__(self) -> None: if chain.id not in networks: raise UnsupportedNetwork('Balancer is not supported on this network') - def __contains__(self, token): + def __contains__(self, token: Any) -> Literal[False]: return False - def is_balancer_pool(self, address): + def is_balancer_pool(self, address: Address) -> bool: return is_balancer_pool_cached(address) + def get_version(self) -> str: + return "v1" + + def get_tokens(self, token: Address) -> List: + pool = contract(token) + return pool.getCurrentTokens() + @ttl_cache(ttl=600) - def get_price(self, token, block=None): + def get_price(self, token: Address, block: Optional[Block] = None) -> float: pool = contract(token) tokens, supply = fetch_multicall([pool, "getCurrentTokens"], [pool, "totalSupply"], block=block) supply = supply / 1e18 @@ -39,8 +46,8 @@ def get_price(self, token, block=None): total = sum(balance * magic.get_price(token, block=block) for balance, token in zip(balances, tokens)) return total / supply -balancer = None +balancer_v1 = None try: - balancer = Balancer() + balancer_v1 = BalancerV1() except UnsupportedNetwork: pass diff --git a/yearn/prices/balancer/v2.py b/yearn/prices/balancer/v2.py new file mode 100644 index 000000000..6fffc7ccd --- /dev/null +++ b/yearn/prices/balancer/v2.py @@ -0,0 +1,56 @@ +from typing import Any, Literal, Optional, List +from brownie import chain +from cachetools.func import ttl_cache + +from yearn.cache import memory +from yearn.multicall2 import fetch_multicall +from yearn.prices import magic +from yearn.utils import contract, Singleton +from yearn.networks import Network +from yearn.typing import Address, Block +from yearn.exceptions import UnsupportedNetwork + +networks = [ Network.Mainnet ] + +@memory.cache() +def is_balancer_pool_cached(address: Address) -> bool: + pool = contract(address) + required = {"getVault", "getPoolId", "totalSupply"} + return required.issubset(set(pool.__dict__)) + +class BalancerV2(metaclass=Singleton): + def __init__(self) -> None: + if chain.id not in networks: + raise UnsupportedNetwork('Balancer is not supported on this network') + + def __contains__(self, token: Any) -> Literal[False]: + return False + + def is_balancer_pool(self, address: Address) -> bool: + return is_balancer_pool_cached(address) + + def get_version(self) -> str: + return "v2" + + def get_tokens(self, token: Address, block: Optional[Block] = None) -> List: + pool = contract(token) + pool_id = pool.getPoolId() + vault = contract(pool.getVault()) + return vault.getPoolTokens(pool_id, block_identifier=block)[0] + + @ttl_cache(ttl=600) + def get_price(self, token: Address, block: Optional[Block] = None) -> float: + pool = contract(token) + pool_id = pool.getPoolId() + vault = contract(pool.getVault()) + tokens = vault.getPoolTokens(pool_id, block_identifier=block) + balances = [balance for t, balance in zip(tokens[0], tokens[1]) if t != token] + total = sum(balance * magic.get_price(t, block=block) for t, balance in zip(tokens[0], tokens[1]) if t != token) + supply = sum(balances) + return total / supply + +balancer_v2 = None +try: + balancer_v2 = BalancerV2() +except UnsupportedNetwork: + pass diff --git a/yearn/prices/band.py b/yearn/prices/band.py index 92e31350d..a70369646 100644 --- a/yearn/prices/band.py +++ b/yearn/prices/band.py @@ -1,12 +1,13 @@ -from functools import cached_property -from brownie import chain +from typing import Optional +from brownie import chain +from brownie.exceptions import VirtualMachineError from cachetools.func import ttl_cache -from yearn.utils import Singleton -from yearn.networks import Network -from yearn.exceptions import UnsupportedNetwork -from yearn.utils import contract +from yearn.exceptions import UnsupportedNetwork +from yearn.networks import Network +from yearn.typing import Address, AddressOrContract, Block +from yearn.utils import Singleton, contract addresses = { # https://docs.fantom.foundation/tutorials/band-protocol-standard-dataset @@ -38,21 +39,23 @@ } class Band(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork('band is not supported on this network') self.oracle = contract(addresses[chain.id]) - def __contains__(self, asset): + def __contains__(self, asset: AddressOrContract) -> bool: return chain.id in addresses and asset in supported_assets[chain.id] @ttl_cache(maxsize=None, ttl=600) - def get_price(self, asset, block=None): + def get_price(self, asset: Address, block: Optional[Block] = None) -> Optional[float]: asset_symbol = contract(asset).symbol() try: return self.oracle.getReferenceData(asset_symbol, 'USDC', block_identifier=block)[0] / 1e18 except ValueError: return None + except VirtualMachineError: + return None band = None diff --git a/yearn/prices/chainlink.py b/yearn/prices/chainlink.py index 1f55a6606..663c5ff86 100644 --- a/yearn/prices/chainlink.py +++ b/yearn/prices/chainlink.py @@ -1,10 +1,14 @@ import logging +from typing import Optional -from brownie import ZERO_ADDRESS, chain +from brownie import ZERO_ADDRESS, Contract, chain, convert +from brownie.exceptions import VirtualMachineError from cachetools.func import ttl_cache + from yearn.events import decode_logs, get_logs_asap from yearn.exceptions import UnsupportedNetwork from yearn.networks import Network +from yearn.typing import Address, AddressOrContract, Block from yearn.utils import Singleton, contract logger = logging.getLogger(__name__) @@ -30,7 +34,9 @@ Network.Fantom: { "0x21be370D5312f44cB42ce377BC9b8a0cEF1A4C83": "0xf4766552D15AE4d256Ad41B6cf2933482B0680dc", # wftm "0x321162Cd933E2Be498Cd2267a90534A804051b11": "0x8e94C22142F4A64b99022ccDd994f4e9EC86E4B4", # wbtc + "0x2406dCe4dA5aB125A18295f4fB9FD36a0f7879A2": "0x8e94C22142F4A64b99022ccDd994f4e9EC86E4B4", # anybtc "0x74b23882a30290451A17c44f4F05243b6b58C76d": "0x11DdD3d147E5b83D01cee7070027092397d63658", # weth + "0xBDC8fd437C489Ca3c6DA3B5a336D11532a532303": "0x11DdD3d147E5b83D01cee7070027092397d63658", # anyeth "0xd6070ae98b8069de6B494332d1A1a81B6179D960": "0x4F5Cc6a2291c964dEc4C7d6a50c0D89492d4D91B", # bifi "0x1E4F97b9f9F913c46F1632781732927B9019C68b": "0xa141D7E3B44594cc65142AE5F2C7844Abea66D2B", # crv "0x6a07A792ab2965C72a5B8088d3a069A7aC3a993B": "0xE6ecF7d2361B6459cBb3b4fb065E0eF4B175Fe74", # aave @@ -44,16 +50,43 @@ "0xe105621721D1293c27be7718e041a4Ce0EbB227E": "0x3E68e68ea2c3698400465e3104843597690ae0f7", # feur "0x29b0Da86e484E1C0029B56e817912d778aC0EC69": "0x9B25eC3d6acfF665DfbbFD68B3C1D896E067F0ae", # yfi }, + + Network.Gnosis: { + "0x8e5bBbb09Ed1ebdE8674Cda39A0c169401db4252" : "0x6c1d7e76ef7304a40e8456ce883bc56d3dea3f7d", # wbtc + "0x6A023CCd1ff6F2045C3309768eAd9E68F978f6e1" : "0xa767f745331d267c7751297d982b050c93985627", # weth + "0xDDAfbb505ad214D7b80b1f830fcCc89B60fb7A83" : "0x26c31ac71010af62e6b486d1132e266d6298857d", # usdc + "0x712b3d230F3C1c19db860d80619288b1F0BDd0Bd" : "0xc77b83ac3dd2a761073bd0f281f7b880b2ddde18", # crv + "0xDF613aF6B44a31299E48131e9347F034347E2F00" : "0x2b481dc923aa050e009113dca8dcb0dab4b68cdf", # aave + "0xE2e73A1c69ecF83F464EFCE6A5be353a37cA09b2" : "0xed322a5ac55bae091190dff9066760b86751947b", # link + "0x3A00E08544d589E19a8e7D97D0294331341cdBF6" : "0x3b84d6e6976d5826500572600eb44f9f1753827b", # snx + "0x2995D1317DcD4f0aB89f4AE60F3f020A4F17C7CE" : "0xc0a6bf8d5d408b091d022c3c0653d4056d4b9c01", # sushi + "0x44fA8E6f47987339850636F88629646662444217" : "0x678df3415fc31947da4324ec63212874be5a82f8", # dai + "0xbf65bfcb5da067446CeE6A706ba3Fe2fB1a9fdFd" : "0x14030d5a0c9e63d9606c6f2c8771fc95b34b07e0", # yfi + "0x7f7440C5098462f833E123B44B8A03E1d9785BAb" : "0xfdf9eb5fafc11efa65f6fd144898da39a7920ae8", # 1inch + "0xDf6FF92bfDC1e8bE45177DC1f4845d391D3ad8fD" : "0xba95bc8418ebcdf8a690924e1d4ad5292139f2ea", # comp + }, + + Network.Arbitrum: { + "0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f" : "0x6ce185860a4963106506C203335A2910413708e9", # wbtc + "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1" : "0x639Fe6ab55C921f74e7fac1ee960C0B6293ba612", # weth + "0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1" : "0xc5C8E77B397E531B8EC06BFb0048328B30E9eCfB", # dai + "0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9" : "0x3f3f5dF88dC9F13eac63DF89EC16ef6e7E25DdE7", # usdt + "0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8" : "0x50834F3163758fcC1Df9973b6e91f0F0F0434aD3", # usdc + "0xFEa7a6a0B346362BF88A9e4A88416B77a57D6c2A" : "0x87121F6c9A9F6E90E59591E4Cf4804873f54A95b", # mim + "0x82e3A8F066a6989666b031d916c43672085b1582" : "0x745Ab5b69E01E2BE1104Ca84937Bb71f96f5fB21", # yfi + } } registries = { # https://docs.chain.link/docs/feed-registry/#contract-addresses Network.Mainnet: '0x47Fb2585D2C56Fe188D0E6ec628a38b74fCeeeDf', Network.Fantom: None, + Network.Gnosis: None, + Network.Arbitrum: None, } class Chainlink(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in registries: raise UnsupportedNetwork('chainlink is not supported on this network') @@ -63,7 +96,7 @@ def __init__(self): else: self.feeds = ADDITIONAL_FEEDS[chain.id] - def load_feeds(self): + def load_feeds(self) -> None: logs = decode_logs( get_logs_asap(str(self.registry), [self.registry.topics['FeedConfirmed']]) ) @@ -75,19 +108,22 @@ def load_feeds(self): self.feeds.update(ADDITIONAL_FEEDS[chain.id]) logger.info(f'loaded {len(self.feeds)} feeds') - def get_feed(self, asset): - return contract(self.feeds[asset]) + def get_feed(self, asset: AddressOrContract) -> Contract: + return contract(self.feeds[convert.to_address(asset)]) - def __contains__(self, asset): - return asset in self.feeds + def __contains__(self, asset: AddressOrContract) -> bool: + return convert.to_address(asset) in self.feeds @ttl_cache(maxsize=None, ttl=600) - def get_price(self, asset, block=None): + def get_price(self, asset: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: if asset == ZERO_ADDRESS: return None try: - return self.get_feed(asset).latestAnswer(block_identifier=block) / 1e8 - except ValueError: + price = self.get_feed(convert.to_address(asset)).latestAnswer(block_identifier=block) / 1e8 + # latestAnswer can return 0 before the feed is in use + if price: + return price + except (ValueError, VirtualMachineError): return None diff --git a/yearn/prices/compound.py b/yearn/prices/compound.py index 3c6c21f9e..2aaaf1898 100644 --- a/yearn/prices/compound.py +++ b/yearn/prices/compound.py @@ -1,33 +1,40 @@ +import logging +from dataclasses import dataclass from functools import cached_property -from sys import base_prefix -from typing import Optional -from brownie import chain, Contract +from typing import Any, Callable, List, Optional, Union + +from brownie import Contract, chain +from brownie.convert.datatypes import EthAddress from brownie.network.contract import ContractContainer +from cachetools.func import ttl_cache + from yearn.exceptions import UnsupportedNetwork -from yearn.utils import Singleton, contract -from yearn.multicall2 import fetch_multicall from yearn.networks import Network -import logging -from cachetools.func import ttl_cache -from dataclasses import dataclass from yearn.prices.constants import usdc, weth -from typing import Callable, Union +from yearn.typing import Address, AddressOrContract, Block +from yearn.utils import Singleton, contract logger = logging.getLogger(__name__) -def get_fantom_ironbank(): +def get_fantom_ironbank() -> Contract: # HACK ironbank on fantom uses a non-standard proxy pattern unitroller = contract('0x4250A6D3BD57455d7C6821eECb6206F507576cD2') implementation = contract(unitroller.comptrollerImplementation()) return Contract.from_abi(unitroller._name, str(unitroller), abi=implementation.abi) +def get_fantom_scream() -> Contract: + # HACK ironbank on fantom uses a non-standard proxy pattern + unitroller = contract('0x260e596dabe3afc463e75b6cc05d8c46acacfb09') + implementation = contract(unitroller.comptrollerImplementation()) + return Contract.from_abi(unitroller._name, str(unitroller), abi=implementation.abi) + @dataclass class CompoundConfig: name: str - address: Union[str, Callable[[], ContractContainer]] - oracle_base: str = usdc + address: Union[Address, Callable[[], ContractContainer]] + oracle_base: Address = usdc addresses = { @@ -51,6 +58,10 @@ class CompoundConfig: name='ironbank', address=get_fantom_ironbank, ), + CompoundConfig( + name='scream', + address=get_fantom_scream, + ) ], Network.Arbitrum: [ CompoundConfig( @@ -63,19 +74,19 @@ class CompoundConfig: @dataclass class CompoundMarket: - token: str + token: Address unitroller: ContractContainer @cached_property - def name(self): + def name(self) -> str: return self.ctoken.symbol() @cached_property - def ctoken(self): + def ctoken(self) -> Contract: return contract(self.token) @cached_property - def underlying(self): + def underlying(self) -> Contract: # ceth, creth -> weth if self.token in ['0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5', '0xD06527D5e56A3495252A528C4987003b712860eE']: return contract('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2') @@ -83,14 +94,14 @@ def underlying(self): return contract(self.ctoken.underlying()) @cached_property - def cdecimals(self): + def cdecimals(self) -> int: return self.ctoken.decimals() @cached_property - def under_decimals(self): + def under_decimals(self) -> int: return self.underlying.decimals() - def __eq__(self, other): + def __eq__(self, other: Any) -> bool: if isinstance(other, str): return self.token == other elif isinstance(other, CompoundMarket): @@ -98,13 +109,13 @@ def __eq__(self, other): raise TypeError('can only compare to [str, CompoundMarket]') - def get_exchange_rate(self, block=None): + def get_exchange_rate(self, block: Optional[Block] = None) -> float: exchange_rate = ( self.ctoken.exchangeRateCurrent.call(block_identifier=block) / 1e18 ) return exchange_rate * 10 ** (self.cdecimals - self.under_decimals) - def get_underlying_price(self, block=None): + def get_underlying_price(self, block: Optional[Block] = None) -> float: # query the oracle in case it was changed oracle = contract(self.unitroller.oracle(block_identifier=block)) price = oracle.getUnderlyingPrice( @@ -114,24 +125,24 @@ def get_underlying_price(self, block=None): class Compound: - def __init__(self, name, unitroller, oracle_base): + def __init__(self, name: str, unitroller: Address, oracle_base: Address) -> None: self.name = name self.unitroller = contract(unitroller) if isinstance(unitroller, str) else unitroller() self.oracle_base = oracle_base self.markets # load markets on init - def __repr__(self): + def __repr__(self) -> str: return f'' @property @ttl_cache(ttl=3600) - def markets(self): + def markets(self) -> List[CompoundMarket]: all_markets = self.unitroller.getAllMarkets() markets = [CompoundMarket(token, self.unitroller) for token in all_markets] logger.info(f'loaded {len(markets)} {self.name} markets') return markets - def get_price(self, token, block=None): + def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Union[float,List[Union[float,str]]]: market = next(x for x in self.markets if x == token) exchange_rate = market.get_exchange_rate(block) underlying_price = market.get_underlying_price(block) @@ -142,7 +153,7 @@ def get_price(self, token, block=None): class CompoundMultiplexer(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork('uniswap v2 is not supported on this network') self.compounds = [ @@ -150,10 +161,13 @@ def __init__(self): for conf in addresses[chain.id] ] - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: + if isinstance(token, EthAddress): + # Must convert in order to compare to CompoundMarket. + token = str(token) return any(token in comp.markets for comp in self.compounds) - def get_price(self, token, block=None): + def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> float: comp = next(comp for comp in self.compounds if token in comp.markets) return comp.get_price(token, block) diff --git a/yearn/prices/constants.py b/yearn/prices/constants.py index 406e0efee..3330358f5 100644 --- a/yearn/prices/constants.py +++ b/yearn/prices/constants.py @@ -1,4 +1,5 @@ -from brownie import chain +from brownie import chain, convert + from yearn.networks import Network tokens_by_network = { @@ -7,6 +8,11 @@ 'usdc': '0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48', 'dai': '0x6B175474E89094C44Da98b954EedeAC495271d0F', }, + Network.Gnosis: { + 'weth': '0x6A023CCd1ff6F2045C3309768eAd9E68F978f6e1', + 'usdc': '0xDDAfbb505ad214D7b80b1f830fcCc89B60fb7A83', + 'dai': '0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d', # wxdai address + }, Network.Fantom: { 'weth': '0x21be370D5312f44cB42ce377BC9b8a0cEF1A4C83', 'usdc': '0x04068DA6C83AFCFA0e13ba15A6696662335D5B75', @@ -43,7 +49,13 @@ "0x5BC25f649fc4e26069dDF4cF4010F9f706c23831": "dusd", "0xe2f2a5C287993345a840Db3B0845fbC70f5935a5": "musd", "0x739ca6D71365a08f584c8FC4e1029045Fa8ABC4B": "anydai", - "0xbbc4A8d076F4B1888fec42581B6fc58d242CF2D5": "anymin", + "0xbbc4A8d076F4B1888fec42581B6fc58d242CF2D5": "anymim", + "0x865377367054516e17014CcdED1e7d814EDC9ce4": "dola", + }, + Network.Gnosis: { + "0xDDAfbb505ad214D7b80b1f830fcCc89B60fb7A83": "usdc", + "0x4ECaBa5870353805a9F068101A40E0f32ed605C6": "usdt", + "0xe91d153e0b41518a2ce8dd3d7944fa863463a97d": "wxdai" }, Network.Fantom: { "0x04068DA6C83AFCFA0e13ba15A6696662335D5B75": "usdc", @@ -54,21 +66,28 @@ "0x82f0B8B456c1A451378467398982d4834b6829c1": "mim", "0x049d68029688eAbF473097a2fC38ef61633A3C7A": "fusdt", "0xdc301622e621166BD8E82f2cA0A26c13Ad0BE355": "frax", + "0x95bf7E307BC1ab0BA38ae10fc27084bC36FcD605": "anyusdc", + "0xd652776dE7Ad802be5EC7beBfafdA37600222B48": "anydai", + "0x3129662808bEC728a27Ab6a6b9AFd3cBacA8A43c": "dola", }, Network.Arbitrum: { '0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8': 'usdc', '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1': 'dai', + '0xFEa7a6a0B346362BF88A9e4A88416B77a57D6c2A': 'mim', + '0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9': 'usdt' }, } ib_snapshot_block_by_network = { Network.Mainnet: 14051986, Network.Fantom: 28680044, + Network.Gnosis: 1, # TODO revisit as IB is not deployed in gnosis Network.Arbitrum: 1 } -weth = tokens_by_network[chain.id]['weth'] -usdc = tokens_by_network[chain.id]['usdc'] -dai = tokens_by_network[chain.id]['dai'] -stablecoins = stablecoins_by_network[chain.id] +# We convert to checksum address here to prevent minor annoyances. It's worth it. +weth = convert.to_address(tokens_by_network[chain.id]['weth']) +usdc = convert.to_address(tokens_by_network[chain.id]['usdc']) +dai = convert.to_address(tokens_by_network[chain.id]['dai']) +stablecoins = {convert.to_address(coin): symbol for coin, symbol in stablecoins_by_network[chain.id].items()} ib_snapshot_block = ib_snapshot_block_by_network[chain.id] diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index 6500e6964..da8a44f5f 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -17,19 +17,21 @@ import time from collections import defaultdict from enum import IntEnum -from functools import lru_cache +from typing import Dict, List, Optional -from brownie import ZERO_ADDRESS, chain +from brownie import ZERO_ADDRESS, Contract, chain, convert, interface from brownie.convert import to_address +from brownie.convert.datatypes import EthAddress from cachetools.func import lru_cache, ttl_cache + +from yearn.decorators import sentry_catch_all, wait_or_exit_after from yearn.events import create_filter, decode_logs from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network -from yearn.utils import Singleton, contract -from yearn.decorators import sentry_catch_all, wait_or_exit_after - from yearn.prices import magic +from yearn.typing import Address, AddressOrContract, Block +from yearn.utils import Singleton, contract logger = logging.getLogger(__name__) @@ -60,6 +62,11 @@ 'voting_escrow': '0x5f3b5DfEb7B28CDbD7FAba78963EE202a494e2A2', 'gauge_controller': '0x2F50D538606Fa9EDD2B11E2446BEb18C9D5846bB', }, + Network.Gnosis: { + # Curve has not properly initialized the provider. contract(self.address_provider.get_address(5)) returns 0x0. + # CurveRegistry class has extra handling to fetch registry in this case. + 'address_provider': ADDRESS_PROVIDER, + }, Network.Fantom: { 'address_provider': ADDRESS_PROVIDER, }, @@ -82,7 +89,7 @@ class Ids(IntEnum): class CurveRegistry(metaclass=Singleton): @wait_or_exit_after - def __init__(self): + def __init__(self) -> None: if chain.id not in curve_contracts: raise UnsupportedNetwork("curve is not supported on this network") @@ -104,34 +111,44 @@ def __init__(self): self._thread.start() @sentry_catch_all - def watch_events(self): + def watch_events(self) -> None: address_provider_filter = create_filter(str(self.address_provider)) registries = [] registries_filter = None + registry_logs = [] + address_logs = address_provider_filter.get_all_entries() while True: # fetch all registries and factories from address provider - for event in decode_logs(address_provider_filter.get_new_entries()): + for event in decode_logs(address_logs): if event.name == 'NewAddressIdentifier': self.identifiers[Ids(event['id'])].append(event['addr']) if event.name == 'AddressModified': self.identifiers[Ids(event['id'])].append(event['new_address']) - + + # NOTE: Gnosis chain's address provider fails to provide registry via events. + if not self.identifiers[Ids.Main_Registry]: + self.identifiers[Ids.Main_Registry] = self.address_provider.get_registry() + # if registries were updated, recreate the filter _registries = [ self.identifiers[i][-1] for i in [Ids.Main_Registry, Ids.CryptoSwap_Registry] + if self.identifiers[i] ] if _registries != registries: registries = _registries registries_filter = create_filter(registries) + registry_logs = registries_filter.get_all_entries() # fetch pools from the latest registries - for event in decode_logs(registries_filter.get_new_entries()): + for event in decode_logs(registry_logs): if event.name == 'PoolAdded': self.registries[event.address].add(event['pool']) lp_token = contract(event.address).get_lp_token(event['pool']) self.token_to_pool[lp_token] = event['pool'] + elif event.name == 'PoolRemoved': + self.registries[event.address].discard(event['pool']) # load metapool and curve v5 factories self.load_factories() @@ -142,13 +159,18 @@ def watch_events(self): time.sleep(600) - def read_pools(self, registry): - registry = contract(registry) + # read new logs at end of loop + address_logs = address_provider_filter.get_new_entries() + if registries_filter: + registry_logs = registries_filter.get_new_entries() + + def read_pools(self, registry: Address) -> List[EthAddress]: + registry = interface.CurveRegistry(registry) return fetch_multicall( *[[registry, 'pool_list', i] for i in range(registry.pool_count())] ) - def load_factories(self): + def load_factories(self) -> None: # factory events are quite useless, so we use a different method for factory in self.identifiers[Ids.Metapool_Factory]: pool_list = self.read_pools(factory) @@ -169,7 +191,7 @@ def load_factories(self): self.token_to_pool[lp_token] = pool self.factories[factory].add(pool) - def get_factory(self, pool): + def get_factory(self, pool: AddressOrContract) -> EthAddress: """ Get metapool factory that has spawned a pool. """ @@ -177,12 +199,12 @@ def get_factory(self, pool): return next( factory for factory, factory_pools in self.factories.items() - if str(pool) in factory_pools + if convert.to_address(pool) in factory_pools ) except StopIteration: return None - def get_registry(self, pool): + def get_registry(self, pool: AddressOrContract) -> EthAddress: """ Get registry containing a pool. """ @@ -190,16 +212,16 @@ def get_registry(self, pool): return next( registry for registry, pools in self.registries.items() - if str(pool) in pools + if convert.to_address(pool) in pools ) except StopIteration: return None - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: return self.get_pool(token) is not None @lru_cache(maxsize=None) - def get_pool(self, token): + def get_pool(self, token: AddressOrContract) -> EthAddress: """ Get Curve pool (swap) address by LP token address. Supports factory pools. """ @@ -208,7 +230,7 @@ def get_pool(self, token): return self.token_to_pool[token] @lru_cache(maxsize=None) - def get_gauge(self, pool): + def get_gauge(self, pool: AddressOrContract) -> EthAddress: """ Get liquidity gauge address by pool. """ @@ -219,13 +241,13 @@ def get_gauge(self, pool): gauge = contract(factory).get_gauge(pool) if gauge != ZERO_ADDRESS: return gauge - elif registry: - gauges, types = contract(registry).get_gauges(pool) + if registry: + gauges, _ = contract(registry).get_gauges(pool) if gauges[0] != ZERO_ADDRESS: return gauges[0] @lru_cache(maxsize=None) - def get_coins(self, pool): + def get_coins(self, pool: AddressOrContract) -> List[EthAddress]: """ Get coins of pool. """ @@ -244,7 +266,7 @@ def get_coins(self, pool): return [coin for coin in coins if coin not in {None, ZERO_ADDRESS}] @lru_cache(maxsize=None) - def get_underlying_coins(self, pool): + def get_underlying_coins(self, pool: AddressOrContract) -> List[EthAddress]: pool = to_address(pool) factory = self.get_factory(pool) registry = self.get_registry(pool) @@ -275,7 +297,7 @@ def get_underlying_coins(self, pool): return [coin for coin in coins if coin != ZERO_ADDRESS] @lru_cache(maxsize=None) - def get_decimals(self, pool): + def get_decimals(self, pool: AddressOrContract) -> List[int]: pool = to_address(pool) factory = self.get_factory(pool) registry = self.get_registry(pool) @@ -291,7 +313,7 @@ def get_decimals(self, pool): return [dec for dec in decimals if dec != 0] - def get_balances(self, pool, block=None): + def get_balances(self, pool: AddressOrContract, block: Optional[Block] = None) -> Dict[EthAddress,float]: """ Get {token: balance} of liquidity in the pool. """ @@ -305,9 +327,15 @@ def get_balances(self, pool, block=None): source = contract(factory or registry) balances = source.get_balances(pool, block_identifier=block) # fallback for historical queries - except ValueError: + except ValueError as e: + if str(e) not in [ + 'execution reverted', + 'No data was returned - the call likely reverted' + ]: raise + balances = fetch_multicall( - *[[contract(pool), 'balances', i] for i, _ in enumerate(coins)] + *[[contract(pool), 'balances', i] for i, _ in enumerate(coins)], + block=block ) if not any(balances): @@ -317,8 +345,17 @@ def get_balances(self, pool, block=None): coin: balance / 10 ** dec for coin, balance, dec in zip(coins, balances, decimals) } + + def get_virtual_price(self, pool: Address, block: Optional[Block] = None) -> Optional[float]: + pool = contract(pool) + try: + return pool.get_virtual_price(block_identifier=block) / 1e18 + except ValueError as e: + if str(e) == "execution reverted": + return None + raise - def get_tvl(self, pool, block=None): + def get_tvl(self, pool: AddressOrContract, block: Optional[Block] = None) -> float: """ Get total value in Curve pool. """ @@ -331,7 +368,7 @@ def get_tvl(self, pool, block=None): ) @ttl_cache(maxsize=None, ttl=600) - def get_price(self, token, block=None): + def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: token = to_address(token) pool = self.get_pool(token) # crypto pools can have different tokens, use slow method @@ -351,11 +388,13 @@ def get_price(self, token, block=None): coin = (set(coins) & BASIC_TOKENS).pop() except KeyError: coin = coins[0] + + virtual_price = self.get_virtual_price(pool, block) + + if virtual_price: + return virtual_price * magic.get_price(coin, block) - virtual_price = contract(pool).get_virtual_price(block_identifier=block) / 1e18 - return virtual_price * magic.get_price(coin, block) - - def calculate_boost(self, gauge, addr, block=None): + def calculate_boost(self, gauge: Contract, addr: Address, block: Optional[Block] = None) -> Dict[str,float]: results = fetch_multicall( [gauge, "balanceOf", addr], [gauge, "totalSupply"], @@ -405,7 +444,7 @@ def calculate_boost(self, gauge, addr, block=None): "min vecrv": min_vecrv, } - def calculate_apy(self, gauge, lp_token, block=None): + def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, block: Optional[Block] = None) -> Dict[str,float]: crv_price = magic.get_price(self.crv) pool = contract(self.get_pool(lp_token)) results = fetch_multicall( diff --git a/yearn/prices/fixed_forex.py b/yearn/prices/fixed_forex.py index 71dcb8b7f..ecaf62a10 100644 --- a/yearn/prices/fixed_forex.py +++ b/yearn/prices/fixed_forex.py @@ -1,10 +1,14 @@ -from brownie import chain +import logging +from typing import List, Optional + +from brownie import Contract, chain +from brownie.convert.datatypes import EthAddress from cachetools.func import ttl_cache from yearn.exceptions import UnsupportedNetwork from yearn.networks import Network +from yearn.typing import AddressOrContract, Block from yearn.utils import Singleton, contract, contract_creation_block -import logging logger = logging.getLogger(__name__) @@ -14,20 +18,20 @@ class FixedForex(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork("fixed forex is not supported on this network") - self.registry = contract(addresses[chain.id]) + self.registry: Contract = contract(addresses[chain.id]) self.registry_deploy_block = contract_creation_block(addresses[chain.id]) - self.markets = self.registry.forex() + self.markets: List[EthAddress] = self.registry.forex() logger.info(f'loaded {len(self.markets)} fixed forex markets') - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: return token in self.markets @ttl_cache(maxsize=None, ttl=600) - def get_price(self, token, block=None): + def get_price(self, token: AddressOrContract, block: Optional[Block]=None) -> float: if block is None or block >= self.registry_deploy_block: return self.registry.price(token, block_identifier=block) / 1e18 else: diff --git a/yearn/prices/generic_amm.py b/yearn/prices/generic_amm.py new file mode 100644 index 000000000..c626dc1c0 --- /dev/null +++ b/yearn/prices/generic_amm.py @@ -0,0 +1,46 @@ + +from functools import lru_cache +from typing import List, Optional + +from brownie.convert.datatypes import EthAddress + +from yearn.multicall2 import fetch_multicall +from yearn.prices import magic +from yearn.typing import Address, Block +from yearn.utils import contract + + +class GenericAmm: + def __contains__(self, lp_token_address: Address) -> bool: + return self.is_generic_amm(lp_token_address) + + @lru_cache(maxsize=None) + def is_generic_amm(self, lp_token_address: Address) -> bool: + try: + token_contract = contract(lp_token_address) + return all(hasattr(token_contract, attr) for attr in ['getReserves','token0','token1']) + except Exception as e: + if 'has not been verified' in str(e): + return False + raise + + def get_price(self, lp_token_address: Address, block: Optional[Block] = None) -> float: + lp_token_contract = contract(lp_token_address) + total_supply, decimals = fetch_multicall(*[[lp_token_contract, attr] for attr in ['totalSupply','decimals']], block=block) + total_supply_readable = total_supply / 10 ** decimals + return self.get_tvl(lp_token_address, block) / total_supply_readable + + @lru_cache(maxsize=None) + def get_tokens(self, lp_token_address: Address) -> List[EthAddress]: + lp_token_contract = contract(lp_token_address) + return fetch_multicall(*[[lp_token_contract,attr] for attr in ['token0', 'token1']]) + + def get_tvl(self, lp_token_address: Address, block: Optional[Block] = None) -> float: + lp_token_contract = contract(lp_token_address) + tokens = self.get_tokens(lp_token_address) + reserves = lp_token_contract.getReserves(block_identifier=block) + reserves = [reserves[i] / 10 ** contract(token).decimals() for i, token in enumerate(tokens)] + return sum(reserve * magic.get_price(token) for token, reserve in zip(tokens, reserves)) + + +generic_amm = GenericAmm() diff --git a/yearn/prices/incidents.py b/yearn/prices/incidents.py new file mode 100644 index 000000000..5c26c26ee --- /dev/null +++ b/yearn/prices/incidents.py @@ -0,0 +1,49 @@ +from collections import defaultdict + +from brownie import chain + +from yearn.networks import Network + +INCIDENTS = defaultdict(list) + +INCIDENTS.update({ + Network.Mainnet: { + # yUSDC getPricePerFullShare reverts from block 10532764 to block 10532775 because all liquidity was removed for testing + "0x597aD1e0c13Bfe8025993D9e79C69E1c0233522e": [{"start":10532764,"end":10532775,"result":1}], + "0x629c759D1E83eFbF63d84eb3868B564d9521C129": [{"start":11221202,"end":11238201,"result":1.037773031500707}], + "0xcC7E70A958917cCe67B4B87a8C30E6297451aE98": [{"start":11512085,"end":11519723,"result":1.0086984562068226}], + # GUSD vault state was broken due to an incident + # https://github.com/yearn/yearn-security/blob/master/disclosures/2021-01-17.md + "0xec0d8D3ED5477106c6D4ea27D90a60e594693C90": [{"start":11603873,"end":11645877,"result":0}], + "0x5533ed0a3b83F70c3c4a1f69Ef5546D3D4713E44": [{"start":11865718,"end":11884721,"result":1.0345005219440915}], + # yvcrvAAVE vault state was broken due to an incident + # https://github.com/yearn/yearn-security/blob/master/disclosures/2021-05-13.md + "0x03403154afc09Ce8e44C3B185C82C6aD5f86b9ab": [{"start":12430455,"end":12430661,"result":1.091553}], + # yvust3CRV v1 + "0xF6C9E9AF314982A4b38366f4AbfAa00595C5A6fC": [ + {"start":11833643,"end":11833971,"result":1.0094921430595167}, + {"start":11893317,"end":12020551,"result":1.0107300938482453}, + {"start":12028696,"end":12194529,"result":1.0125968580471483}, + ], + + # for these, price cannot be fetched from chain because totalSupply == 0 + # on block of last withdrawal we return price at block - 1 + # after that block, returns 0 + + # yvhusd3CRV v1 + "0x39546945695DCb1c037C836925B355262f551f55": [ + {"start":12074825,"end":12074825,"result":1.0110339337578227}, + {"start":12074826,"end":chain.height,"result":0}, + ], + # yvobtccrv v1 + "0x7F83935EcFe4729c4Ea592Ab2bC1A32588409797": [ + {"start":12582511,"end":12582511,"result":37611.70819906929}, + {"start":12582512,"end":chain.height,"result":0}, + ], + # yvpbtccrv v1 + "0x123964EbE096A920dae00Fb795FFBfA0c9Ff4675": [ + {"start":12868929,"end":12868929,"result":1456401056701488300032}, + {"start":12868930,"end":chain.height,"result":0}, + ], + }, +}.get(chain.id, {})) diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index 782dae49a..04c083b13 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -1,33 +1,42 @@ import logging +from typing import Optional from brownie import chain +from brownie.convert.datatypes import EthAddress from cachetools.func import ttl_cache + from yearn.exceptions import PriceError from yearn.networks import Network +from yearn.prices.balancer import balancer as bal +from yearn.prices import constants, curve from yearn.prices.aave import aave from yearn.prices.band import band from yearn.prices.chainlink import chainlink from yearn.prices.compound import compound -import yearn.prices.balancer as bal from yearn.prices.fixed_forex import fixed_forex +from yearn.prices.generic_amm import generic_amm +from yearn.prices.incidents import INCIDENTS from yearn.prices.synthetix import synthetix -from yearn.prices.uniswap.v1 import uniswap_v1 +from yearn.prices.uniswap.uniswap import uniswaps from yearn.prices.uniswap.v2 import uniswap_v2 -from yearn.prices.uniswap.v3 import uniswap_v3 -from yearn.prices import curve from yearn.prices.yearn import yearn_lens -from yearn.utils import contract - -from yearn.prices import constants +from yearn.special import Backscratcher +from yearn.typing import Address, AddressOrContract, AddressString, Block +from yearn.utils import contract, contract_creation_block logger = logging.getLogger(__name__) -@ttl_cache(10000) -def get_price(token, block=None): +def get_price( + token: AddressOrContract, + block: Optional[Block] = None, + return_price_during_vault_downtime: bool = False + ) -> float: + token = unwrap_token(token) - return find_price(token, block) + block = chain.height if block is None else block + return find_price(token, block, return_price_during_vault_downtime=return_price_during_vault_downtime) -def unwrap_token(token): +def unwrap_token(token: AddressOrContract) -> AddressString: token = str(token) logger.debug("unwrapping %s", token) @@ -40,16 +49,35 @@ def unwrap_token(token): return "0x0cec1a9154ff802e7934fc916ed7ca50bde6844e" # PPOOL -> POOL if chain.id in [ Network.Mainnet, Network.Fantom ]: - if aave and token in aave: - token = aave.atoken_underlying(token) - logger.debug("aave -> %s", token) + if aave: + asset = contract(token) + # wrapped aDAI -> aDAI + if "ATOKEN" in asset.__dict__: + token = asset.ATOKEN() - return token + if token in aave: + token = aave.atoken_underlying(token) + logger.debug("aave -> %s", token) + return token -def find_price(token, block): +@ttl_cache(10000) +def find_price( + token: Address, + block: Block, + return_price_during_vault_downtime: bool = False + ) -> float: + assert block is not None, "You must pass a valid block number as this function is cached." price = None if token in constants.stablecoins: + if chainlink and token in chainlink and block >= contract_creation_block(chainlink.get_feed(token).address): + price = chainlink.get_price(token, block=block) + logger.debug("stablecoin chainlink -> %s", price) + # If we can't get price from chainlink but `block` is after feed + # deploy block,feed is probably dead and coin is possibly dead. + if price is not None: + return price + # TODO Code better handling for stablecoin pricing logger.debug("stablecoin -> %s", 1) return 1 @@ -57,9 +85,10 @@ def find_price(token, block): price = uniswap_v2.lp_price(token, block=block) logger.debug("uniswap pool -> %s", price) - elif bal.balancer and bal.balancer.is_balancer_pool(token): - price = bal.balancer.get_price(token, block=block) - logger.debug("balancer pool -> %s", price) + elif bal.selector.get_balancer_for_pool(token): + bal_for_pool = bal.selector.get_balancer_for_pool(token) + price = bal_for_pool.get_price(token, block=block) + logger.debug("balancer %s pool -> %s", bal_for_pool.get_version(), price) elif yearn_lens and yearn_lens.is_yearn_vault(token): price = yearn_lens.get_price(token, block=block) @@ -75,20 +104,22 @@ def find_price(token, block): elif chain.id == Network.Mainnet: # no liquid market for yveCRV-DAO -> return CRV token price - if token == '0xc5bDdf9843308380375a611c18B50Fb9341f502A' and block and block < 11786563: + if token == Backscratcher().vault.address and block < 11786563: if curve.curve and curve.curve.crv: return get_price(curve.curve.crv, block=block) + # no liquidity for curve pool (yvecrv-f) -> return 0 + elif token == "0x7E46fd8a30869aa9ed55af031067Df666EfE87da" and block < 14987514: + return 0 markets = [ chainlink, curve.curve, compound, fixed_forex, + generic_amm, synthetix, band, - uniswap_v2, - uniswap_v3, - uniswap_v1 + uniswaps ] for market in markets: if price: @@ -109,20 +140,25 @@ def find_price(token, block): logger.debug("peel %s %s", price, underlying) return price * get_price(underlying, block=block) + if price is None and return_price_during_vault_downtime: + for incident in INCIDENTS[token]: + if incident['start'] <= block <= incident['end']: + return incident['result'] + if price is None: - logger.error(f"failed to get price for {describe_err(token, block)}'") - raise PriceError(f'could not fetch price for {describe_err(token, block)}') + logger.error(f"failed to get price for {_describe_err(token, block)}") + raise PriceError(f'could not fetch price for {_describe_err(token, block)}') return price -def describe_err(token, block) -> str: +def _describe_err(token: Address, block: Optional[Block]) -> str: ''' Assembles a string used to provide as much useful information as possible in PriceError messages ''' try: symbol = contract(token).symbol() - except: + except AttributeError: symbol = None if block is None: @@ -131,7 +167,7 @@ def describe_err(token, block) -> str: return f"malformed token {token} on {Network(chain.id).name}" - if not symbol: + if symbol: return f"{symbol} {token} on {Network(chain.id).name} at {block}" return f"malformed token {token} on {Network(chain.id).name} at {block}" diff --git a/yearn/prices/synthetix.py b/yearn/prices/synthetix.py index 2c2d82528..690361c9a 100644 --- a/yearn/prices/synthetix.py +++ b/yearn/prices/synthetix.py @@ -1,12 +1,15 @@ import logging +from typing import List, Optional from brownie import chain +from brownie.convert.datatypes import EthAddress, HexString from cachetools.func import lru_cache, ttl_cache from eth_abi import encode_single from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network +from yearn.typing import Address, AddressOrContract, Block from yearn.utils import Singleton, contract logger = logging.getLogger(__name__) @@ -17,7 +20,7 @@ class Synthetix(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork("synthetix is not supported on this network") @@ -25,7 +28,7 @@ def __init__(self): logger.info(f'loaded {len(self.synths)} synths') @lru_cache(maxsize=None) - def get_address(self, name): + def get_address(self, name: str) -> EthAddress: """ Get contract from Synthetix registry. See also https://docs.synthetix.io/addresses/ @@ -35,7 +38,7 @@ def get_address(self, name): proxy = contract(address) return contract(proxy.target()) if hasattr(proxy, 'target') else proxy - def load_synths(self): + def load_synths(self) -> List[EthAddress]: """ Get target addresses of all synths. """ @@ -48,7 +51,7 @@ def load_synths(self): ) @lru_cache(maxsize=None) - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: """ Check if a token is a synth. """ @@ -61,12 +64,12 @@ def __contains__(self, token): return False @lru_cache(maxsize=None) - def get_currency_key(self, token): + def get_currency_key(self, token: Address) -> HexString: target = contract(token).target() return contract(target).currencyKey() @ttl_cache(maxsize=None, ttl=600) - def get_price(self, token, block=None): + def get_price(self, token: Address, block: Optional[Block] = None) -> Optional[float]: """ Get a price of a synth in dollars. """ diff --git a/yearn/prices/uniswap/uniswap.py b/yearn/prices/uniswap/uniswap.py new file mode 100644 index 000000000..85e2eb5d4 --- /dev/null +++ b/yearn/prices/uniswap/uniswap.py @@ -0,0 +1,69 @@ + +from typing import Any, Dict, Optional, Union + +from brownie import chain, convert +from yearn.constants import WRAPPED_GAS_COIN +from yearn.networks import Network +from yearn.prices import constants +from yearn.prices.chainlink import chainlink +from yearn.prices.uniswap.v1 import UniswapV1, uniswap_v1 +from yearn.prices.uniswap.v2 import UniswapV2Multiplexer, uniswap_v2 +from yearn.prices.uniswap.v3 import UniswapV3, uniswap_v3 +from yearn.typing import Address, AddressOrContract, Block +from yearn.utils import contract, contract_creation_block + +Uniswap = Union[UniswapV1,UniswapV2Multiplexer,UniswapV3] + +UNISWAPS: Dict[str,Optional[Uniswap]] = { + 'v1': uniswap_v1, + 'v2': uniswap_v2, + 'v3': uniswap_v3 +} + +class UniswapVersionMultiplexer: + def __init__(self) -> None: + self.uniswaps: Dict[str,Uniswap] = {version: uniswap for version, uniswap in UNISWAPS.items() if uniswap is not None} + + def __contains__(self, token: Any) -> bool: + return len(self.uniswaps) > 0 + + def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: + token = convert.to_address(token) + + # NOTE Following our usual logic with WETH is a big no-no. Too many calls. + if token in [constants.weth, WRAPPED_GAS_COIN] and block <= contract_creation_block(chainlink.get_feed(token)): + return self._early_exit_for_gas_coin(token, block=block) + + deepest_uniswap = self.deepest_uniswap(token, block) + if deepest_uniswap: + return deepest_uniswap.get_price(token, block=block) + return None + + def deepest_uniswap(self, token_in: Address, block: Optional[Block] = None) -> Optional[Uniswap]: + deepest_uniswap = None + deepest_uniswap_balance = 0 + for uniswap in self.uniswaps.values(): + deepest_pool_balance = uniswap.deepest_pool_balance(token_in, block) + if deepest_pool_balance and deepest_pool_balance > deepest_uniswap_balance: + deepest_uniswap = uniswap + deepest_uniswap_balance = deepest_pool_balance + return deepest_uniswap + + def _early_exit_for_gas_coin(self, token: Address, block: Optional[Block] = None) -> Optional[float]: + ''' We use this to bypass the liquidity checker for ETH prior to deployment of the chainlink feed. ''' + amount_in = 1e18 + path = [token, constants.usdc] + best_market = { + Network.Mainnet: "uniswap", + Network.Fantom: "spookyswap", + Network.Gnosis: "sushiswap", + }[chain.id] + for uni in self.uniswaps['v2'].uniswaps: + if uni.name != best_market: + continue + quote = uni.router.getAmountsOut(amount_in, path, block_identifier=block)[-1] + quote /= 10 ** contract(constants.usdc).decimals() + fees = 0.997 ** (len(path) - 1) + return quote / fees + +uniswaps = UniswapVersionMultiplexer() diff --git a/yearn/prices/uniswap/v1.py b/yearn/prices/uniswap/v1.py index 11ede2c90..f349bdf1e 100644 --- a/yearn/prices/uniswap/v1.py +++ b/yearn/prices/uniswap/v1.py @@ -1,37 +1,62 @@ -from brownie import chain, interface +from typing import Any, Optional + +from brownie import ZERO_ADDRESS, Contract, chain, interface +from brownie.convert.datatypes import EthAddress from brownie.exceptions import ContractNotFound from cachetools.func import ttl_cache +from yearn.cache import memory from yearn.exceptions import UnsupportedNetwork from yearn.networks import Network from yearn.prices.constants import usdc +from yearn.typing import Address, AddressOrContract, Block from yearn.utils import Singleton, contract addresses = { Network.Mainnet: '0xc0a47dFe034B400B47bDaD5FecDa2621de6c4d95', } +@memory.cache() +def _get_exchange(factory: Contract, token: AddressOrContract) -> EthAddress: + """ + I extracted this fn for caching purposes. + On-disk caching should be fine since no new pools should be added to uni v1 + which means a response equal to `ZERO_ADDRESS` implies there will never be a uni v1 pool for `token`. + """ + return factory.getExchange(token) + + class UniswapV1(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork('uniswap v1 is not supported on this network') self.factory = contract(addresses[chain.id]) - def __contains__(self, asset): + def __contains__(self, asset: Any) -> bool: return chain.id in addresses @ttl_cache(ttl=600) - def get_price(self, asset, block=None): + def get_price(self, asset: Address, block: Optional[Block] = None) -> Optional[float]: try: asset = contract(asset) - exchange = interface.UniswapV1Exchange(self.factory.getExchange(asset)) + exchange = interface.UniswapV1Exchange(self.get_exchange(asset)) eth_bought = exchange.getTokenToEthInputPrice(10 ** asset.decimals(), block_identifier=block) - exchange = interface.UniswapV1Exchange(self.factory.getExchange(usdc)) + exchange = interface.UniswapV1Exchange(self.get_exchange(usdc)) usdc_bought = exchange.getEthToTokenInputPrice(eth_bought, block_identifier=block) / 1e6 fees = 0.997 ** 2 return usdc_bought / fees except (ContractNotFound, ValueError) as e: - pass + return None + + def get_exchange(self, token: AddressOrContract) -> EthAddress: + return _get_exchange(self.factory, token) + + def deepest_pool_balance(self, token_in: Address, block: Optional[Block] = None) -> int: + exchange = self.get_exchange(token_in) + if exchange == ZERO_ADDRESS: + return None + reserves = contract(token_in).balanceOf(exchange, block_identifier=block) + return reserves uniswap_v1 = None diff --git a/yearn/prices/uniswap/v2.py b/yearn/prices/uniswap/v2.py index 07ad32a98..bd3ca43a9 100644 --- a/yearn/prices/uniswap/v2.py +++ b/yearn/prices/uniswap/v2.py @@ -1,13 +1,24 @@ -from brownie import Contract, chain +import logging +from collections import defaultdict +from functools import cached_property +from typing import Any, Dict, List, Optional + +from brownie import Contract, chain, convert, interface +from brownie.convert.datatypes import EthAddress +from brownie.exceptions import EventLookupError, VirtualMachineError from cachetools.func import lru_cache, ttl_cache +from yearn.events import decode_logs, get_logs_asap from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network -from yearn.prices.constants import usdc, weth +from yearn.prices.constants import stablecoins, usdc, weth +from yearn.typing import Address, AddressOrContract, Block from yearn.utils import Singleton, contract +logger = logging.getLogger(__name__) + # NOTE insertion order defines priority, higher priority get queried first. -addresses = { +addresses: List[Dict[str,str]] = { Network.Mainnet: [ { 'name': 'sushiswap', @@ -20,6 +31,13 @@ 'router': '0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D', }, ], + Network.Gnosis: [ + { + 'name': 'sushiswap', + 'factory': '0xc35DADB65012eC5796536bD9864eD8773aBc74C4', + 'router': '0x1b02dA8Cb0d097eB8D57A175b88c7D8b47997506', + }, + ], Network.Fantom: [ { 'name': 'spookyswap', @@ -31,46 +49,80 @@ 'factory': '0xEF45d134b73241eDa7703fa787148D9C9F4950b0', 'router': '0x16327E3FbDaCA3bcF7E38F5Af2599D2DDc33aE52', }, + { + 'name': 'sushiswap', + 'factory': '0xc35DADB65012eC5796536bD9864eD8773aBc74C4', + "router": '0x1b02dA8Cb0d097eB8D57A175b88c7D8b47997506', + } ], + Network.Arbitrum: [ + { + 'name': 'sushiswap', + 'factory': '0xc35DADB65012eC5796536bD9864eD8773aBc74C4', + 'router': '0x1b02dA8Cb0d097eB8D57A175b88c7D8b47997506', + }, + { + 'name': 'fraxswap', + 'factory': '0x5Ca135cB8527d76e932f34B5145575F9d8cbE08E', + 'router': '0xc2544A32872A91F4A553b404C6950e89De901fdb', + } + ] } -class UniswapV2: - name: str - factory: str - router: str +class CantFindSwapPath(Exception): + pass - def __init__(self, name, factory, router): + +class UniswapV2: + def __init__(self, name: str, factory: Address, router: Address) -> None: self.name = name - self.factory = contract(factory) - self.router = contract(router) + self.factory: Contract = contract(factory) + self.router: Contract = contract(router) - def __repr__(self): + # Used for internals + self._depth_cache: Dict[Address,Dict[Block,int]] = defaultdict(dict) + + def __repr__(self) -> str: return f'' @ttl_cache(ttl=600) - def get_price(self, token_in, token_out=usdc, block=None): + def get_price(self, token_in: AddressOrContract, token_out: AddressOrContract = usdc, block: Optional[Block] = None) -> Optional[float]: """ Calculate a price based on Uniswap Router quote for selling one `token_in`. Always uses intermediate WETH pair. """ + token_in = convert.to_address(token_in) + token_out = convert.to_address(token_out) tokens = [contract(str(token)) for token in [token_in, token_out]] amount_in = 10 ** tokens[0].decimals() - path = ( - [token_in, token_out] - if weth in (token_in, token_out) - else [token_in, weth, token_out] - ) + path = None + if token_out in stablecoins: + try: + path = self.get_path_to_stables(token_in) + except CantFindSwapPath: + pass + + if not path: + path = ( + [token_in, token_out] + if weth in (token_in, token_out) + else [token_in, weth, token_out] + ) + fees = 0.997 ** (len(path) - 1) try: quote = self.router.getAmountsOut(amount_in, path, block_identifier=block) amount_out = quote[-1] / 10 ** tokens[1].decimals() return amount_out / fees - except ValueError: + except VirtualMachineError as e: + okay_errs = ['INSUFFICIENT_INPUT_AMOUNT'] + if not any([err in str(e) for err in okay_errs]): + raise return None @ttl_cache(ttl=600) - def lp_price(self, address, block=None): + def lp_price(self, address: Address, block: Optional[Block] = None) -> Optional[float]: """Get Uniswap/Sushiswap LP token price.""" pair = contract(address) token0, token1, supply, reserves = fetch_multicall( @@ -80,7 +132,7 @@ def lp_price(self, address, block=None): [pair, "getReserves"], block=block, ) - tokens = [Contract(token) for token in [token0, token1]] + tokens = [contract(token) for token in [token0, token1]] scales = [10 ** token.decimals() for token in tokens] prices = [self.get_price(token, block=block) for token in tokens] supply = supply / 1e18 @@ -90,10 +142,142 @@ def lp_price(self, address, block=None): res / scale * price for res, scale, price in zip(reserves, scales, prices) ] return sum(balances) / supply + + @cached_property + def pools(self) -> Dict[Address,Dict[Address,Address]]: + ''' + Returns a dictionary with all pools + {pool:{'token0':token0,'token1':token1}} + ''' + logger.info(f'Fetching pools for {self.name} on {Network.label()}. If this is your first time running yearn-exporter, this can take a while. Please wait patiently...') + PairCreated = ['0x0d3648bd0f6ba80134a33ba9275ac585d9d315f0ad8355cddefde31afa28d0e9'] + events = decode_logs(get_logs_asap(self.factory.address, PairCreated)) + try: + pairs = { + event['']: { + convert.to_address(event['pair']): { + 'token0':convert.to_address(event['token0']), + 'token1':convert.to_address(event['token1']), + } + } + for event in events + } + pools = {pool: tokens for i, pair in pairs.items() for pool, tokens in pair.items()} + except EventLookupError: + pairs, pools = {}, {} + + all_pairs_len = self.factory.allPairsLength() + if len(pairs) < all_pairs_len: + logger.debug("Oh no! looks like your node can't look back that far. Checking for the missing pools...") + poolids_your_node_couldnt_get = [i for i in range(all_pairs_len) if i not in pairs] + logger.debug(f'missing poolids: {poolids_your_node_couldnt_get}') + pools_your_node_couldnt_get = fetch_multicall(*[[self.factory,'allPairs',i] for i in poolids_your_node_couldnt_get]) + token0s = fetch_multicall(*[[contract(pool), 'token0'] for pool in pools_your_node_couldnt_get]) + token1s = fetch_multicall(*[[contract(pool), 'token1'] for pool in pools_your_node_couldnt_get]) + additional_pools = { + convert.to_address(pool): { + 'token0':convert.to_address(token0), + 'token1':convert.to_address(token1), + } + for pool, token0, token1 in zip(pools_your_node_couldnt_get,token0s,token1s) + } + pools.update(additional_pools) + + return pools + + @cached_property + def pool_mapping(self) -> Dict[Address,Dict[Address,Address]]: + ''' + Returns a dictionary with all available combinations of {token_in:{pool:token_out}} + ''' + pool_mapping: Dict[Address,Dict[Address,Address]] = defaultdict(dict) + + for pool, tokens in self.pools.items(): + token0, token1 = tokens.values() + pool_mapping[token0][pool] = token1 + pool_mapping[token1][pool] = token0 + logger.info(f'Loaded {len(self.pools)} pools supporting {len(pool_mapping)} tokens on {self.name}') + return pool_mapping + + def pools_for_token(self, token_address: Address) -> Dict[Address,Address]: + try: + return self.pool_mapping[token_address] + except KeyError: + return {} + + def deepest_pool(self, token_address: AddressOrContract, block: Optional[Block] = None, _ignore_pools: List[Address] = []) -> Optional[EthAddress]: + token_address = convert.to_address(token_address) + if token_address == weth or token_address in stablecoins: + return self.deepest_stable_pool(token_address) + pools = self.pools_for_token(token_address) + reserves = fetch_multicall(*[[contract(pool),'getReserves'] for pool in pools], block=block, require_success=False) + + deepest_pool = None + deepest_pool_balance = 0 + for pool, reserves in zip(pools,reserves): + if reserves is None or pool in _ignore_pools: + continue + if token_address == self.pools[pool]['token0']: + reserve = reserves[0] + elif token_address == self.pools[pool]['token1']: + reserve = reserves[1] + if reserve > deepest_pool_balance: + deepest_pool = pool + deepest_pool_balance = reserve + return deepest_pool + + def deepest_stable_pool(self, token_address: AddressOrContract, block: Optional[Block] = None) -> Optional[EthAddress]: + token_address = convert.to_address(token_address) + pools = {pool: paired_with for pool, paired_with in self.pools_for_token(token_address).items() if paired_with in stablecoins} + reserves = fetch_multicall(*[[contract(pool), 'getReserves'] for pool in pools], block=block, require_success=False) + + deepest_stable_pool = None + deepest_stable_pool_balance = 0 + for pool, reserves in zip(pools, reserves): + if reserves is None: + continue + if token_address == self.pools[pool]['token0']: + reserve = reserves[0] + elif token_address == self.pools[pool]['token1']: + reserve = reserves[1] + if reserve > deepest_stable_pool_balance: + deepest_stable_pool = pool + deepest_stable_pool_balance = reserve + return deepest_stable_pool + + def get_path_to_stables(self, token_address: AddressOrContract, block: Optional[Block] = None, _loop_count: int = 0, _ignore_pools: List[Address] = []) -> List[AddressOrContract]: + if _loop_count > 10: + raise CantFindSwapPath + + token_address = convert.to_address(token_address) + path = [token_address] + deepest_pool = self.deepest_pool(token_address, block, _ignore_pools) + if deepest_pool: + paired_with = self.pool_mapping[token_address][deepest_pool] + deepest_stable_pool = self.deepest_stable_pool(token_address, block) + if deepest_stable_pool and deepest_pool == deepest_stable_pool: + last_step = self.pool_mapping[token_address][deepest_stable_pool] + path.append(last_step) + return path + + if path == [token_address]: + try: path.extend( + self.get_path_to_stables( + paired_with, + block=block, + _loop_count=_loop_count+1, + _ignore_pools=_ignore_pools + [deepest_pool] + ) + ) + except CantFindSwapPath: pass + + if path == [token_address]: raise CantFindSwapPath(f'Unable to find swap path for {token_address} on {Network.label()}') + + return path class UniswapV2Multiplexer(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork('uniswap v2 is not supported on this network') self.uniswaps = [ @@ -101,17 +285,17 @@ def __init__(self): for conf in addresses[chain.id] ] - def __contains__(self, asset): + def __contains__(self, asset: Any) -> bool: return chain.id in addresses - def get_price(self, token, block=None): - for exchange in self.uniswaps: - price = exchange.get_price(token, block=block) - if price: - return price + def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: + deepest_uniswap = self.deepest_uniswap(token, block) + if deepest_uniswap: + return deepest_uniswap.get_price(token, block=block) + return None @lru_cache(maxsize=None) - def is_uniswap_pool(self, address): + def is_uniswap_pool(self, address: Address) -> bool: try: return contract(address).factory() in [x.factory for x in self.uniswaps] except (ValueError, OverflowError, AttributeError): @@ -119,7 +303,7 @@ def is_uniswap_pool(self, address): return False @ttl_cache(ttl=600) - def lp_price(self, token, block=None): + def lp_price(self, token: Address, block: Optional[Block] = None) -> Optional[float]: pair = contract(token) factory = pair.factory() try: @@ -128,6 +312,40 @@ def lp_price(self, token, block=None): return None else: return exchange.lp_price(token, block) + + @lru_cache(maxsize=100) + def deepest_uniswap(self, token_in: AddressOrContract, block: Optional[Block] = None) -> Optional[UniswapV2]: + token_in = convert.to_address(token_in) + pool_to_uniswap = {pool: uniswap for uniswap in self.uniswaps for pool in uniswap.pools_for_token(token_in)} + reserves = fetch_multicall(*[[interface.UniswapPair(pool), 'getReserves'] for pool in pool_to_uniswap], block=block) + + deepest_uniswap = None + deepest_uniswap_balance = 0 + for uniswap, pool, reserves in zip(pool_to_uniswap.values(), pool_to_uniswap.keys(),reserves): + if reserves is None: + continue + if token_in == uniswap.pools[pool]['token0']: + reserve = reserves[0] + elif token_in == uniswap.pools[pool]['token1']: + reserve = reserves[1] + if reserve > deepest_uniswap_balance: + deepest_uniswap = uniswap + deepest_uniswap_balance = reserve + + if deepest_uniswap: + if block is not None: + deepest_uniswap._depth_cache[token_in][block] = deepest_uniswap_balance + return deepest_uniswap + return None + + def deepest_pool_balance(self, token_in: AddressOrContract, block: Optional[Block] = None) -> Optional[Block]: + if block is None: + block = chain.height + token_in = convert.to_address(token_in) + deepest_uniswap = self.deepest_uniswap(token_in, block) + if deepest_uniswap: + return deepest_uniswap._depth_cache[token_in][block] + return None uniswap_v2 = None diff --git a/yearn/prices/uniswap/v3.py b/yearn/prices/uniswap/v3.py index 27ab6fc97..e96848764 100644 --- a/yearn/prices/uniswap/v3.py +++ b/yearn/prices/uniswap/v3.py @@ -1,17 +1,34 @@ +import logging import math +from collections import defaultdict +from functools import cached_property from itertools import cycle +from typing import Any, Dict, List, Optional, Tuple, Union -from brownie import chain +from brownie import Contract, chain, convert from eth_abi.packed import encode_abi_packed +from yearn.events import decode_logs, get_logs_asap from yearn.exceptions import UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network from yearn.prices.constants import usdc, weth +from yearn.typing import Address, Block from yearn.utils import Singleton, contract, contract_creation_block +logger = logging.getLogger(__name__) + +class FeeTier(int): + def __init__(self, v) -> None: + super().__init__() + +Path = List[Union[Address,FeeTier]] + + # https://github.com/Uniswap/uniswap-v3-periphery/blob/main/deploys.md UNISWAP_V3_FACTORY = '0x1F98431c8aD98523631AE4a59f267346ea31F984' UNISWAP_V3_QUOTER = '0xb27308f9F90D607463bb33eA1BeBb41C27CE5AB6' +FEE_DENOMINATOR = 1_000_000 +USDC_SCALE = 1e6 # same addresses on all networks addresses = { @@ -27,46 +44,38 @@ }, } -FEE_DENOMINATOR = 1_000_000 - class UniswapV3(metaclass=Singleton): - def __init__(self): + def __init__(self) -> None: if chain.id not in addresses: raise UnsupportedNetwork('compound is not supported on this network') conf = addresses[chain.id] - self.factory = contract(conf['factory']) - self.quoter = contract(conf['quoter']) - self.fee_tiers = conf['fee_tiers'] + self.factory: Contract = contract(conf['factory']) + self.quoter: Contract = contract(conf['quoter']) + self.fee_tiers = [FeeTier(fee) for fee in conf['fee_tiers']] - def __contains__(self, asset): + def __contains__(self, asset: Any) -> bool: return chain.id in addresses - def encode_path(self, path): + def encode_path(self, path: Path) -> bytes: types = [type for _, type in zip(path, cycle(['address', 'uint24']))] return encode_abi_packed(types, path) - def undo_fees(self, path): + def undo_fees(self, path: Path) -> float: fees = [1 - fee / FEE_DENOMINATOR for fee in path if isinstance(fee, int)] return math.prod(fees) - def get_price(self, token, block=None): + def get_price(self, token: Address, block: Optional[Block] = None) -> Optional[float]: if block and block < contract_creation_block(UNISWAP_V3_QUOTER): return None - if token == usdc: - return 1 - - paths = [] - if token != weth: - paths += [ - [token, fee, weth, self.fee_tiers[0], usdc] for fee in self.fee_tiers - ] - - paths += [[token, fee, usdc] for fee in self.fee_tiers] + paths = self.get_paths(token) - scale = 10 ** contract(token).decimals() + try: + scale = 10 ** contract(token).decimals() + except AttributeError: + return None results = fetch_multicall( *[ @@ -77,11 +86,90 @@ def get_price(self, token, block=None): ) outputs = [ - amount / self.undo_fees(path) / 1e6 + amount / self.undo_fees(path) / USDC_SCALE for amount, path in zip(results, paths) if amount ] return max(outputs) if outputs else None + + @cached_property + def pools(self) -> Dict[Address,Dict[str,Union[str,int]]]: + ''' + Returns a dict {pool:{attr:value}} where attr is one of: 'token0', 'token1', 'fee', 'tick spacing' + ''' + logger.info(f'Fetching pools for uniswap v3 on {Network.label()}. If this is your first time running yearn-exporter, this can take a while. Please wait patiently...') + PoolCreated = ['0x783cca1c0412dd0d695e784568c96da2e9c22ff989357a2e8b1d9b2b4e6b7118'] + events = decode_logs(get_logs_asap(self.factory.address, PoolCreated)) + return { + event['pool']: { + 'token0': event['token0'], + 'token1': event['token1'], + 'fee': event['fee'], + 'tick spacing': event['tickSpacing'] + } + for event in events + } + + @cached_property + def pool_mapping(self) -> Dict[Address,Dict[Address,Address]]: + ''' + Returns a dict {token_in:{pool:token_out}} + ''' + pool_mapping: Dict[str,Dict[str,str]] = defaultdict(dict) + for pool, attributes in self.pools.items(): + token0, token1, fee, tick_spacing = attributes.values() + pool_mapping[token0][pool] = token1 + pool_mapping[token1][pool] = token0 + logger.info(f'Loaded {len(self.pools)} pools supporting {len(pool_mapping)} tokens on uniswap v3') + return pool_mapping + + def deepest_pool_balance(self, token: Address, block: Optional[Block] = None) -> int: + ''' + Returns the depth of the deepest pool for `token`, used to compary liquidity across dexes. + ''' + token_contract = contract(token) + pools = self.pool_mapping[token_contract.address] + reserves = fetch_multicall(*[[token_contract,'balanceOf',pool] for pool in pools], block=block) + + deepest_pool_balance = 0 + for pool, balance in zip(pools, reserves): + if balance > deepest_pool_balance: + deepest_pool_balance = balance + + return deepest_pool_balance + + def get_paths(self, token: Address) -> List[Path]: + token = convert.to_address(token) + paths = [[token, fee, usdc] for fee in self.fee_tiers] + + if token == weth: + return paths + + pools = self.pool_mapping[token] + for pool in pools: + token0, token1, fee, tick_spacing = self.pools[pool].values() + if token == token0 and token1 == weth: + paths += [[token0, fee, token1, tier, usdc] for tier in self.fee_tiers] + elif token == token0: + paths += [[token0, fee, token1, tier0, weth, tier1, usdc] for tier0, tier1 in self.tier_pairs] + elif token == token1 and token0 == weth: + paths += [[token1, fee, token0, tier, usdc] for tier in self.fee_tiers] + elif token == token1: + paths += [[token1, fee, token0, tier0, weth, tier1, usdc] for tier0, tier1 in self.tier_pairs] + + return paths + + @cached_property + def tier_pairs(self) -> List[Tuple[FeeTier,FeeTier]]: + ''' + Returns a list containing all possible pairs of fees for a 2 hop swap. + ''' + return [ + (tier0,tier1) + for tier0 in self.fee_tiers + for tier1 in self.fee_tiers + ] + uniswap_v3 = None diff --git a/yearn/prices/yearn.py b/yearn/prices/yearn.py index 3aee742f3..7ef51d607 100644 --- a/yearn/prices/yearn.py +++ b/yearn/prices/yearn.py @@ -1,10 +1,15 @@ +import logging +from typing import Dict, List, Optional + +from brownie import chain +from brownie.convert.datatypes import EthAddress +from cachetools.func import ttl_cache + +from yearn.exceptions import MulticallError, UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network +from yearn.typing import Address, AddressOrContract, Block, VaultVersion from yearn.utils import Singleton, contract -from brownie import chain -from yearn.exceptions import MulticallError, UnsupportedNetwork -import logging -from cachetools.func import ttl_cache logger = logging.getLogger(__name__) @@ -27,14 +32,17 @@ class YearnLens(metaclass=Singleton): - def __init__(self): - if chain.id not in addresses: + def __init__(self, force_init: bool = False) -> None: + if chain.id not in addresses and not force_init: raise UnsupportedNetwork('yearn is not supported on this network') self.markets @property @ttl_cache(ttl=3600) - def markets(self): + def markets(self) -> Dict[VaultVersion,List[EthAddress]]: + if chain.id not in addresses: + return {} + markets = { name: list(contract(addr).assetsAddresses()) for name, addr in addresses[chain.id].items() @@ -44,11 +52,11 @@ def markets(self): logger.info(f'loaded {log_counts} markets') return markets - def __contains__(self, token): + def __contains__(self, token: AddressOrContract) -> bool: # hard check, works with production vaults return any(token in market for market in self.markets.values()) - def is_yearn_vault(self, token): + def is_yearn_vault(self, token: Address) -> bool: # soft check, works with any contracts using a compatible interface vault = contract(token) return any( @@ -59,7 +67,7 @@ def is_yearn_vault(self, token): ] ) - def get_price(self, token, block=None): + def get_price(self, token: Address, block: Optional[Block] = None) -> Optional[float]: # v2 vaults use pricePerShare scaled to underlying token decimals vault = contract(token) if hasattr(vault, 'pricePerShare'): @@ -91,8 +99,4 @@ def get_price(self, token, block=None): return [share_price / 1e18, underlying] -yearn_lens = None -try: - yearn_lens = YearnLens() -except UnsupportedNetwork: - pass +yearn_lens = YearnLens(force_init=True) diff --git a/yearn/typing.py b/yearn/typing.py new file mode 100644 index 000000000..edd1b839b --- /dev/null +++ b/yearn/typing.py @@ -0,0 +1,16 @@ +from typing import List, Literal, Union + +from brownie import Contract +from brownie.convert.datatypes import EthAddress, HexBytes +from eth_typing import AnyAddress, BlockNumber + +VaultVersion = Literal['v1','v2'] + +AddressString = str +Address = Union[str,HexBytes,AnyAddress,EthAddress] +AddressOrContract = Union[Address,Contract] + +Block = Union[int,BlockNumber] + +Topic = Union[str,HexBytes,None] +Topics = List[Union[Topic,List[Topic]]] \ No newline at end of file diff --git a/yearn/utils.py b/yearn/utils.py index f3a043c16..8cf2dfb20 100644 --- a/yearn/utils.py +++ b/yearn/utils.py @@ -1,17 +1,25 @@ import logging -from functools import lru_cache import threading +import json +from functools import lru_cache +from typing import List -from brownie import Contract, chain, web3, interface +import eth_retry +from brownie import Contract, chain, convert, interface, web3 +from web3 import Web3 +from brownie.network.contract import _resolve_address, _fetch_from_explorer +from brownie.exceptions import CompilerError from yearn.cache import memory -from yearn.exceptions import ArchiveNodeRequired +from yearn.exceptions import ArchiveNodeRequired, NodeNotSynced from yearn.networks import Network +from yearn.typing import Address, AddressOrContract logger = logging.getLogger(__name__) BINARY_SEARCH_BARRIER = { Network.Mainnet: 0, + Network.Gnosis: 15_659_482, # gnosis returns "No state available for block 0x3f9e020290502d1d41f4b5519e7d456f0935dea980ec310935206cac8239117e" Network.Fantom: 4_564_024, # fantom returns "missing trie node" before that Network.Arbitrum: 0, } @@ -24,7 +32,7 @@ } } -def safe_views(abi): +def safe_views(abi: List) -> List[str]: return [ item["name"] for item in abi @@ -41,10 +49,12 @@ def get_block_timestamp(height): An optimized variant of `chain[height].timestamp` """ if chain.id == Network.Mainnet: - header = web3.manager.request_blocking(f"erigon_getHeaderByNumber", [height]) - return int(header.timestamp, 16) - else: - return chain[height].timestamp + try: + header = web3.manager.request_blocking(f"erigon_getHeaderByNumber", [height]) + return int(header.timestamp, 16) + except: + pass + return chain[height].timestamp @memory.cache() @@ -76,17 +86,23 @@ def get_code(address, block=None): @memory.cache() -def contract_creation_block(address) -> int: +def contract_creation_block(address: AddressOrContract) -> int: """ Find contract creation block using binary search. NOTE Requires access to historical state. Doesn't account for CREATE2 or SELFDESTRUCT. """ logger.info("contract creation block %s", address) + address = convert.to_address(address) barrier = BINARY_SEARCH_BARRIER[chain.id] lo = barrier hi = end = chain.height + if hi == 0: + raise NodeNotSynced(f''' + `chain.height` returns 0 on your node, which means it is not fully synced. + You can only use contract_creation_block on a fully synced node.''') + while hi - lo > 1: mid = lo + (hi - lo) // 2 try: @@ -95,6 +111,11 @@ def contract_creation_block(address) -> int: logger.error(exc) # with no access to historical state, we'll have to scan logs from start return 0 + except ValueError as exc: + # ValueError occurs in gnosis when there is no state for a block + # with no access to historical state, we'll have to scan logs from start + logger.error(exc) + return 0 if code: hi = mid else: @@ -125,22 +146,85 @@ def __call__(self, *args, **kwargs): _contract_lock = threading.Lock() _contract = lru_cache(maxsize=None)(Contract) -def contract(address): +@eth_retry.auto_retry +def contract(address: Address) -> Contract: with _contract_lock: + address = web3.toChecksumAddress(address) + if chain.id in PREFER_INTERFACE: if address in PREFER_INTERFACE[chain.id]: _interface = PREFER_INTERFACE[chain.id][address] - return _interface(address) - + i = _interface(address) + return _squeeze(i) + + failed_attempts = 0 + while True: + try: + c = _contract(address) + return _squeeze(c) + except (AssertionError, CompilerError) as e: + if failed_attempts == 10: + raise + logger.warning(e) + Contract.remove_deployment(address) + failed_attempts += 1 + + +@eth_retry.auto_retry +def _resolve_proxy(address): + data = _fetch_from_explorer(address, "getsourcecode", False) + name = data["result"][0]["ContractName"] + abi = json.loads(data["result"][0]["ABI"]) + as_proxy_for = None + + # always check for an EIP1967 proxy - https://eips.ethereum.org/EIPS/eip-1967 + implementation_eip1967 = web3.eth.get_storage_at( + address, int(web3.keccak(text="eip1967.proxy.implementation").hex(), 16) - 1 + ) + # always check for an EIP1822 proxy - https://eips.ethereum.org/EIPS/eip-1822 + implementation_eip1822 = web3.eth.get_storage_at(address, web3.keccak(text="PROXIABLE")) + if len(implementation_eip1967) > 0 and int(implementation_eip1967.hex(), 16): + as_proxy_for = _resolve_address(implementation_eip1967[-20:]) + elif len(implementation_eip1822) > 0 and int(implementation_eip1822.hex(), 16): + as_proxy_for = _resolve_address(implementation_eip1822[-20:]) + elif data["result"][0].get("Implementation"): + # for other proxy patterns, we only check if etherscan indicates + # the contract is a proxy. otherwise we could have a false positive + # if there is an `implementation` method on a regular contract. + try: + # first try to call `implementation` per EIP897 + # https://eips.ethereum.org/EIPS/eip-897 + c = Contract.from_abi(name, address, abi) + as_proxy_for = c.implementation.call() + except Exception: + # if that fails, fall back to the address provided by etherscan + as_proxy_for = _resolve_address(data["result"][0]["Implementation"]) + + if as_proxy_for: + data = _fetch_from_explorer(as_proxy_for, "getsourcecode", False) + name = data["result"][0]["ContractName"] + abi = json.loads(data["result"][0]["ABI"]) + return Contract.from_abi(name, as_proxy_for, abi) + else: return _contract(address) + +@lru_cache(maxsize=None) def is_contract(address: str) -> bool: '''checks to see if the input address is a contract''' - return web3.eth.get_code(address) != '0x' + return web3.eth.get_code(address) not in ['0x',b''] def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): - yield lst[i:i + n] \ No newline at end of file + yield lst[i:i + n] + + +def _squeeze(it): + """ Reduce the contract size in RAM significantly. """ + for k in ["ast", "bytecode", "coverageMap", "deployedBytecode", "deployedSourceMap", "natspec", "opcodes", "pcMap"]: + if it._build and k in it._build.keys(): + it._build[k] = {} + return it \ No newline at end of file From b74ccf57a43c97d74324cfb37d07026998ade49c Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 12 Aug 2022 13:02:04 -0400 Subject: [PATCH 44/86] fix: keepcrv display --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 3c719c8c0..638a2ef8f 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -602,7 +602,7 @@ def format_dev_telegram(r, t): df["Debt Added"] = "{:,.2f}".format(r.debt_added) + " | " + "${:,.2f}".format(r.debt_added * r.want_price_at_block) df["Total Debt"] = "{:,.2f}".format(r.total_debt) + " | " + "${:,.2f}".format(r.total_debt * r.want_price_at_block) df["Debt Ratio"] = r.debt_ratio - if r.keep_crv > 0: + if chain.id == 1 and r.keep_crv > 0: df["CRV Locked"] = "{:,.2f}".format(r.keep_crv) + " | " + "${:,.2f}".format(r.keep_crv_value_usd) if r.rough_apr_pre_fee is not None: From f3e41060b8f47a9854e9612d510777de82e92a05 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 15 Aug 2022 21:30:50 -0400 Subject: [PATCH 45/86] fix: utils --- yearn/utils.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/yearn/utils.py b/yearn/utils.py index 8cf2dfb20..23cd953f1 100644 --- a/yearn/utils.py +++ b/yearn/utils.py @@ -14,24 +14,19 @@ from yearn.exceptions import ArchiveNodeRequired, NodeNotSynced from yearn.networks import Network from yearn.typing import Address, AddressOrContract - logger = logging.getLogger(__name__) - BINARY_SEARCH_BARRIER = { Network.Mainnet: 0, Network.Gnosis: 15_659_482, # gnosis returns "No state available for block 0x3f9e020290502d1d41f4b5519e7d456f0935dea980ec310935206cac8239117e" Network.Fantom: 4_564_024, # fantom returns "missing trie node" before that Network.Arbitrum: 0, } - _erc20 = lru_cache(maxsize=None)(interface.ERC20) - PREFER_INTERFACE = { Network.Arbitrum: { "0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f": _erc20, # empty ABI for WBTC when compiling the contract } } - def safe_views(abi: List) -> List[str]: return [ item["name"] @@ -41,8 +36,6 @@ def safe_views(abi: List) -> List[str]: and not item["inputs"] and all(x["type"] in ["uint256", "bool"] for x in item["outputs"]) ] - - @memory.cache() def get_block_timestamp(height): """ @@ -55,27 +48,20 @@ def get_block_timestamp(height): except: pass return chain[height].timestamp - - @memory.cache() def closest_block_after_timestamp(timestamp): logger.debug('closest block after timestamp %d', timestamp) height = chain.height lo, hi = 0, height - while hi - lo > 1: mid = lo + (hi - lo) // 2 if get_block_timestamp(mid) > timestamp: hi = mid else: lo = mid - if get_block_timestamp(hi) < timestamp: raise IndexError('timestamp is in the future') - return hi - - def get_code(address, block=None): try: return web3.eth.get_code(address, block_identifier=block) @@ -83,8 +69,6 @@ def get_code(address, block=None): if isinstance(exc.args[0], dict) and 'missing trie node' in exc.args[0]['message']: raise ArchiveNodeRequired('querying historical state requires an archive node') raise exc - - @memory.cache() def contract_creation_block(address: AddressOrContract) -> int: """ @@ -93,16 +77,13 @@ def contract_creation_block(address: AddressOrContract) -> int: """ logger.info("contract creation block %s", address) address = convert.to_address(address) - barrier = BINARY_SEARCH_BARRIER[chain.id] lo = barrier hi = end = chain.height - if hi == 0: raise NodeNotSynced(f''' `chain.height` returns 0 on your node, which means it is not fully synced. You can only use contract_creation_block on a fully synced node.''') - while hi - lo > 1: mid = lo + (hi - lo) // 2 try: @@ -120,32 +101,24 @@ def contract_creation_block(address: AddressOrContract) -> int: hi = mid else: lo = mid - # only happens on fantom if hi == barrier + 1: logger.warning('could not determine creation block for a contract deployed prior to barrier') return 0 - return hi if hi != end else None - - class Singleton(type): def __init__(self, *args, **kwargs): self.__instance = None super().__init__(*args, **kwargs) - def __call__(self, *args, **kwargs): if self.__instance is None: self.__instance = super().__call__(*args, **kwargs) return self.__instance else: return self.__instance - - # cached Contract instance, saves about 20ms of init time _contract_lock = threading.Lock() _contract = lru_cache(maxsize=None)(Contract) - @eth_retry.auto_retry def contract(address: Address) -> Contract: with _contract_lock: @@ -156,7 +129,6 @@ def contract(address: Address) -> Contract: _interface = PREFER_INTERFACE[chain.id][address] i = _interface(address) return _squeeze(i) - failed_attempts = 0 while True: try: @@ -214,14 +186,10 @@ def _resolve_proxy(address): def is_contract(address: str) -> bool: '''checks to see if the input address is a contract''' return web3.eth.get_code(address) not in ['0x',b''] - - def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): yield lst[i:i + n] - - def _squeeze(it): """ Reduce the contract size in RAM significantly. """ for k in ["ast", "bytecode", "coverageMap", "deployedBytecode", "deployedSourceMap", "natspec", "opcodes", "pcMap"]: From 695051d1b88f2d67a526d644194628c1fa5c4d25 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 15 Aug 2022 21:42:04 -0400 Subject: [PATCH 46/86] fix: multicall --- yearn/multicall2.py | 90 ++++++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/yearn/multicall2.py b/yearn/multicall2.py index d5c649202..2d673c0dc 100644 --- a/yearn/multicall2.py +++ b/yearn/multicall2.py @@ -1,52 +1,77 @@ +import os from collections import defaultdict from itertools import count, product from operator import itemgetter +from typing import Any, List, Optional import requests -from brownie import Contract, chain, web3 +from brownie import chain, web3 from eth_abi.exceptions import InsufficientDataBytes -from yearn.networks import Network -from yearn.utils import contract_creation_block, contract from yearn.exceptions import MulticallError +from yearn.networks import Network +from yearn.typing import Block +from yearn.utils import contract, contract_creation_block +MULTICALL_MAX_SIZE = int(os.environ.get("MULTICALL_MAX_SIZE", 500)) # Currently set arbitrarily MULTICALL2 = { Network.Mainnet: '0x5BA1e12693Dc8F9c48aAD8770482f4739bEeD696', + Network.Gnosis: '0xFAa296891cA6CECAF2D86eF5F7590316d0A17dA0', # maker has not yet deployed multicall2. This is from another deployment Network.Fantom: '0xD98e3dBE5950Ca8Ce5a4b59630a5652110403E5c', Network.Arbitrum: '0x5B5CFE992AdAC0C9D48E05854B2d91C73a003858', } multicall2 = contract(MULTICALL2[chain.id]) -def fetch_multicall(*calls, block=None, require_success=False): +def fetch_multicall(*calls, block: Optional[Block] = None, require_success: bool = False) -> List[Any]: + # Before doing anything, make sure the load is manageable and size down if necessary. + if (num_calls := len(calls)) > MULTICALL_MAX_SIZE: + batches = [calls[i:i + MULTICALL_MAX_SIZE] for i in range(0, num_calls, MULTICALL_MAX_SIZE)] + return [result for batch in batches for result in fetch_multicall(*batch, block=block, require_success=require_success)] + # https://github.com/makerdao/multicall multicall_input = [] + attribute_errors = [] fn_list = [] decoded = [] - for contract, fn_name, *fn_inputs in calls: - fn = getattr(contract, fn_name) - - # check that there aren't multiple functions with the same name - if hasattr(fn, "_get_fn_from_args"): - fn = fn._get_fn_from_args(fn_inputs) - - fn_list.append(fn) - multicall_input.append((contract, fn.encode_input(*fn_inputs))) - - if isinstance(block, int) and block < contract_creation_block(MULTICALL2[chain.id]): - # use state override to resurrect the contract prior to deployment - data = multicall2.tryAggregate.encode_input(False, multicall_input) - call = web3.eth.call( - {'to': str(multicall2), 'data': data}, - block or 'latest', - {str(multicall2): {'code': f'0x{multicall2.bytecode}'}}, - ) - result = multicall2.tryAggregate.decode_output(call) - else: - result = multicall2.tryAggregate.call( - False, multicall_input, block_identifier=block or 'latest' - ) + for i, (contract, fn_name, *fn_inputs) in enumerate(calls): + try: + fn = getattr(contract, fn_name) + + # check that there aren't multiple functions with the same name + if hasattr(fn, "_get_fn_from_args"): + fn = fn._get_fn_from_args(fn_inputs) + + fn_list.append(fn) + multicall_input.append((contract, fn.encode_input(*fn_inputs))) + except AttributeError: + if not require_success: + attribute_errors.append(i) + continue + raise + + try: + if isinstance(block, int) and block < contract_creation_block(MULTICALL2[chain.id]): + # use state override to resurrect the contract prior to deployment + data = multicall2.tryAggregate.encode_input(False, multicall_input) + call = web3.eth.call( + {'to': str(multicall2), 'data': data}, + block or 'latest', + {str(multicall2): {'code': f'0x{multicall2.bytecode}'}}, + ) + result = multicall2.tryAggregate.decode_output(call) + else: + result = multicall2.tryAggregate.call( + False, multicall_input, block_identifier=block or 'latest' + ) + except ValueError as e: + if 'out of gas' in str(e) or 'execution aborted (timeout = 10s)' in str(e): + halfpoint = len(calls) // 2 + batch0 = fetch_multicall(*calls[:halfpoint],block=block,require_success=require_success) + batch1 = fetch_multicall(*calls[halfpoint:],block=block,require_success=require_success) + return batch0 + batch1 + raise for fn, (ok, data) in zip(fn_list, result): try: @@ -57,6 +82,10 @@ def fetch_multicall(*calls, block=None, require_success=False): raise MulticallError() decoded.append(None) + # NOTE this will only run if `require_success` is True + for i in attribute_errors: + decoded.insert(i, None) + return decoded @@ -96,13 +125,16 @@ def batch_call(calls): 'method': 'eth_call', 'params': [ {'to': str(contract), 'data': fn.encode_input(*fn_inputs)}, - block, + hex(block), ], } ) response = requests.post(web3.provider.endpoint_uri, json=jsonrpc_batch).json() + if isinstance(response, dict) and isinstance(response['result'], dict) and 'message' in response['result']: + raise ValueError(response['result']['message']) + return [ fn.decode_output(res['result']) if res['result'] != '0x' else None for res in sorted(response, key=itemgetter('id')) - ] + ] \ No newline at end of file From 87aee69dcb76d62375babbcc587bc9fb0b8ece46 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 4 Oct 2022 15:46:11 -0400 Subject: [PATCH 47/86] feat: opti support --- .gitignore | 3 +- brownie-config.yaml | 3 +- contracts/interfaces/yearn/IV2Registry.sol | 42 + contracts/interfaces/yearn/IV2Vault.sol | 7 + contracts/registryhelper.sol | 974 +++++++++++++++++++++ interfaces/c.json | 1 + scripts/collect_reports.py | 24 +- scripts/deploy.py | 15 + scripts/transactions_exporter.py | 7 +- yearn/constants.py | 17 +- yearn/ironbank.py | 1 + yearn/middleware/middleware.py | 2 + yearn/multicall2.py | 9 +- yearn/networks.py | 3 + yearn/prices/chainlink.py | 12 + yearn/prices/compound.py | 6 + yearn/prices/constants.py | 15 +- yearn/utils.py | 1 + 18 files changed, 1129 insertions(+), 13 deletions(-) create mode 100644 contracts/interfaces/yearn/IV2Registry.sol create mode 100644 contracts/interfaces/yearn/IV2Vault.sol create mode 100644 contracts/registryhelper.sol create mode 100644 interfaces/c.json create mode 100644 scripts/deploy.py diff --git a/.gitignore b/.gitignore index 758c9574e..b92d68c2a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ setup_db.py fees2.py scripts/tracking scripts/update_crv.py -yvboost_fees.py \ No newline at end of file +yvboost_fees.py +env \ No newline at end of file diff --git a/brownie-config.yaml b/brownie-config.yaml index 8f863923c..f98da23a8 100644 --- a/brownie-config.yaml +++ b/brownie-config.yaml @@ -6,5 +6,4 @@ autofetch_sources: true compiler: solc: use_latest_patch: - - '0x514910771AF9Ca656af840dff83E8264EcF986CA' - - '0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f' + - '0x514910771AF9Ca656af840dff83E8264EcF986CA' \ No newline at end of file diff --git a/contracts/interfaces/yearn/IV2Registry.sol b/contracts/interfaces/yearn/IV2Registry.sol new file mode 100644 index 000000000..f08ffb98b --- /dev/null +++ b/contracts/interfaces/yearn/IV2Registry.sol @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.4; + +interface IV2Registry { + function wrappedVaults(address _vault) external view returns (address); + + function isDelegatedVault(address _vault) external view returns (bool); + + // Vaults getters + function getVault(uint256 index) external view returns (address vault); + + function getVaults() external view returns (address[] memory); + + function numTokens() external view returns (uint256 _numTokens); + + function tokens(uint256 _index) external view returns (address _token); + + function vaults(address _token, uint256 _index) external view returns (address _vault); + + function getVaultInfo(address _vault) + external + view + returns ( + address controller, + address token, + address strategy, + bool isWrapped, + bool isDelegated + ); + + function getVaultsInfo() + external + view + returns ( + address[] memory controllerArray, + address[] memory tokenArray, + address[] memory strategyArray, + bool[] memory isWrappedArray, + bool[] memory isDelegatedArray + ); +} \ No newline at end of file diff --git a/contracts/interfaces/yearn/IV2Vault.sol b/contracts/interfaces/yearn/IV2Vault.sol new file mode 100644 index 000000000..5c1576d3d --- /dev/null +++ b/contracts/interfaces/yearn/IV2Vault.sol @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.4; + +interface IV2Vault { + function withdrawalQueue(uint256 _index) external view returns (address _strategy); +} \ No newline at end of file diff --git a/contracts/registryhelper.sol b/contracts/registryhelper.sol new file mode 100644 index 000000000..eda55f158 --- /dev/null +++ b/contracts/registryhelper.sol @@ -0,0 +1,974 @@ +// SPDX-License-Identifier: MIT + +pragma solidity 0.8.4; + + +import './interfaces/yearn/IV2Registry.sol'; +import './interfaces/yearn/IV2Vault.sol'; + +/** + * @dev Collection of functions related to the address type + */ +library Address { + /** + * @dev Returns true if `account` is a contract. + * + * [IMPORTANT] + * ==== + * It is unsafe to assume that an address for which this function returns + * false is an externally-owned account (EOA) and not a contract. + * + * Among others, `isContract` will return false for the following + * types of addresses: + * + * - an externally-owned account + * - a contract in construction + * - an address where a contract will be created + * - an address where a contract lived, but was destroyed + * ==== + */ + function isContract(address account) internal view returns (bool) { + // This method relies on extcodesize, which returns 0 for contracts in + // construction, since the code is only stored at the end of the + // constructor execution. + + uint256 size; + assembly { + size := extcodesize(account) + } + return size > 0; + } + + /** + * @dev Replacement for Solidity's `transfer`: sends `amount` wei to + * `recipient`, forwarding all available gas and reverting on errors. + * + * https://eips.ethereum.org/EIPS/eip-1884[EIP1884] increases the gas cost + * of certain opcodes, possibly making contracts go over the 2300 gas limit + * imposed by `transfer`, making them unable to receive funds via + * `transfer`. {sendValue} removes this limitation. + * + * https://diligence.consensys.net/posts/2019/09/stop-using-soliditys-transfer-now/[Learn more]. + * + * IMPORTANT: because control is transferred to `recipient`, care must be + * taken to not create reentrancy vulnerabilities. Consider using + * {ReentrancyGuard} or the + * https://solidity.readthedocs.io/en/v0.5.11/security-considerations.html#use-the-checks-effects-interactions-pattern[checks-effects-interactions pattern]. + */ + function sendValue(address payable recipient, uint256 amount) internal { + require(address(this).balance >= amount, "Address: insufficient balance"); + + (bool success, ) = recipient.call{value: amount}(""); + require(success, "Address: unable to send value, recipient may have reverted"); + } + + /** + * @dev Performs a Solidity function call using a low level `call`. A + * plain `call` is an unsafe replacement for a function call: use this + * function instead. + * + * If `target` reverts with a revert reason, it is bubbled up by this + * function (like regular Solidity function calls). + * + * Returns the raw returned data. To convert to the expected return value, + * use https://solidity.readthedocs.io/en/latest/units-and-global-variables.html?highlight=abi.decode#abi-encoding-and-decoding-functions[`abi.decode`]. + * + * Requirements: + * + * - `target` must be a contract. + * - calling `target` with `data` must not revert. + * + * _Available since v3.1._ + */ + function functionCall(address target, bytes memory data) internal returns (bytes memory) { + return functionCall(target, data, "Address: low-level call failed"); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-}[`functionCall`], but with + * `errorMessage` as a fallback revert reason when `target` reverts. + * + * _Available since v3.1._ + */ + function functionCall( + address target, + bytes memory data, + string memory errorMessage + ) internal returns (bytes memory) { + return functionCallWithValue(target, data, 0, errorMessage); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-}[`functionCall`], + * but also transferring `value` wei to `target`. + * + * Requirements: + * + * - the calling contract must have an ETH balance of at least `value`. + * - the called Solidity function must be `payable`. + * + * _Available since v3.1._ + */ + function functionCallWithValue( + address target, + bytes memory data, + uint256 value + ) internal returns (bytes memory) { + return functionCallWithValue(target, data, value, "Address: low-level call with value failed"); + } + + /** + * @dev Same as {xref-Address-functionCallWithValue-address-bytes-uint256-}[`functionCallWithValue`], but + * with `errorMessage` as a fallback revert reason when `target` reverts. + * + * _Available since v3.1._ + */ + function functionCallWithValue( + address target, + bytes memory data, + uint256 value, + string memory errorMessage + ) internal returns (bytes memory) { + require(address(this).balance >= value, "Address: insufficient balance for call"); + require(isContract(target), "Address: call to non-contract"); + + (bool success, bytes memory returndata) = target.call{value: value}(data); + return verifyCallResult(success, returndata, errorMessage); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-}[`functionCall`], + * but performing a static call. + * + * _Available since v3.3._ + */ + function functionStaticCall(address target, bytes memory data) internal view returns (bytes memory) { + return functionStaticCall(target, data, "Address: low-level static call failed"); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-string-}[`functionCall`], + * but performing a static call. + * + * _Available since v3.3._ + */ + function functionStaticCall( + address target, + bytes memory data, + string memory errorMessage + ) internal view returns (bytes memory) { + require(isContract(target), "Address: static call to non-contract"); + + (bool success, bytes memory returndata) = target.staticcall(data); + return verifyCallResult(success, returndata, errorMessage); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-}[`functionCall`], + * but performing a delegate call. + * + * _Available since v3.4._ + */ + function functionDelegateCall(address target, bytes memory data) internal returns (bytes memory) { + return functionDelegateCall(target, data, "Address: low-level delegate call failed"); + } + + /** + * @dev Same as {xref-Address-functionCall-address-bytes-string-}[`functionCall`], + * but performing a delegate call. + * + * _Available since v3.4._ + */ + function functionDelegateCall( + address target, + bytes memory data, + string memory errorMessage + ) internal returns (bytes memory) { + require(isContract(target), "Address: delegate call to non-contract"); + + (bool success, bytes memory returndata) = target.delegatecall(data); + return verifyCallResult(success, returndata, errorMessage); + } + + /** + * @dev Tool to verifies that a low level call was successful, and revert if it wasn't, either by bubbling the + * revert reason using the provided one. + * + * _Available since v4.3._ + */ + function verifyCallResult( + bool success, + bytes memory returndata, + string memory errorMessage + ) internal pure returns (bytes memory) { + if (success) { + return returndata; + } else { + // Look for revert reason and bubble it up if present + if (returndata.length > 0) { + // The easiest way to bubble the revert reason is using memory via assembly + + assembly { + let returndata_size := mload(returndata) + revert(add(32, returndata), returndata_size) + } + } else { + revert(errorMessage); + } + } + } +} + +/** + * @dev Library for managing + * https://en.wikipedia.org/wiki/Set_(abstract_data_type)[sets] of primitive + * types. + * + * Sets have the following properties: + * + * - Elements are added, removed, and checked for existence in constant time + * (O(1)). + * - Elements are enumerated in O(n). No guarantees are made on the ordering. + * + * ``` + * contract Example { + * // Add the library methods + * using EnumerableSet for EnumerableSet.AddressSet; + * + * // Declare a set state variable + * EnumerableSet.AddressSet private mySet; + * } + * ``` + * + * As of v3.3.0, sets of type `bytes32` (`Bytes32Set`), `address` (`AddressSet`) + * and `uint256` (`UintSet`) are supported. + */ +library EnumerableSet { + // To implement this library for multiple types with as little code + // repetition as possible, we write it in terms of a generic Set type with + // bytes32 values. + // The Set implementation uses private functions, and user-facing + // implementations (such as AddressSet) are just wrappers around the + // underlying Set. + // This means that we can only create new EnumerableSets for types that fit + // in bytes32. + + struct Set { + // Storage of set values + bytes32[] _values; + // Position of the value in the `values` array, plus 1 because index 0 + // means a value is not in the set. + mapping(bytes32 => uint256) _indexes; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function _add(Set storage set, bytes32 value) private returns (bool) { + if (!_contains(set, value)) { + set._values.push(value); + // The value is stored at length-1, but we add 1 to all indexes + // and use 0 as a sentinel value + set._indexes[value] = set._values.length; + return true; + } else { + return false; + } + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function _remove(Set storage set, bytes32 value) private returns (bool) { + // We read and store the value's index to prevent multiple reads from the same storage slot + uint256 valueIndex = set._indexes[value]; + + if (valueIndex != 0) { + // Equivalent to contains(set, value) + // To delete an element from the _values array in O(1), we swap the element to delete with the last one in + // the array, and then remove the last element (sometimes called as 'swap and pop'). + // This modifies the order of the array, as noted in {at}. + + uint256 toDeleteIndex = valueIndex - 1; + uint256 lastIndex = set._values.length - 1; + + if (lastIndex != toDeleteIndex) { + bytes32 lastvalue = set._values[lastIndex]; + + // Move the last value to the index where the value to delete is + set._values[toDeleteIndex] = lastvalue; + // Update the index for the moved value + set._indexes[lastvalue] = valueIndex; // Replace lastvalue's index to valueIndex + } + + // Delete the slot where the moved value was stored + set._values.pop(); + + // Delete the index for the deleted slot + delete set._indexes[value]; + + return true; + } else { + return false; + } + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function _contains(Set storage set, bytes32 value) private view returns (bool) { + return set._indexes[value] != 0; + } + + /** + * @dev Returns the number of values on the set. O(1). + */ + function _length(Set storage set) private view returns (uint256) { + return set._values.length; + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function _at(Set storage set, uint256 index) private view returns (bytes32) { + return set._values[index]; + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function _values(Set storage set) private view returns (bytes32[] memory) { + return set._values; + } + + // Bytes32Set + + struct Bytes32Set { + Set _inner; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(Bytes32Set storage set, bytes32 value) internal returns (bool) { + return _add(set._inner, value); + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(Bytes32Set storage set, bytes32 value) internal returns (bool) { + return _remove(set._inner, value); + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(Bytes32Set storage set, bytes32 value) internal view returns (bool) { + return _contains(set._inner, value); + } + + /** + * @dev Returns the number of values in the set. O(1). + */ + function length(Bytes32Set storage set) internal view returns (uint256) { + return _length(set._inner); + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(Bytes32Set storage set, uint256 index) internal view returns (bytes32) { + return _at(set._inner, index); + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(Bytes32Set storage set) internal view returns (bytes32[] memory) { + return _values(set._inner); + } + + // AddressSet + + struct AddressSet { + Set _inner; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(AddressSet storage set, address value) internal returns (bool) { + return _add(set._inner, bytes32(uint256(uint160(value)))); + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(AddressSet storage set, address value) internal returns (bool) { + return _remove(set._inner, bytes32(uint256(uint160(value)))); + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(AddressSet storage set, address value) internal view returns (bool) { + return _contains(set._inner, bytes32(uint256(uint160(value)))); + } + + /** + * @dev Returns the number of values in the set. O(1). + */ + function length(AddressSet storage set) internal view returns (uint256) { + return _length(set._inner); + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(AddressSet storage set, uint256 index) internal view returns (address) { + return address(uint160(uint256(_at(set._inner, index)))); + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(AddressSet storage set) internal view returns (address[] memory) { + bytes32[] memory store = _values(set._inner); + address[] memory result; + + assembly { + result := store + } + + return result; + } + + // UintSet + + struct UintSet { + Set _inner; + } + + /** + * @dev Add a value to a set. O(1). + * + * Returns true if the value was added to the set, that is if it was not + * already present. + */ + function add(UintSet storage set, uint256 value) internal returns (bool) { + return _add(set._inner, bytes32(value)); + } + + /** + * @dev Removes a value from a set. O(1). + * + * Returns true if the value was removed from the set, that is if it was + * present. + */ + function remove(UintSet storage set, uint256 value) internal returns (bool) { + return _remove(set._inner, bytes32(value)); + } + + /** + * @dev Returns true if the value is in the set. O(1). + */ + function contains(UintSet storage set, uint256 value) internal view returns (bool) { + return _contains(set._inner, bytes32(value)); + } + + /** + * @dev Returns the number of values on the set. O(1). + */ + function length(UintSet storage set) internal view returns (uint256) { + return _length(set._inner); + } + + /** + * @dev Returns the value stored at position `index` in the set. O(1). + * + * Note that there are no guarantees on the ordering of values inside the + * array, and it may change when more values are added or removed. + * + * Requirements: + * + * - `index` must be strictly less than {length}. + */ + function at(UintSet storage set, uint256 index) internal view returns (uint256) { + return uint256(_at(set._inner, index)); + } + + /** + * @dev Return the entire set in an array + * + * WARNING: This operation will copy the entire storage to memory, which can be quite expensive. This is designed + * to mostly be used by view accessors that are queried without any gas fees. Developers should keep in mind that + * this function has an unbounded cost, and using it as part of a state-changing function may render the function + * uncallable if the set grows to a point where copying to memory consumes too much gas to fit in a block. + */ + function values(UintSet storage set) internal view returns (uint256[] memory) { + bytes32[] memory store = _values(set._inner); + uint256[] memory result; + + assembly { + result := store + } + + return result; + } +} + +pragma solidity ^0.8.0; + +/** + * @dev Interface of the ERC20 standard as defined in the EIP. + */ +interface IERC20 { + /** + * @dev Returns the amount of tokens in existence. + */ + function totalSupply() external view returns (uint256); + + /** + * @dev Returns the amount of tokens owned by `account`. + */ + function balanceOf(address account) external view returns (uint256); + + /** + * @dev Moves `amount` tokens from the caller's account to `recipient`. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * Emits a {Transfer} event. + */ + function transfer(address recipient, uint256 amount) external returns (bool); + + /** + * @dev Returns the remaining number of tokens that `spender` will be + * allowed to spend on behalf of `owner` through {transferFrom}. This is + * zero by default. + * + * This value changes when {approve} or {transferFrom} are called. + */ + function allowance(address owner, address spender) external view returns (uint256); + + /** + * @dev Sets `amount` as the allowance of `spender` over the caller's tokens. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * IMPORTANT: Beware that changing an allowance with this method brings the risk + * that someone may use both the old and the new allowance by unfortunate + * transaction ordering. One possible solution to mitigate this race + * condition is to first reduce the spender's allowance to 0 and set the + * desired value afterwards: + * https://github.com/ethereum/EIPs/issues/20#issuecomment-263524729 + * + * Emits an {Approval} event. + */ + function approve(address spender, uint256 amount) external returns (bool); + + /** + * @dev Moves `amount` tokens from `sender` to `recipient` using the + * allowance mechanism. `amount` is then deducted from the caller's + * allowance. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * Emits a {Transfer} event. + */ + function transferFrom( + address sender, + address recipient, + uint256 amount + ) external returns (bool); + + /** + * @dev Emitted when `value` tokens are moved from one account (`from`) to + * another (`to`). + * + * Note that `value` may be zero. + */ + event Transfer(address indexed from, address indexed to, uint256 value); + + /** + * @dev Emitted when the allowance of a `spender` for an `owner` is set by + * a call to {approve}. `value` is the new allowance. + */ + event Approval(address indexed owner, address indexed spender, uint256 value); +} + + + +/** + * @title SafeERC20 + * @dev Wrappers around ERC20 operations that throw on failure (when the token + * contract returns false). Tokens that return no value (and instead revert or + * throw on failure) are also supported, non-reverting calls are assumed to be + * successful. + * To use this library you can add a `using SafeERC20 for IERC20;` statement to your contract, + * which allows you to call the safe operations as `token.safeTransfer(...)`, etc. + */ +library SafeERC20 { + using Address for address; + + function safeTransfer( + IERC20 token, + address to, + uint256 value + ) internal { + _callOptionalReturn(token, abi.encodeWithSelector(token.transfer.selector, to, value)); + } + + function safeTransferFrom( + IERC20 token, + address from, + address to, + uint256 value + ) internal { + _callOptionalReturn(token, abi.encodeWithSelector(token.transferFrom.selector, from, to, value)); + } + + /** + * @dev Deprecated. This function has issues similar to the ones found in + * {IERC20-approve}, and its usage is discouraged. + * + * Whenever possible, use {safeIncreaseAllowance} and + * {safeDecreaseAllowance} instead. + */ + function safeApprove( + IERC20 token, + address spender, + uint256 value + ) internal { + // safeApprove should only be called when setting an initial allowance, + // or when resetting it to zero. To increase and decrease it, use + // 'safeIncreaseAllowance' and 'safeDecreaseAllowance' + require( + (value == 0) || (token.allowance(address(this), spender) == 0), + "SafeERC20: approve from non-zero to non-zero allowance" + ); + _callOptionalReturn(token, abi.encodeWithSelector(token.approve.selector, spender, value)); + } + + function safeIncreaseAllowance( + IERC20 token, + address spender, + uint256 value + ) internal { + uint256 newAllowance = token.allowance(address(this), spender) + value; + _callOptionalReturn(token, abi.encodeWithSelector(token.approve.selector, spender, newAllowance)); + } + + function safeDecreaseAllowance( + IERC20 token, + address spender, + uint256 value + ) internal { + unchecked { + uint256 oldAllowance = token.allowance(address(this), spender); + require(oldAllowance >= value, "SafeERC20: decreased allowance below zero"); + uint256 newAllowance = oldAllowance - value; + _callOptionalReturn(token, abi.encodeWithSelector(token.approve.selector, spender, newAllowance)); + } + } + + /** + * @dev Imitates a Solidity high-level call (i.e. a regular function call to a contract), relaxing the requirement + * on the return value: the return value is optional (but if data is returned, it must not be false). + * @param token The token targeted by the call. + * @param data The call data (encoded using abi.encode or one of its variants). + */ + function _callOptionalReturn(IERC20 token, bytes memory data) private { + // We need to perform a low level call here, to bypass Solidity's return data size checking mechanism, since + // we're implementing it ourselves. We use {Address.functionCall} to perform this call, which verifies that + // the target address contains contract code and also asserts for success in the low-level call. + + bytes memory returndata = address(token).functionCall(data, "SafeERC20: low-level call failed"); + if (returndata.length > 0) { + // Return data is optional + require(abi.decode(returndata, (bool)), "SafeERC20: ERC20 operation did not succeed"); + } + } +} + +interface ICollectableDust { + event DustSent(address _to, address token, uint256 amount); + + function sendDust( + address _to, + address _token, + uint256 _amount + ) external; +} + +abstract contract CollectableDust is ICollectableDust { + using SafeERC20 for IERC20; + using EnumerableSet for EnumerableSet.AddressSet; + + address public constant ETH_ADDRESS = 0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE; + EnumerableSet.AddressSet internal protocolTokens; + + constructor() {} + + function _addProtocolToken(address _token) internal { + require(!protocolTokens.contains(_token), 'collectable-dust/token-is-part-of-the-protocol'); + protocolTokens.add(_token); + } + + function _removeProtocolToken(address _token) internal { + require(protocolTokens.contains(_token), 'collectable-dust/token-not-part-of-the-protocol'); + protocolTokens.remove(_token); + } + + function _sendDust( + address _to, + address _token, + uint256 _amount + ) internal { + require(_to != address(0), 'collectable-dust/cant-send-dust-to-zero-address'); + require(!protocolTokens.contains(_token), 'collectable-dust/token-is-part-of-the-protocol'); + if (_token == ETH_ADDRESS) { + payable(_to).transfer(_amount); + } else { + IERC20(_token).safeTransfer(_to, _amount); + } + emit DustSent(_to, _token, _amount); + } +} + +interface IGovernable { + event PendingGovernorSet(address pendingGovernor); + event GovernorAccepted(); + + function setPendingGovernor(address _pendingGovernor) external; + + function acceptGovernor() external; + + function governor() external view returns (address _governor); + + function pendingGovernor() external view returns (address _pendingGovernor); + + function isGovernor(address _account) external view returns (bool _isGovernor); +} + +contract Governable is IGovernable { + address public override governor; + address public override pendingGovernor; + + constructor(address _governor) { + require(_governor != address(0), 'governable/governor-should-not-be-zero-address'); + governor = _governor; + } + + function setPendingGovernor(address _pendingGovernor) external virtual override onlyGovernor { + _setPendingGovernor(_pendingGovernor); + } + + function acceptGovernor() external virtual override onlyPendingGovernor { + _acceptGovernor(); + } + + function _setPendingGovernor(address _pendingGovernor) internal { + require(_pendingGovernor != address(0), 'governable/pending-governor-should-not-be-zero-addres'); + pendingGovernor = _pendingGovernor; + emit PendingGovernorSet(_pendingGovernor); + } + + function _acceptGovernor() internal { + governor = pendingGovernor; + pendingGovernor = address(0); + emit GovernorAccepted(); + } + + function isGovernor(address _account) public view override returns (bool _isGovernor) { + return _account == governor; + } + + modifier onlyGovernor() { + require(isGovernor(msg.sender), 'governable/only-governor'); + _; + } + + modifier onlyPendingGovernor() { + require(msg.sender == pendingGovernor, 'governable/only-pending-governor'); + _; + } +} + + +interface IPausable { + event Paused(bool _paused); + + function pause(bool _paused) external; +} + +abstract contract Pausable is IPausable { + bool public paused; + + constructor() {} + + modifier notPaused() { + require(!paused, 'paused'); + _; + } + + function _pause(bool _paused) internal { + require(paused != _paused, 'no-change'); + paused = _paused; + emit Paused(_paused); + } +} + +abstract contract UtilsReady is Governable, CollectableDust, Pausable { + constructor() Governable(msg.sender) {} + + // Governable: restricted-access + function setPendingGovernor(address _pendingGovernor) external override onlyGovernor { + _setPendingGovernor(_pendingGovernor); + } + + function acceptGovernor() external override onlyPendingGovernor { + _acceptGovernor(); + } + + // Collectable Dust: restricted-access + function sendDust( + address _to, + address _token, + uint256 _amount + ) external virtual override onlyGovernor { + _sendDust(_to, _token, _amount); + } + + // Pausable: restricted-access + function pause(bool _paused) external override onlyGovernor { + _pause(_paused); + } +} + +interface IVaultsRegistryHelper { + function registry() external view returns (address _registry); + + function getVaults() external view returns (address[] memory _vaults); + + function getVaultStrategies(address _vault) external view returns (address[] memory _strategies); + + function getVaultsAndStrategies() external view returns (address[] memory _vaults, address[] memory _strategies); +} + +contract VaultsRegistryHelper is UtilsReady, IVaultsRegistryHelper { + using Address for address; + + address public immutable override registry; + + constructor(address _registry) UtilsReady() { + registry = _registry; + } + + function getVaults() public view override returns (address[] memory _vaults) { + uint256 _tokensLength = IV2Registry(registry).numTokens(); + // vaults = []; + address[] memory _vaultsArray = new address[](_tokensLength * 20); // MAX length + uint256 _vaultIndex = 0; + for (uint256 i; i < _tokensLength; i++) { + address _token = IV2Registry(registry).tokens(i); + for (uint256 j; j < 20; j++) { + address _vault = IV2Registry(registry).vaults(_token, j); + if (_vault == address(0)) break; + _vaultsArray[_vaultIndex] = _vault; + _vaultIndex++; + } + } + _vaults = new address[](_vaultIndex); + for (uint256 i; i < _vaultIndex; i++) { + _vaults[i] = _vaultsArray[i]; + } + } + + function getVaultStrategies(address _vault) public view override returns (address[] memory _strategies) { + address[] memory _strategiesArray = new address[](20); // MAX length + uint256 i; + for (i; i < 20; i++) { + address _strategy = IV2Vault(_vault).withdrawalQueue(i); + if (_strategy == address(0)) break; + _strategiesArray[i] = _strategy; + } + _strategies = new address[](i); + for (uint256 j; j < i; j++) { + _strategies[j] = _strategiesArray[j]; + } + } + + function getVaultsAndStrategies() external view override returns (address[] memory _vaults, address[] memory _strategies) { + _vaults = getVaults(); + address[] memory _strategiesArray = new address[](_vaults.length * 20); // MAX length + uint256 _strategiesIndex; + for (uint256 i; i < _vaults.length; i++) { + address[] memory _vaultStrategies = getVaultStrategies(_vaults[i]); + for (uint256 j; j < _vaultStrategies.length; j++) { + _strategiesArray[_strategiesIndex + j] = _vaultStrategies[j]; + } + _strategiesIndex += _vaultStrategies.length; + } + + _strategies = new address[](_strategiesIndex); + for (uint256 j; j < _strategiesIndex; j++) { + _strategies[j] = _strategiesArray[j]; + } + } +} \ No newline at end of file diff --git a/interfaces/c.json b/interfaces/c.json new file mode 100644 index 000000000..75da07c0c --- /dev/null +++ b/interfaces/c.json @@ -0,0 +1 @@ +[{"inputs":[{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call[]","name":"calls","type":"tuple[]"}],"name":"aggregate","outputs":[{"internalType":"uint256","name":"blockNumber","type":"uint256"},{"internalType":"bytes[]","name":"returnData","type":"bytes[]"}],"stateMutability":"payable","type":"function"},{"inputs":[{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bool","name":"allowFailure","type":"bool"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call3[]","name":"calls","type":"tuple[]"}],"name":"aggregate3","outputs":[{"components":[{"internalType":"bool","name":"success","type":"bool"},{"internalType":"bytes","name":"returnData","type":"bytes"}],"internalType":"struct Multicall3.Result[]","name":"returnData","type":"tuple[]"}],"stateMutability":"payable","type":"function"},{"inputs":[{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bool","name":"allowFailure","type":"bool"},{"internalType":"uint256","name":"value","type":"uint256"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call3Value[]","name":"calls","type":"tuple[]"}],"name":"aggregate3Value","outputs":[{"components":[{"internalType":"bool","name":"success","type":"bool"},{"internalType":"bytes","name":"returnData","type":"bytes"}],"internalType":"struct Multicall3.Result[]","name":"returnData","type":"tuple[]"}],"stateMutability":"payable","type":"function"},{"inputs":[{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call[]","name":"calls","type":"tuple[]"}],"name":"blockAndAggregate","outputs":[{"internalType":"uint256","name":"blockNumber","type":"uint256"},{"internalType":"bytes32","name":"blockHash","type":"bytes32"},{"components":[{"internalType":"bool","name":"success","type":"bool"},{"internalType":"bytes","name":"returnData","type":"bytes"}],"internalType":"struct Multicall3.Result[]","name":"returnData","type":"tuple[]"}],"stateMutability":"payable","type":"function"},{"inputs":[],"name":"getBasefee","outputs":[{"internalType":"uint256","name":"basefee","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"blockNumber","type":"uint256"}],"name":"getBlockHash","outputs":[{"internalType":"bytes32","name":"blockHash","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getBlockNumber","outputs":[{"internalType":"uint256","name":"blockNumber","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getChainId","outputs":[{"internalType":"uint256","name":"chainid","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getCurrentBlockCoinbase","outputs":[{"internalType":"address","name":"coinbase","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getCurrentBlockDifficulty","outputs":[{"internalType":"uint256","name":"difficulty","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getCurrentBlockGasLimit","outputs":[{"internalType":"uint256","name":"gaslimit","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getCurrentBlockTimestamp","outputs":[{"internalType":"uint256","name":"timestamp","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"addr","type":"address"}],"name":"getEthBalance","outputs":[{"internalType":"uint256","name":"balance","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"getLastBlockHash","outputs":[{"internalType":"bytes32","name":"blockHash","type":"bytes32"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"bool","name":"requireSuccess","type":"bool"},{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call[]","name":"calls","type":"tuple[]"}],"name":"tryAggregate","outputs":[{"components":[{"internalType":"bool","name":"success","type":"bool"},{"internalType":"bytes","name":"returnData","type":"bytes"}],"internalType":"struct Multicall3.Result[]","name":"returnData","type":"tuple[]"}],"stateMutability":"payable","type":"function"},{"inputs":[{"internalType":"bool","name":"requireSuccess","type":"bool"},{"components":[{"internalType":"address","name":"target","type":"address"},{"internalType":"bytes","name":"callData","type":"bytes"}],"internalType":"struct Multicall3.Call[]","name":"calls","type":"tuple[]"}],"name":"tryBlockAndAggregate","outputs":[{"internalType":"uint256","name":"blockNumber","type":"uint256"},{"internalType":"bytes32","name":"blockHash","type":"bytes32"},{"components":[{"internalType":"bool","name":"success","type":"bool"},{"internalType":"bytes","name":"returnData","type":"bytes"}],"internalType":"struct Multicall3.Result[]","name":"returnData","type":"tuple[]"}],"stateMutability":"payable","type":"function"}] \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 638a2ef8f..d36a0510d 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -30,7 +30,7 @@ inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') -alerts_enabled = True if env == "PROD" or env == "TEST" else False +alerts_enabled = True #if env == "PROD" or env == "TEST" else False test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') if env == "TEST": @@ -133,6 +133,28 @@ "TENDERLY_CHAIN_IDENTIFIER": "arbitrum", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_42161_PUBLIC'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_42161'), + }, + Network.Optimism: { + "NETWORK_NAME": "Optimism", + "NETWORK_SYMBOL": "OPT", + "EMOJI": "🔴", + "START_DATE": datetime(2022, 8, 6, tzinfo=timezone.utc), + "START_BLOCK": 24097341, + "REGISTRY_ADDRESS": "0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0", + "REGISTRY_DEPLOY_BLOCK": 18097341, + "REGISTRY_HELPER_ADDRESS": "0x0983b4899a3168c2509569faf1e4e75c57b4aba6", + "LENS_ADDRESS": "0xD3A93C794ee2798D8f7906493Cd3c2A835aa0074", + "VAULT_ADDRESS030": "0x0fBeA11f39be912096cEc5cE22F46908B5375c19", + "VAULT_ADDRESS031": "0x0fBeA11f39be912096cEc5cE22F46908B5375c19", + "KEEPER_CALL_CONTRACT": "", + "KEEPER_TOKEN": "", + "YEARN_TREASURY": "0x84654e35E504452769757AAe5a8C7C6599cBf954", + "STRATEGIST_MULTISIG": "0xea3a15df68fCdBE44Fdb0DB675B2b3A14a148b26", + "GOVERNANCE_MULTISIG": "0xF5d9D6133b698cE29567a90Ab35CfB874204B3A7", + "EXPLORER_URL": "https://optimistic.etherscan.io/", + "TENDERLY_CHAIN_IDENTIFIER": "optimism", + "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_10_PUBLIC'), + "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_10'), } } diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100644 index 000000000..aada632bf --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,15 @@ +from brownie import config, Wei, Contract, chain, accounts, VaultsRegistryHelper +import requests + +def main(): + wavey = accounts.load('wavey') + # ycrv = wavey.deploy( + # StrategyProxy, + # publish_source=True + # ) + + strat = wavey.deploy( + VaultsRegistryHelper, + '0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0', # Registry + publish_source=True + ) \ No newline at end of file diff --git a/scripts/transactions_exporter.py b/scripts/transactions_exporter.py index 68f805502..512587bcf 100644 --- a/scripts/transactions_exporter.py +++ b/scripts/transactions_exporter.py @@ -26,8 +26,11 @@ BATCH_SIZE = 5000 FIRST_END_BLOCK = { - Network.Mainnet: 9480000, # NOTE block some arbitrary time after iearn's first deployment - Network.Fantom: 5000000, # NOTE block some arbitrary time after v2's first deployment + Network.Mainnet: 9_480_000, # NOTE block some arbitrary time after iearn's first deployment + Network.Fantom: 5_000_000, # NOTE block some arbitrary time after v2's first deployment + Network.Gnosis: 21_440_000, # # NOTE block some arbitrary time after first vault deployment + Network.Arbitrum: 4_837_859, + Network.Optimism: 18_111_485, }[chain.id] def main(): diff --git a/yearn/constants.py b/yearn/constants.py index e7bb181a6..702331515 100644 --- a/yearn/constants.py +++ b/yearn/constants.py @@ -7,6 +7,7 @@ Network.Fantom: "0x21be370D5312f44cB42ce377BC9b8a0cEF1A4C83", Network.Arbitrum: "0x82aF49447D8a07e3bd95BD0d56f35241523fBab1", Network.Gnosis: "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d", + Network.Optimism: "0x4200000000000000000000000000000000000006", }.get(chain.id, None) YEARN_ADDRESSES_PROVIDER = "0x9be19Ee7Bc4099D62737a7255f5c227fBcd6dB93" @@ -29,7 +30,10 @@ }, Network.Arbitrum: { "0x6346282db8323a54e840c6c772b4399c9c655c0d", - } + }, + Network.Optimism: { + "0xea3a15df68fCdBE44Fdb0DB675B2b3A14a148b26", + }, }.get(chain.id,set()) STRATEGIST_MULTISIG = {convert.to_address(address) for address in STRATEGIST_MULTISIG} @@ -39,6 +43,7 @@ Network.Fantom: "0xC0E2830724C946a6748dDFE09753613cd38f6767", Network.Gnosis: "0x22eAe41c7Da367b9a15e942EB6227DF849Bb498C", Network.Arbitrum: "0xb6bc033d34733329971b938fef32fad7e98e56ad", + Network.Optimism: "0xF5d9D6133b698cE29567a90Ab35CfB874204B3A7", }.get(chain.id, None) if YCHAD_MULTISIG: @@ -48,6 +53,7 @@ Network.Mainnet: "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", Network.Fantom: "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", Network.Arbitrum: "0x1deb47dcc9a35ad454bf7f0fcdb03c09792c08c1", + Network.Optimism: "0x84654e35E504452769757AAe5a8C7C6599cBf954", }.get(chain.id, None) if TREASURY_MULTISIG: @@ -68,12 +74,17 @@ }, Network.Gnosis: { YCHAD_MULTISIG, - "0x5FcdC32DfC361a32e9d5AB9A384b890C62D0b8AC", # Yearn Treasury (EOA?) + # TODO replace this with treasury msig + #"0x5FcdC32DfC361a32e9d5AB9A384b890C62D0b8AC", # Yearn Treasury (EOA?) }, Network.Arbitrum: { YCHAD_MULTISIG, TREASURY_MULTISIG, - } + }, + Network.Optimism: { + YCHAD_MULTISIG, + TREASURY_MULTISIG, + }, }.get(chain.id,set()) TREASURY_WALLETS = {convert.to_address(address) for address in TREASURY_WALLETS} \ No newline at end of file diff --git a/yearn/ironbank.py b/yearn/ironbank.py index 3918c71a3..fe764806d 100644 --- a/yearn/ironbank.py +++ b/yearn/ironbank.py @@ -23,6 +23,7 @@ Network.Mainnet: '0xAB1c342C7bf5Ec5F02ADEA1c2270670bCa144CbB', Network.Fantom: get_fantom_ironbank, Network.Arbitrum: '0xbadaC56c9aca307079e8B8FC699987AAc89813ee', + Network.Optimism: '0xE0B57FEEd45e7D908f2d0DaCd26F113Cf26715BF' } diff --git a/yearn/middleware/middleware.py b/yearn/middleware/middleware.py index c0fc7d0a9..af3b956ca 100644 --- a/yearn/middleware/middleware.py +++ b/yearn/middleware/middleware.py @@ -2,6 +2,7 @@ from brownie import chain from brownie import web3 as w3 +from http.client import NETWORK_AUTHENTICATION_REQUIRED from eth_utils import encode_hex from eth_utils import function_signature_to_4byte_selector as fourbyte from requests import Session @@ -19,6 +20,7 @@ Network.Mainnet: 10_000, # 1.58 days Network.Fantom: 100_000, # 1.03 days Network.Arbitrum: 20_000, # 0.34 days + Network.Optimism: 80_000, # 1.02 days } CACHED_CALLS = [ "name()", diff --git a/yearn/multicall2.py b/yearn/multicall2.py index 2d673c0dc..68d07225b 100644 --- a/yearn/multicall2.py +++ b/yearn/multicall2.py @@ -5,7 +5,7 @@ from typing import Any, List, Optional import requests -from brownie import chain, web3 +from brownie import chain, web3, interface from eth_abi.exceptions import InsufficientDataBytes from yearn.exceptions import MulticallError @@ -19,9 +19,12 @@ Network.Gnosis: '0xFAa296891cA6CECAF2D86eF5F7590316d0A17dA0', # maker has not yet deployed multicall2. This is from another deployment Network.Fantom: '0xD98e3dBE5950Ca8Ce5a4b59630a5652110403E5c', Network.Arbitrum: '0x5B5CFE992AdAC0C9D48E05854B2d91C73a003858', + Network.Optimism: '0xcA11bde05977b3631167028862bE2a173976CA11', # Multicall 3 } -multicall2 = contract(MULTICALL2[chain.id]) - +if chain.id == Network.Optimism: + multicall2 = interface.c(MULTICALL2[chain.id]) +else: + multicall2 = contract(MULTICALL2[chain.id]) def fetch_multicall(*calls, block: Optional[Block] = None, require_success: bool = False) -> List[Any]: # Before doing anything, make sure the load is manageable and size down if necessary. diff --git a/yearn/networks.py b/yearn/networks.py index ce5164533..c0b0b3ec0 100644 --- a/yearn/networks.py +++ b/yearn/networks.py @@ -12,6 +12,7 @@ class Network(IntEnum): Gnosis = 100 Fantom = 250 Arbitrum = 42161 + Optimism = 10 @staticmethod def label(chain_id: int = None): @@ -26,6 +27,8 @@ def label(chain_id: int = None): return "FTM" elif chain_id == Network.Arbitrum: return "ARBB" + elif chain_id == Network.Optimism: + return "OPT" else: raise UnsupportedNetwork( f'chainid {chain_id} is not currently supported. Please add network details to yearn-exporter/yearn/networks.py' diff --git a/yearn/prices/chainlink.py b/yearn/prices/chainlink.py index 663c5ff86..5d85a72f9 100644 --- a/yearn/prices/chainlink.py +++ b/yearn/prices/chainlink.py @@ -74,6 +74,17 @@ "0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8" : "0x50834F3163758fcC1Df9973b6e91f0F0F0434aD3", # usdc "0xFEa7a6a0B346362BF88A9e4A88416B77a57D6c2A" : "0x87121F6c9A9F6E90E59591E4Cf4804873f54A95b", # mim "0x82e3A8F066a6989666b031d916c43672085b1582" : "0x745Ab5b69E01E2BE1104Ca84937Bb71f96f5fB21", # yfi + }, + + Network.Optimism: { + "0x68f180fcCe6836688e9084f035309E29Bf0A2095" : "0xD702DD976Fb76Fffc2D3963D037dfDae5b04E593", # wbtc + "0x4200000000000000000000000000000000000006" : "0x13e3Ee699D1909E989722E753853AE30b17e08c5", # weth + "0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1" : "0x8dBa75e83DA73cc766A7e5a0ee71F656BAb470d6", # dai + "0x4200000000000000000000000000000000000042" : "0x0D276FC14719f9292D5C1eA2198673d1f4269246", # op + "0x2E3D870790dC77A83DD1d18184Acc7439A53f475" : "0xc7D132BeCAbE7Dcc4204841F33bae45841e41D9C", # frax + "0x350a791Bfc2C21F9Ed5d10980Dad2e2638ffa7f6" : "0xCc232dcFAAE6354cE191Bd574108c1aD03f86450", # link + "0x7F5c764cBc14f9669B88837ca1490cCa17c31607" : "0x16a9FA2FDa030272Ce99B29CF780dFA30361E0f3", # usdc + "0x8700dAec35aF8Ff88c16BdF0418774CB3D7599B4" : "0x2FCF37343e916eAEd1f1DdaaF84458a359b53877", # snx } } registries = { @@ -82,6 +93,7 @@ Network.Fantom: None, Network.Gnosis: None, Network.Arbitrum: None, + Network.Optimism: None, } diff --git a/yearn/prices/compound.py b/yearn/prices/compound.py index 2aaaf1898..4070235c7 100644 --- a/yearn/prices/compound.py +++ b/yearn/prices/compound.py @@ -69,6 +69,12 @@ class CompoundConfig: address='0xbadaC56c9aca307079e8B8FC699987AAc89813ee', ), ], + Network.Optimism: [ + CompoundConfig( + name='ironbank', + address='0xE0B57FEEd45e7D908f2d0DaCd26F113Cf26715BF', + ) + ], } diff --git a/yearn/prices/constants.py b/yearn/prices/constants.py index 3330358f5..d0e63af52 100644 --- a/yearn/prices/constants.py +++ b/yearn/prices/constants.py @@ -23,6 +23,11 @@ 'usdc': '0xFF970A61A04b1cA14834A43f5dE4533eBDDB5CC8', 'dai': '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1', }, + Network.Optimism: { + 'weth': '0x4200000000000000000000000000000000000006', + 'usdc': '0x7F5c764cBc14f9669B88837ca1490cCa17c31607', + 'dai': '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1', + }, } stablecoins_by_network = { @@ -76,13 +81,21 @@ '0xFEa7a6a0B346362BF88A9e4A88416B77a57D6c2A': 'mim', '0xFd086bC7CD5C481DCC9C85ebE478A1C0b69FCbb9': 'usdt' }, + Network.Optimism: { + '0x94b008aA00579c1307B0EF2c499aD98a8ce58e58': 'usdt', + '0x7F5c764cBc14f9669B88837ca1490cCa17c31607': 'usdc', + '0xDA10009cBd5D07dd0CeCc66161FC93D7c9000da1': 'dai', + '0x2E3D870790dC77A83DD1d18184Acc7439A53f475': 'frax', + '0x8c6f28f2F1A3C87F0f938b96d27520d9751ec8d9': 'susd', + }, } ib_snapshot_block_by_network = { Network.Mainnet: 14051986, Network.Fantom: 28680044, Network.Gnosis: 1, # TODO revisit as IB is not deployed in gnosis - Network.Arbitrum: 1 + Network.Arbitrum: 1, + Network.Optimism: 12658427, } # We convert to checksum address here to prevent minor annoyances. It's worth it. diff --git a/yearn/utils.py b/yearn/utils.py index 23cd953f1..4534c84d8 100644 --- a/yearn/utils.py +++ b/yearn/utils.py @@ -20,6 +20,7 @@ Network.Gnosis: 15_659_482, # gnosis returns "No state available for block 0x3f9e020290502d1d41f4b5519e7d456f0935dea980ec310935206cac8239117e" Network.Fantom: 4_564_024, # fantom returns "missing trie node" before that Network.Arbitrum: 0, + Network.Optimism: 0, } _erc20 = lru_cache(maxsize=None)(interface.ERC20) PREFER_INTERFACE = { From 2957d5d5ff69bb2ee7d0d53f788571a48d442e85 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 6 Oct 2022 21:02:37 -0400 Subject: [PATCH 48/86] feat: add ycrv interface --- interfaces/YCRV.json | 1 + scripts/collect_reports.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 interfaces/YCRV.json diff --git a/interfaces/YCRV.json b/interfaces/YCRV.json new file mode 100644 index 000000000..d4949cd10 --- /dev/null +++ b/interfaces/YCRV.json @@ -0,0 +1 @@ +[{"name":"Transfer","inputs":[{"name":"sender","type":"address","indexed":true},{"name":"receiver","type":"address","indexed":true},{"name":"value","type":"uint256","indexed":false}],"anonymous":false,"type":"event"},{"name":"Mint","inputs":[{"name":"minter","type":"address","indexed":true},{"name":"receiver","type":"address","indexed":true},{"name":"burned","type":"bool","indexed":true},{"name":"value","type":"uint256","indexed":false}],"anonymous":false,"type":"event"},{"name":"Approval","inputs":[{"name":"owner","type":"address","indexed":true},{"name":"spender","type":"address","indexed":true},{"name":"value","type":"uint256","indexed":false}],"anonymous":false,"type":"event"},{"name":"UpdateSweepRecipient","inputs":[{"name":"sweep_recipient","type":"address","indexed":true}],"anonymous":false,"type":"event"},{"stateMutability":"nonpayable","type":"constructor","inputs":[],"outputs":[]},{"stateMutability":"nonpayable","type":"function","name":"transfer","inputs":[{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"outputs":[{"name":"","type":"bool"}]},{"stateMutability":"nonpayable","type":"function","name":"transferFrom","inputs":[{"name":"_from","type":"address"},{"name":"_to","type":"address"},{"name":"_value","type":"uint256"}],"outputs":[{"name":"","type":"bool"}]},{"stateMutability":"nonpayable","type":"function","name":"approve","inputs":[{"name":"_spender","type":"address"},{"name":"_value","type":"uint256"}],"outputs":[{"name":"","type":"bool"}]},{"stateMutability":"nonpayable","type":"function","name":"mint","inputs":[],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"mint","inputs":[{"name":"_amount","type":"uint256"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"mint","inputs":[{"name":"_amount","type":"uint256"},{"name":"_recipient","type":"address"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"burn_to_mint","inputs":[],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"burn_to_mint","inputs":[{"name":"_amount","type":"uint256"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"burn_to_mint","inputs":[{"name":"_amount","type":"uint256"},{"name":"_recipient","type":"address"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"nonpayable","type":"function","name":"set_sweep_recipient","inputs":[{"name":"_proposed_recipient","type":"address"}],"outputs":[]},{"stateMutability":"nonpayable","type":"function","name":"sweep","inputs":[{"name":"_token","type":"address"}],"outputs":[]},{"stateMutability":"nonpayable","type":"function","name":"sweep","inputs":[{"name":"_token","type":"address"},{"name":"_amount","type":"uint256"}],"outputs":[]},{"stateMutability":"nonpayable","type":"function","name":"sweep_yvecrv","inputs":[],"outputs":[]},{"stateMutability":"view","type":"function","name":"name","inputs":[],"outputs":[{"name":"","type":"string"}]},{"stateMutability":"view","type":"function","name":"symbol","inputs":[],"outputs":[{"name":"","type":"string"}]},{"stateMutability":"view","type":"function","name":"decimals","inputs":[],"outputs":[{"name":"","type":"uint8"}]},{"stateMutability":"view","type":"function","name":"balanceOf","inputs":[{"name":"arg0","type":"address"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"view","type":"function","name":"allowance","inputs":[{"name":"arg0","type":"address"},{"name":"arg1","type":"address"}],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"view","type":"function","name":"totalSupply","inputs":[],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"view","type":"function","name":"burned","inputs":[],"outputs":[{"name":"","type":"uint256"}]},{"stateMutability":"view","type":"function","name":"sweep_recipient","inputs":[],"outputs":[{"name":"","type":"address"}]}] \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index d36a0510d..327af195c 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -30,7 +30,7 @@ inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') -alerts_enabled = True #if env == "PROD" or env == "TEST" else False +alerts_enabled = True if env == "PROD" or env == "TEST" else False test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') if env == "TEST": @@ -174,6 +174,7 @@ ) def main(dynamically_find_multi_harvest=False): + ycrv = interface.YCRV('0xFCc5c47bE19d06BF83eB04298b026F81069ff65b') print(f"dynamic multi_harvest detection is enabled: {dynamically_find_multi_harvest}") interval_seconds = 25 From a81f0e96b28412d96fe614ffbff3bd41a41d4ddd Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 6 Oct 2022 21:10:02 -0400 Subject: [PATCH 49/86] feat: add ycrv interface --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 327af195c..cfac12ba6 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -7,7 +7,7 @@ from yearn.cache import memory import pandas as pd from datetime import datetime, timezone -from brownie import chain, web3, Contract, ZERO_ADDRESS +from brownie import chain, web3, Contract, ZERO_ADDRESS, interface from web3._utils.events import construct_event_topic_set from yearn.utils import contract, contract_creation_block from yearn.prices import magic, constants From a776d613df7f0cb4d6f892fec59d18b4a67f82d0 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 7 Oct 2022 14:33:40 -0400 Subject: [PATCH 50/86] feat: unblock ycrv --- scripts/collect_reports.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index cfac12ba6..8ec392d42 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -174,7 +174,9 @@ ) def main(dynamically_find_multi_harvest=False): - ycrv = interface.YCRV('0xFCc5c47bE19d06BF83eB04298b026F81069ff65b') + if chain.id == 1: + ycrv = interface.YCRV('0xFCc5c47bE19d06BF83eB04298b026F81069ff65b') + y = Contract.from_abi('','0xFCc5c47bE19d06BF83eB04298b026F81069ff65b',interface.YCRV.abi) print(f"dynamic multi_harvest detection is enabled: {dynamically_find_multi_harvest}") interval_seconds = 25 From 73bf29870fa2f85c7196dc6d825fa57e2ee25d8e Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 7 Oct 2022 14:40:30 -0400 Subject: [PATCH 51/86] fix: optimism tenderly keyword --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 8ec392d42..9275915ef 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -152,7 +152,7 @@ "STRATEGIST_MULTISIG": "0xea3a15df68fCdBE44Fdb0DB675B2b3A14a148b26", "GOVERNANCE_MULTISIG": "0xF5d9D6133b698cE29567a90Ab35CfB874204B3A7", "EXPLORER_URL": "https://optimistic.etherscan.io/", - "TENDERLY_CHAIN_IDENTIFIER": "optimism", + "TENDERLY_CHAIN_IDENTIFIER": "optimistic", "TELEGRAM_CHAT_ID": os.environ.get('TELEGRAM_CHANNEL_10_PUBLIC'), "DISCORD_CHAN": os.environ.get('DISCORD_CHANNEL_10'), } From 19f27ef2d41c957d9217ee06058bf1e7491ff53c Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 7 Oct 2022 16:01:51 -0400 Subject: [PATCH 52/86] fix: ycrv price --- yearn/prices/curve.py | 91 +++++++++++++++++++++++++++++++++++-------- yearn/prices/magic.py | 11 +++++- yearn/prices/yearn.py | 15 +++++-- yearn/v2/vaults.py | 63 ++++++++++++++++++++++++++---- 4 files changed, 151 insertions(+), 29 deletions(-) diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index da8a44f5f..19fa0ccb9 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -26,7 +26,7 @@ from yearn.decorators import sentry_catch_all, wait_or_exit_after from yearn.events import create_filter, decode_logs -from yearn.exceptions import UnsupportedNetwork +from yearn.exceptions import PriceError, UnsupportedNetwork from yearn.multicall2 import fetch_multicall from yearn.networks import Network from yearn.prices import magic @@ -73,6 +73,9 @@ Network.Arbitrum: { 'address_provider': ADDRESS_PROVIDER, }, + Network.Optimism: { + 'address_provider': ADDRESS_PROVIDER, + } } @@ -103,6 +106,7 @@ def __init__(self) -> None: self.registries = defaultdict(set) # registry -> pools self.factories = defaultdict(set) # factory -> pools self.token_to_pool = dict() # lp_token -> pool + self.coin_to_pools = defaultdict(list) self.address_provider = contract(addrs['address_provider']) self._done = threading.Event() @@ -146,9 +150,17 @@ def watch_events(self) -> None: if event.name == 'PoolAdded': self.registries[event.address].add(event['pool']) lp_token = contract(event.address).get_lp_token(event['pool']) - self.token_to_pool[lp_token] = event['pool'] + pool = event['pool'] + self.token_to_pool[lp_token] = pool + for coin in self.get_coins(pool): + if pool not in self.coin_to_pools[coin]: + self.coin_to_pools[coin].append(pool) elif event.name == 'PoolRemoved': - self.registries[event.address].discard(event['pool']) + pool = event['pool'] + self.registries[event.address].discard(pool) + for coin in self.get_coins(pool): + if pool in self.coin_to_pools[coin]: + self.coin_to_pools[coin].remove(pool) # load metapool and curve v5 factories self.load_factories() @@ -178,6 +190,9 @@ def load_factories(self) -> None: # for metpool factories pool is the same as lp token self.token_to_pool[pool] = pool self.factories[factory].add(pool) + for coin in self.get_coins(pool): + if pool not in self.coin_to_pools[coin]: + self.coin_to_pools[coin].append(pool) # if there are factories that haven't yet been added to the on-chain address provider, # please refer to commit 3f70c4246615017d87602e03272b3ed18d594d3c to see how to add them manually @@ -190,6 +205,9 @@ def load_factories(self) -> None: lp_token = contract(factory).get_token(pool) self.token_to_pool[lp_token] = pool self.factories[factory].add(pool) + for coin in self.get_coins(pool): + if pool not in self.coin_to_pools[coin]: + self.coin_to_pools[coin].append(pool) def get_factory(self, pool: AddressOrContract) -> EthAddress: """ @@ -372,15 +390,16 @@ def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> token = to_address(token) pool = self.get_pool(token) # crypto pools can have different tokens, use slow method - if hasattr(contract(pool), 'price_oracle'): - try: - tvl = self.get_tvl(pool, block=block) - except ValueError: - return None - supply = contract(token).totalSupply(block_identifier=block) / 1e18 - if supply == 0: - return 0 - return tvl / supply + try: + tvl = self.get_tvl(pool, block=block) + except ValueError: + tvl = 0 + supply = contract(token).totalSupply(block_identifier=block) / 1e18 + if supply == 0: + if tvl > 0: + raise ValueError('curve pool has balance but no supply') + return 0 + return tvl / supply # approximate by using the most common base token we find coins = self.get_underlying_coins(pool) @@ -388,11 +407,51 @@ def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> coin = (set(coins) & BASIC_TOKENS).pop() except KeyError: coin = coins[0] + + try: + virtual_price = self.get_virtual_price(pool, block) + if virtual_price: + return virtual_price * magic.get_price(coin, block) + return sum([balance * magic.get_price(coin, block) for coin, balance in self.get_balances(pool, block).items()]) + except ValueError as e: + logger.warn(f'ValueError: {str(e)}') + if 'No data was returned' in str(e): + return None + else: + raise + + def get_coin_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: + + # Get the pool + if len(self.coin_to_pools[token]) == 1: + pool = self.coin_to_pools[token][0] + else: + # TODO: handle this sitch if necessary + return + + # Get the index for `token` + coins = self.get_coins(pool) + token_in_ix = [i for i, coin in enumerate(coins) if coin == token][0] + amount_in = 10 ** contract(str(token)).decimals() + if len(coins) == 2: + # this works for most typical metapools + token_out_ix = 0 if token_in_ix == 1 else 1 if token_in_ix == 0 else None + else: + # TODO: handle this sitch if necessary + return None - virtual_price = self.get_virtual_price(pool, block) + # Get the price for `token` using the selected pool. + try: + dy = contract(pool).get_dy(token_in_ix, token_out_ix, amount_in, block_identifier = block) + except: + return None - if virtual_price: - return virtual_price * magic.get_price(coin, block) + token_out = contract(coins[token_out_ix]) + amount_out = dy / 10 ** token_out.decimals() + try: + return amount_out * magic.get_price(token_out, block = block) + except PriceError: + return None def calculate_boost(self, gauge: Contract, addr: Address, block: Optional[Block] = None) -> Dict[str,float]: results = fetch_multicall( @@ -479,4 +538,4 @@ def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, block: Opt try: curve = CurveRegistry() except UnsupportedNetwork: - pass + pass \ No newline at end of file diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index 04c083b13..cb2f36e9d 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -122,7 +122,8 @@ def find_price( uniswaps ] for market in markets: - if price: + # break on the first numerical price + if price or price == 0: break if not market: continue @@ -139,6 +140,9 @@ def find_price( price, underlying = price logger.debug("peel %s %s", price, underlying) return price * get_price(underlying, block=block) + + if price is None and token in curve.curve.coin_to_pools: + price = curve.curve.get_coin_price(token, block = block) if price is None and return_price_during_vault_downtime: for incident in INCIDENTS[token]: @@ -149,6 +153,9 @@ def find_price( logger.error(f"failed to get price for {_describe_err(token, block)}") raise PriceError(f'could not fetch price for {_describe_err(token, block)}') + if price == 0: + logger.warn("Price is 0 for token %s at block %d", token, block) + return price @@ -170,4 +177,4 @@ def _describe_err(token: Address, block: Optional[Block]) -> str: if symbol: return f"{symbol} {token} on {Network(chain.id).name} at {block}" - return f"malformed token {token} on {Network(chain.id).name} at {block}" + return f"malformed token {token} on {Network(chain.id).name} at {block}" \ No newline at end of file diff --git a/yearn/prices/yearn.py b/yearn/prices/yearn.py index 7ef51d607..8c70f8145 100644 --- a/yearn/prices/yearn.py +++ b/yearn/prices/yearn.py @@ -28,6 +28,9 @@ 'v2': '0x57AA88A0810dfe3f9b71a9b179Dd8bF5F956C46A', 'ib': '0xf900ea42c55D165Ca5d5f50883CddD352AE48F40', }, + Network.Optimism: { + 'v2': '0xBcfCA75fF12E2C1bB404c2C216DBF901BE047690', + }, } @@ -72,31 +75,37 @@ def get_price(self, token: Address, block: Optional[Block] = None) -> Optional[f vault = contract(token) if hasattr(vault, 'pricePerShare'): try: - share_price, underlying, decimals = fetch_multicall( + share_price, underlying, decimals, supply = fetch_multicall( [vault, 'pricePerShare'], [vault, 'token'], [vault, 'decimals'], + [vault, 'totalSupply'], block=block, require_success=True, ) except MulticallError: return None else: + if supply == 0: + return 0 return [share_price / 10 ** decimals, underlying] # v1 vaults use getPricePerFullShare scaled to 18 decimals if hasattr(vault, 'getPricePerFullShare'): try: - share_price, underlying = fetch_multicall( + share_price, underlying, supply = fetch_multicall( [vault, 'getPricePerFullShare'], [vault, 'token'], + [vault, 'totalSupply'], block=block, require_success=True, ) except MulticallError: return None else: + if supply == 0: + return 0 return [share_price / 1e18, underlying] -yearn_lens = YearnLens(force_init=True) +yearn_lens = YearnLens(force_init=True) \ No newline at end of file diff --git a/yearn/v2/vaults.py b/yearn/v2/vaults.py index 871008137..ab4bcfcbf 100644 --- a/yearn/v2/vaults.py +++ b/yearn/v2/vaults.py @@ -2,7 +2,7 @@ import re import threading import time -from typing import List +from typing import Dict, List from brownie import chain from eth_utils import encode_hex, event_abi_to_log_topic @@ -15,10 +15,13 @@ from yearn.multicall2 import fetch_multicall from yearn.prices import magic from yearn.prices.curve import curve +from yearn.special import Ygov +from yearn.typing import Address from yearn.utils import safe_views, contract from yearn.v2.strategies import Strategy from yearn.exceptions import PriceError from yearn.decorators import sentry_catch_all, wait_or_exit_after +from yearn.networks import Network VAULT_VIEWS_SCALED = [ "totalAssets", @@ -49,8 +52,8 @@ class Vault: def __init__(self, vault, api_version=None, token=None, registry=None, watch_events_forever=True): - self._strategies = {} - self._revoked = {} + self._strategies: Dict[Address, Strategy] = {} + self._revoked: Dict[Address, Strategy] = {} self._reports = [] self.vault = vault self.api_version = api_version @@ -87,6 +90,10 @@ def __eq__(self, other): if isinstance(other, str): return self.vault == other + + # Needed for transactions_exporter + if isinstance(other, Ygov): + return False raise ValueError("Vault is only comparable with [Vault, str]") @@ -107,6 +114,12 @@ def revoked_strategies(self) -> List[Strategy]: self.load_strategies() return list(self._revoked.values()) + @property + def reports(self): + # strategy reports are loaded at the same time as other vault strategy events + self.load_strategies() + return self._reports + @property def is_endorsed(self): if not self.registry: @@ -132,8 +145,8 @@ def load_harvests(self): def watch_events(self): start = time.time() self.log_filter = create_filter(str(self.vault), topics=self._topics) + logs = self.log_filter.get_all_entries() while True: - logs = self.log_filter.get_new_entries() events = decode_logs(logs) self.process_events(events) if not self._done.is_set(): @@ -143,15 +156,27 @@ def watch_events(self): return time.sleep(300) + # read new logs at end of loop + logs = self.log_filter.get_new_entries() + + def process_events(self, events): for event in events: + # some issues during the migration of this strat prevented it from being verified so we skip it here... + if chain.id == Network.Optimism: + failed_migration = False + for key in ["newVersion", "oldVersion", "strategy"]: + failed_migration |= (key in event and event[key] == "0x4286a40EB3092b0149ec729dc32AD01942E13C63") + if failed_migration: + continue + if event.name == "StrategyAdded": strategy_address = event["strategy"] logger.debug("%s strategy added %s", self.name, strategy_address) try: self._strategies[strategy_address] = Strategy(strategy_address, self, self._watch_events_forever) except ValueError: - print(f"Error loading strategy {strategy_address}") + logger.error(f"Error loading strategy {strategy_address}") pass elif event.name == "StrategyRevoked": logger.debug("%s strategy revoked %s", self.name, event["strategy"]) @@ -181,7 +206,7 @@ def describe(self, block=None): for strategy in self.strategies: info["strategies"][strategy.unique_name] = strategy.describe(block=block) - info["token price"] = magic.get_price(self.token, block=block) + info["token price"] = magic.get_price(self.token, block=block) if info['totalSupply'] > 0 else 0 if "totalAssets" in info: info["tvl"] = info["token price"] * info["totalAssets"] @@ -192,7 +217,7 @@ def describe(self, block=None): return info def apy(self, samples: ApySamples): - if curve and curve.get_pool(self.token.address): + if self._needs_curve_simple(): return apy.curve.simple(self, samples) elif Version(self.api_version) >= Version("0.3.2"): return apy.v2.average(self, samples) @@ -206,4 +231,26 @@ def tvl(self, block=None): except PriceError: price = None tvl = total_assets * price / 10 ** self.vault.decimals(block_identifier=block) if price else None - return Tvl(total_assets, price, tvl) \ No newline at end of file + return Tvl(total_assets, price, tvl) + + + def _needs_curve_simple(self): + # not able to calculate gauge weighting on chains other than mainnet + curve_simple_excludes = { + Network.Mainnet: [ + "0x3D27705c64213A5DcD9D26880c1BcFa72d5b6B0E", + ], + Network.Fantom: [ + "0xCbCaF8cB8cbeAFA927ECEE0c5C56560F83E9B7D9", + "0xA97E7dA01C7047D6a65f894c99bE8c832227a8BC", + ], + Network.Arbitrum: [ + "0x239e14A19DFF93a17339DCC444f74406C17f8E67", + "0x1dBa7641dc69188D6086a73B972aC4bda29Ec35d", + ] + } + needs_simple = True + if chain.id in curve_simple_excludes: + needs_simple = self.vault.address not in curve_simple_excludes[chain.id] + + return needs_simple and curve and curve.get_pool(self.token.address) \ No newline at end of file From 130be20822eef0e0f5e6b0484d713c88f7c117ad Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 17 Oct 2022 22:19:37 -0400 Subject: [PATCH 53/86] feat: treasury --- scripts/collect_reports.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 9275915ef..fea25902e 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -31,6 +31,7 @@ invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') alerts_enabled = True if env == "PROD" or env == "TEST" else False +alerts_enabled = False test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') if env == "TEST": @@ -325,13 +326,14 @@ def handle_event(event, multi_harvest): crv = '0xD533a949740bb3306d119CC777fa900bA034cd52' yvecrv = '0xc5bDdf9843308380375a611c18B50Fb9341f502A' voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' + treasury = '0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde' token_abi = Contract(crv).abi crv_token = web3.eth.contract(crv, abi=token_abi) decoded_events = crv_token.events.Transfer().processReceipt(tx) r.keep_crv = 0 for tfr in decoded_events: _from, _to, _val = tfr.args.values() - if tfr.address == crv and _from == r.strategy_address and _to == voter: + if tfr.address == crv and _from == r.strategy_address and (_to == voter or _to == treasury): r.keep_crv = _val / 1e18 r.crv_price_usd = magic.get_price(crv, r.block) r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd From 5bd2fe225cf537f75a2a69365958ea2e7bc2e49a Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 17 Oct 2022 22:40:46 -0400 Subject: [PATCH 54/86] fix: pricing --- yearn/decorators.py | 13 ++++- yearn/prices/curve.py | 1 + yearn/utils.py | 124 ++++++++++++++++++++++++++++++++---------- 3 files changed, 106 insertions(+), 32 deletions(-) diff --git a/yearn/decorators.py b/yearn/decorators.py index 409583f9c..8be231071 100644 --- a/yearn/decorators.py +++ b/yearn/decorators.py @@ -1,13 +1,20 @@ import _thread import functools +import logging + +import sentry_sdk + +logger = logging.getLogger(__name__) def sentry_catch_all(func): @functools.wraps(func) def wrap(self): try: func(self) - except: + except Exception as e: + sentry_sdk.capture_exception(e) self._has_exception = True + self._exception = e self._done.set() raise return wrap @@ -18,6 +25,7 @@ def wait_or_exit_before(func): def wrap(self): self._done.wait() if self._has_exception: + logger.error(self._exception) _thread.interrupt_main() return func(self) return wrap @@ -29,5 +37,6 @@ def wrap(self): func(self) self._done.wait() if self._has_exception: + logger.error(self._exception) _thread.interrupt_main() - return wrap + return wrap \ No newline at end of file diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index 19fa0ccb9..dee62f6bd 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -87,6 +87,7 @@ class Ids(IntEnum): Fee_Distributor = 4 CryptoSwap_Registry = 5 CryptoPool_Factory = 6 + MetaFactory = 7 class CurveRegistry(metaclass=Singleton): diff --git a/yearn/utils.py b/yearn/utils.py index 4534c84d8..47443d460 100644 --- a/yearn/utils.py +++ b/yearn/utils.py @@ -1,20 +1,21 @@ +import json import logging import threading -import json from functools import lru_cache +from time import sleep from typing import List import eth_retry from brownie import Contract, chain, convert, interface, web3 -from web3 import Web3 -from brownie.network.contract import _resolve_address, _fetch_from_explorer -from brownie.exceptions import CompilerError +from brownie.network.contract import _fetch_from_explorer, _resolve_address from yearn.cache import memory from yearn.exceptions import ArchiveNodeRequired, NodeNotSynced from yearn.networks import Network -from yearn.typing import Address, AddressOrContract +from yearn.typing import AddressOrContract + logger = logging.getLogger(__name__) + BINARY_SEARCH_BARRIER = { Network.Mainnet: 0, Network.Gnosis: 15_659_482, # gnosis returns "No state available for block 0x3f9e020290502d1d41f4b5519e7d456f0935dea980ec310935206cac8239117e" @@ -22,12 +23,15 @@ Network.Arbitrum: 0, Network.Optimism: 0, } + _erc20 = lru_cache(maxsize=None)(interface.ERC20) + PREFER_INTERFACE = { Network.Arbitrum: { "0x2f2a2543B76A4166549F7aaB2e75Bef0aefC5B0f": _erc20, # empty ABI for WBTC when compiling the contract } } + def safe_views(abi: List) -> List[str]: return [ item["name"] @@ -37,6 +41,8 @@ def safe_views(abi: List) -> List[str]: and not item["inputs"] and all(x["type"] in ["uint256", "bool"] for x in item["outputs"]) ] + + @memory.cache() def get_block_timestamp(height): """ @@ -49,20 +55,37 @@ def get_block_timestamp(height): except: pass return chain[height].timestamp + + @memory.cache() -def closest_block_after_timestamp(timestamp): +def closest_block_after_timestamp(timestamp: int, wait_for: bool = False) -> int: + """ + Set `wait_for = True` to make this work for future `timestamp` values. + """ + + while wait_for: + try: + return closest_block_after_timestamp(timestamp) + except IndexError: + sleep(.2) + logger.debug('closest block after timestamp %d', timestamp) height = chain.height lo, hi = 0, height + while hi - lo > 1: mid = lo + (hi - lo) // 2 if get_block_timestamp(mid) > timestamp: hi = mid else: lo = mid + if get_block_timestamp(hi) < timestamp: raise IndexError('timestamp is in the future') + return hi + + def get_code(address, block=None): try: return web3.eth.get_code(address, block_identifier=block) @@ -70,6 +93,8 @@ def get_code(address, block=None): if isinstance(exc.args[0], dict) and 'missing trie node' in exc.args[0]['message']: raise ArchiveNodeRequired('querying historical state requires an archive node') raise exc + + @memory.cache() def contract_creation_block(address: AddressOrContract) -> int: """ @@ -78,13 +103,16 @@ def contract_creation_block(address: AddressOrContract) -> int: """ logger.info("contract creation block %s", address) address = convert.to_address(address) + barrier = BINARY_SEARCH_BARRIER[chain.id] lo = barrier hi = end = chain.height + if hi == 0: raise NodeNotSynced(f''' `chain.height` returns 0 on your node, which means it is not fully synced. You can only use contract_creation_block on a fully synced node.''') + while hi - lo > 1: mid = lo + (hi - lo) // 2 try: @@ -102,65 +130,91 @@ def contract_creation_block(address: AddressOrContract) -> int: hi = mid else: lo = mid + # only happens on fantom if hi == barrier + 1: logger.warning('could not determine creation block for a contract deployed prior to barrier') return 0 + return hi if hi != end else None + + class Singleton(type): def __init__(self, *args, **kwargs): self.__instance = None super().__init__(*args, **kwargs) + def __call__(self, *args, **kwargs): if self.__instance is None: self.__instance = super().__call__(*args, **kwargs) return self.__instance else: return self.__instance + + # cached Contract instance, saves about 20ms of init time _contract_lock = threading.Lock() _contract = lru_cache(maxsize=None)(Contract) + @eth_retry.auto_retry -def contract(address: Address) -> Contract: +def contract(address: AddressOrContract) -> Contract: with _contract_lock: - address = web3.toChecksumAddress(address) + address = web3.toChecksumAddress(str(address)) if chain.id in PREFER_INTERFACE: if address in PREFER_INTERFACE[chain.id]: _interface = PREFER_INTERFACE[chain.id][address] i = _interface(address) return _squeeze(i) - failed_attempts = 0 - while True: - try: - c = _contract(address) - return _squeeze(c) - except (AssertionError, CompilerError) as e: - if failed_attempts == 10: - raise - logger.warning(e) - Contract.remove_deployment(address) - failed_attempts += 1 + + # autofetch-sources: false + # Try to fetch the contract from the local sqlite db. + try: + c = _contract(address) + # If we don't already have the contract in the db, we'll try to fetch it from the explorer. + except ValueError as e: + c = _resolve_proxy(address) + + # Lastly, get rid of unnecessary memory-hog properties + return _squeeze(c) +# These tokens have trouble when resolving the implementation via the chain. +FORCE_IMPLEMENTATION = { + Network.Mainnet: { + "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48": "0xa2327a938Febf5FEC13baCFb16Ae10EcBc4cbDCF", # USDC as of 2022-08-10 + }, +}.get(chain.id, {}) + @eth_retry.auto_retry def _resolve_proxy(address): - data = _fetch_from_explorer(address, "getsourcecode", False) - name = data["result"][0]["ContractName"] - abi = json.loads(data["result"][0]["ABI"]) + name, abi, implementation = _extract_abi_data(address) as_proxy_for = None + if address in FORCE_IMPLEMENTATION: + implementation = FORCE_IMPLEMENTATION[address] + name, abi, _ = _extract_abi_data(implementation) + return Contract.from_abi(name, address, abi) + # always check for an EIP1967 proxy - https://eips.ethereum.org/EIPS/eip-1967 implementation_eip1967 = web3.eth.get_storage_at( address, int(web3.keccak(text="eip1967.proxy.implementation").hex(), 16) - 1 ) # always check for an EIP1822 proxy - https://eips.ethereum.org/EIPS/eip-1822 implementation_eip1822 = web3.eth.get_storage_at(address, web3.keccak(text="PROXIABLE")) + + # Just leave this code where it is for a helpful debugger as needed. + if address == "": + raise Exception( + f"""implementation: {implementation} + implementation_eip1967: {len(implementation_eip1967)} {implementation_eip1967} + implementation_eip1822: {len(implementation_eip1822)} {implementation_eip1822}""") + if len(implementation_eip1967) > 0 and int(implementation_eip1967.hex(), 16): as_proxy_for = _resolve_address(implementation_eip1967[-20:]) elif len(implementation_eip1822) > 0 and int(implementation_eip1822.hex(), 16): as_proxy_for = _resolve_address(implementation_eip1822[-20:]) - elif data["result"][0].get("Implementation"): + elif implementation: # for other proxy patterns, we only check if etherscan indicates # the contract is a proxy. otherwise we could have a false positive # if there is an `implementation` method on a regular contract. @@ -171,26 +225,36 @@ def _resolve_proxy(address): as_proxy_for = c.implementation.call() except Exception: # if that fails, fall back to the address provided by etherscan - as_proxy_for = _resolve_address(data["result"][0]["Implementation"]) + as_proxy_for = _resolve_address(implementation) if as_proxy_for: - data = _fetch_from_explorer(as_proxy_for, "getsourcecode", False) - name = data["result"][0]["ContractName"] - abi = json.loads(data["result"][0]["ABI"]) - return Contract.from_abi(name, as_proxy_for, abi) - else: - return _contract(address) + name, abi, _ = _extract_abi_data(as_proxy_for) + return Contract.from_abi(name, address, abi) + +def _extract_abi_data(address): + data = _fetch_from_explorer(address, "getsourcecode", False) + is_verified = bool(data["result"][0].get("SourceCode")) + if not is_verified: + raise ValueError(f"Contract source code not verified: {address}") + name = data["result"][0]["ContractName"] + abi = json.loads(data["result"][0]["ABI"]) + implementation = data["result"][0].get("Implementation") + return name, abi, implementation @lru_cache(maxsize=None) def is_contract(address: str) -> bool: '''checks to see if the input address is a contract''' return web3.eth.get_code(address) not in ['0x',b''] + + def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): yield lst[i:i + n] + + def _squeeze(it): """ Reduce the contract size in RAM significantly. """ for k in ["ast", "bytecode", "coverageMap", "deployedBytecode", "deployedSourceMap", "natspec", "opcodes", "pcMap"]: From fbfed427401903b206409589942eccd46a5b4329 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 18 Oct 2022 11:01:22 -0400 Subject: [PATCH 55/86] chore: enable alerts --- scripts/collect_reports.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index fea25902e..f45ad8b25 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -31,7 +31,6 @@ invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') alerts_enabled = True if env == "PROD" or env == "TEST" else False -alerts_enabled = False test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') if env == "TEST": From 18b6e90dfcf84669db1e93923a48133299bbf0ae Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 24 Oct 2022 18:49:56 -0400 Subject: [PATCH 56/86] feat: properly display stycrv apr --- scripts/collect_reports.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index f45ad8b25..930cc3559 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -423,12 +423,21 @@ def compute_apr(report, previous_report): seconds_between_reports = report.timestamp - previous_report.timestamp pre_fee_apr = 0 post_fee_apr = 0 - if int(previous_report.total_debt) == 0 or seconds_between_reports == 0: - return 0, 0 - else: - pre_fee_apr = report.gain / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + + if report.vault_address == '0x27B5739e22ad9033bcBf192059122d163b60349D': + vault = Contract(report.vault_address) + if vault.totalAssets() == 0 or seconds_between_reports == 0: + return 0, 0 + pre_fee_apr = report.gain / int(vault.totalAssets()/10**vault.decimals()) * (SECONDS_IN_A_YEAR / seconds_between_reports) if report.gain_post_fees != 0: - post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + post_fee_apr = report.gain_post_fees / int(vault.totalAssets()/10**vault.decimals()) * (SECONDS_IN_A_YEAR / seconds_between_reports) + else: + if int(previous_report.total_debt) == 0 or seconds_between_reports == 0: + return 0, 0 + else: + pre_fee_apr = report.gain / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) + if report.gain_post_fees != 0: + post_fee_apr = report.gain_post_fees / int(previous_report.total_debt) * (SECONDS_IN_A_YEAR / seconds_between_reports) return pre_fee_apr, post_fee_apr def parse_fees(tx, vault_address, strategy_address, decimals, gain, vault_version): From a07e0398e2c05a35e1f8db89b7316f4dcff39652 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 27 Oct 2022 18:37:39 -0400 Subject: [PATCH 57/86] feat add inverse vault --- scripts/collect_reports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 930cc3559..f9a6c5d84 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -60,6 +60,7 @@ INVERSE_PRIVATE_VAULTS = [ "0xD4108Bb1185A5c30eA3f4264Fd7783473018Ce17", "0x67B9F46BCbA2DF84ECd41cC6511ca33507c9f4E9", + "0xd395DEC4F1733ff09b750D869eEfa7E0D37C3eE6", ] CHAIN_VALUES = { From f14e48a0694e4a9a9c8e8b5536ade8e86678c23d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 31 Oct 2022 22:04:50 -0400 Subject: [PATCH 58/86] feat: add opti to daily report --- scripts/daily_harvest_report.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/daily_harvest_report.py b/scripts/daily_harvest_report.py index 658bc5a22..919a1dc10 100644 --- a/scripts/daily_harvest_report.py +++ b/scripts/daily_harvest_report.py @@ -8,6 +8,7 @@ telegram_key = os.environ.get('HARVEST_TRACKER_BOT_KEY') mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') +opti_public_channel = os.environ.get('TELEGRAM_CHANNEL_10_PUBLIC') dev_channel = os.environ.get('TELEGRAM_CHANNEL_DEV') discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') discord_ftm = os.environ.get('DISCORD_CHANNEL_250') @@ -27,6 +28,16 @@ "txn_cost_usd": 0, "message": "" }, + 10: { + "network_symbol": "OPT", + "network_name": "Optimism", + "telegram_channel": opti_public_channel, + "profit_usd": 0, + "num_harvests": 0, + "txn_cost_eth": 0, + "txn_cost_usd": 0, + "message": "" + }, 250: { "network_symbol": "FTM", "network_name": "Fantom", From 0ceadca7b74efe9d1e210e90f39da619b2496e94 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 9 Dec 2022 10:59:27 -0500 Subject: [PATCH 59/86] fix: ycrv pricing error due to spammy pools --- yearn/prices/curve.py | 52 +++++++++++++++++++-------------------- yearn/prices/magic.py | 5 ++++ yearn/prices/synthetix.py | 8 +++--- 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/yearn/prices/curve.py b/yearn/prices/curve.py index dee62f6bd..c57efc387 100644 --- a/yearn/prices/curve.py +++ b/yearn/prices/curve.py @@ -332,7 +332,7 @@ def get_decimals(self, pool: AddressOrContract) -> List[int]: return [dec for dec in decimals if dec != 0] - def get_balances(self, pool: AddressOrContract, block: Optional[Block] = None) -> Dict[EthAddress,float]: + def get_balances(self, pool: AddressOrContract, block: Optional[Block] = None, should_raise_err: bool = True) -> Optional[Dict[EthAddress,float]]: """ Get {token: balance} of liquidity in the pool. """ @@ -358,7 +358,9 @@ def get_balances(self, pool: AddressOrContract, block: Optional[Block] = None) - ) if not any(balances): - raise ValueError(f'could not fetch balances {pool} at {block}') + if should_raise_err: + raise ValueError(f'could not fetch balances {pool} at {block}') + return None return { coin: balance / 10 ** dec @@ -402,33 +404,29 @@ def get_price(self, token: AddressOrContract, block: Optional[Block] = None) -> return 0 return tvl / supply - # approximate by using the most common base token we find - coins = self.get_underlying_coins(pool) - try: - coin = (set(coins) & BASIC_TOKENS).pop() - except KeyError: - coin = coins[0] - - try: - virtual_price = self.get_virtual_price(pool, block) - if virtual_price: - return virtual_price * magic.get_price(coin, block) - return sum([balance * magic.get_price(coin, block) for coin, balance in self.get_balances(pool, block).items()]) - except ValueError as e: - logger.warn(f'ValueError: {str(e)}') - if 'No data was returned' in str(e): - return None - else: - raise - def get_coin_price(self, token: AddressOrContract, block: Optional[Block] = None) -> Optional[float]: - # Get the pool - if len(self.coin_to_pools[token]) == 1: - pool = self.coin_to_pools[token][0] - else: - # TODO: handle this sitch if necessary + # Select the most appropriate pool + pools = self.coin_to_pools[token] + if not pools: return + elif len(pools) == 1: + pool = pools[0] + else: + # We need to find the pool with the deepest liquidity + balances = [self.get_balances(pool, block, should_raise_err=False) for pool in pools] + balances = [bal for bal in balances if bal] + deepest_pool, deepest_bal = None, 0 + for pool, pool_bals in zip(pools, balances): + if isinstance(pool_bals, Exception): + if str(pool_bals).startswith("could not fetch balances"): + continue + raise pool_bals + for _token, bal in pool_bals.items(): + if _token == token and bal > deepest_bal: + deepest_pool = pool + deepest_bal = bal + pool = deepest_pool # Get the index for `token` coins = self.get_coins(pool) @@ -539,4 +537,4 @@ def calculate_apy(self, gauge: Contract, lp_token: AddressOrContract, block: Opt try: curve = CurveRegistry() except UnsupportedNetwork: - pass \ No newline at end of file + pass diff --git a/yearn/prices/magic.py b/yearn/prices/magic.py index cb2f36e9d..02435eda8 100644 --- a/yearn/prices/magic.py +++ b/yearn/prices/magic.py @@ -110,6 +110,9 @@ def find_price( # no liquidity for curve pool (yvecrv-f) -> return 0 elif token == "0x7E46fd8a30869aa9ed55af031067Df666EfE87da" and block < 14987514: return 0 + # no continuous price data before 2020-10-10 + elif token == "0xEB4C2781e4ebA804CE9a9803C67d0893436bB27D" and block < 11024342: + return 0 markets = [ chainlink, @@ -142,11 +145,13 @@ def find_price( return price * get_price(underlying, block=block) if price is None and token in curve.curve.coin_to_pools: + logger.debug(f'Curve.get_coin_price -> {price}') price = curve.curve.get_coin_price(token, block = block) if price is None and return_price_during_vault_downtime: for incident in INCIDENTS[token]: if incident['start'] <= block <= incident['end']: + logger.debug(f"incidents -> {price}") return incident['result'] if price is None: diff --git a/yearn/prices/synthetix.py b/yearn/prices/synthetix.py index 690361c9a..6a1cb914a 100644 --- a/yearn/prices/synthetix.py +++ b/yearn/prices/synthetix.py @@ -27,14 +27,14 @@ def __init__(self) -> None: self.synths = self.load_synths() logger.info(f'loaded {len(self.synths)} synths') - @lru_cache(maxsize=None) - def get_address(self, name: str) -> EthAddress: + @lru_cache(maxsize=128) + def get_address(self, name: str, block: Block = None) -> EthAddress: """ Get contract from Synthetix registry. See also https://docs.synthetix.io/addresses/ """ address_resolver = contract(addresses[chain.id]) - address = address_resolver.getAddress(encode_single('bytes32', name.encode())) + address = address_resolver.getAddress(encode_single('bytes32', name.encode()), block_identifier=block) proxy = contract(address) return contract(proxy.target()) if hasattr(proxy, 'target') else proxy @@ -73,9 +73,9 @@ def get_price(self, token: Address, block: Optional[Block] = None) -> Optional[f """ Get a price of a synth in dollars. """ - rates = self.get_address('ExchangeRates') key = self.get_currency_key(token) try: + rates = self.get_address('ExchangeRates', block=block) return rates.rateForCurrency(key, block_identifier=block) / 1e18 except ValueError: return None From 8e91afbbed7ee307260ae94698e7a73e5034f4f9 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 23 Jan 2023 14:29:17 -0500 Subject: [PATCH 60/86] feat: support factory vaults --- scripts/collect_reports.py | 39 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index f9a6c5d84..79534da96 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -70,9 +70,9 @@ "EMOJI": "🇪🇹", "START_DATE": datetime(2020, 2, 12, tzinfo=timezone.utc), "START_BLOCK": 11563389, - "REGISTRY_ADDRESS": "0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804", + "REGISTRY_ADDRESSES": ["0x50c1a2eA0a861A967D9d0FFE2AE4012c2E053804","0xaF1f5e1c19cB68B30aAD73846eFfDf78a5863319"], "REGISTRY_DEPLOY_BLOCK": 12045555, - "REGISTRY_HELPER_ADDRESS": "0x52CbF68959e082565e7fd4bBb23D9Ccfb8C8C057", + "REGISTRY_HELPER_ADDRESS": "0xec85C894be162268c834b784CC232398E3E89A12", "LENS_ADDRESS": "0x5b4F3BE554a88Bd0f8d8769B9260be865ba03B4a", "LENS_DEPLOY_BLOCK": 12707450, "VAULT_ADDRESS030": "0x19D3364A399d251E894aC732651be8B0E4e85001", @@ -94,7 +94,7 @@ "EMOJI": "👻", "START_DATE": datetime(2021, 4, 30, tzinfo=timezone.utc), "START_BLOCK": 18450847, - "REGISTRY_ADDRESS": "0x727fe1759430df13655ddb0731dE0D0FDE929b04", + "REGISTRY_ADDRESSES": ["0x727fe1759430df13655ddb0731dE0D0FDE929b04"], "REGISTRY_DEPLOY_BLOCK": 18455565, "REGISTRY_HELPER_ADDRESS": "0x8CC45f739104b3Bdb98BFfFaF2423cC0f817ccc1", "REGISTRY_HELPER_DEPLOY_BLOCK": 18456459, @@ -119,7 +119,7 @@ "EMOJI": "🤠", "START_DATE": datetime(2021, 9, 14, tzinfo=timezone.utc), "START_BLOCK": 4841854, - "REGISTRY_ADDRESS": "0x3199437193625DCcD6F9C9e98BDf93582200Eb1f", + "REGISTRY_ADDRESSES": ["0x3199437193625DCcD6F9C9e98BDf93582200Eb1f"], "REGISTRY_DEPLOY_BLOCK": 12045555, "REGISTRY_HELPER_ADDRESS": "0x237C3623bed7D115Fc77fEB08Dd27E16982d972B", "LENS_ADDRESS": "0xcAd10033C86B0C1ED6bfcCAa2FF6779938558E9f", @@ -141,7 +141,7 @@ "EMOJI": "🔴", "START_DATE": datetime(2022, 8, 6, tzinfo=timezone.utc), "START_BLOCK": 24097341, - "REGISTRY_ADDRESS": "0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0", + "REGISTRY_ADDRESSES": ["0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0"], "REGISTRY_DEPLOY_BLOCK": 18097341, "REGISTRY_HELPER_ADDRESS": "0x0983b4899a3168c2509569faf1e4e75c57b4aba6", "LENS_ADDRESS": "0xD3A93C794ee2798D8f7906493Cd3c2A835aa0074", @@ -182,7 +182,8 @@ def main(dynamically_find_multi_harvest=False): interval_seconds = 25 last_reported_block, last_reported_block030 = last_harvest_block() - + last_reported_block = 16215519 + last_reported_block030 = 16215519 print("latest block (v0.3.1+ API)",last_reported_block) print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) if chain.id == 1: @@ -499,19 +500,25 @@ def get_vault_endorsement_block(vault_address): return block except KeyError: pass - registry = contract(CHAIN_VALUES[chain.id]["REGISTRY_ADDRESS"]) + registries = CHAIN_VALUES[chain.id]["REGISTRY_ADDRESSES"] height = chain.height lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height - while hi - lo > 1: - mid = lo + (hi - lo) // 2 - try: - num_vaults = registry.numVaults(token, block_identifier=mid) - if registry.vaults(token, num_vaults-1, block_identifier=mid) == vault_address: - hi = mid - else: + + for r in registries: + r = contract(r) + while hi - lo > 1: + mid = lo + (hi - lo) // 2 + try: + num_vaults = r.numVaults(token, block_identifier=mid) + if r.vaults(token, num_vaults-1, block_identifier=mid) == vault_address: + hi = mid + else: + lo = mid + except: lo = mid - except: - lo = mid + if hi < height: + print(f'🍿REGSITERD AT BLOCK {mid}') + return mid return hi def normalize_event_values(vals, decimals): From 95a1d3b1b7ddaf991286e4842eb750c763afc1ff Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 23 Jan 2023 14:30:01 -0500 Subject: [PATCH 61/86] feat: support factory vaults --- scripts/collect_reports.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 79534da96..db0ecfa38 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -182,8 +182,6 @@ def main(dynamically_find_multi_harvest=False): interval_seconds = 25 last_reported_block, last_reported_block030 = last_harvest_block() - last_reported_block = 16215519 - last_reported_block030 = 16215519 print("latest block (v0.3.1+ API)",last_reported_block) print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) if chain.id == 1: From b6a8d50792e23d0a781eba3d10618beeecee55c6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 23 Jan 2023 17:05:04 -0500 Subject: [PATCH 62/86] feat: contract getter --- interfaces/ICurveBoostedStrat.json | 1 + scripts/collect_reports.py | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 interfaces/ICurveBoostedStrat.json diff --git a/interfaces/ICurveBoostedStrat.json b/interfaces/ICurveBoostedStrat.json new file mode 100644 index 000000000..178560caf --- /dev/null +++ b/interfaces/ICurveBoostedStrat.json @@ -0,0 +1 @@ +[{"inputs":[{"internalType":"address","name":"_vault","type":"address"},{"internalType":"address","name":"_tradeFactory","type":"address"},{"internalType":"address","name":"_proxy","type":"address"},{"internalType":"address","name":"_gauge","type":"address"}],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"address","name":"clone","type":"address"}],"name":"Cloned","type":"event"},{"anonymous":false,"inputs":[],"name":"EmergencyExitEnabled","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"bool","name":"triggerState","type":"bool"}],"name":"ForcedHarvestTrigger","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint256","name":"profit","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"loss","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"debtPayment","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"debtOutstanding","type":"uint256"}],"name":"Harvested","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"bool","name":"","type":"bool"}],"name":"SetDoHealthCheck","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"","type":"address"}],"name":"SetHealthCheck","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"baseFeeOracle","type":"address"}],"name":"UpdatedBaseFeeOracle","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint256","name":"creditThreshold","type":"uint256"}],"name":"UpdatedCreditThreshold","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"newKeeper","type":"address"}],"name":"UpdatedKeeper","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint256","name":"delay","type":"uint256"}],"name":"UpdatedMaxReportDelay","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"string","name":"metadataURI","type":"string"}],"name":"UpdatedMetadataURI","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"uint256","name":"delay","type":"uint256"}],"name":"UpdatedMinReportDelay","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"rewards","type":"address"}],"name":"UpdatedRewards","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"address","name":"newStrategist","type":"address"}],"name":"UpdatedStrategist","type":"event"},{"inputs":[],"name":"apiVersion","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"pure","type":"function"},{"inputs":[],"name":"balanceOfWant","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"baseFeeOracle","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_vault","type":"address"},{"internalType":"address","name":"_strategist","type":"address"},{"internalType":"address","name":"_rewards","type":"address"},{"internalType":"address","name":"_keeper","type":"address"},{"internalType":"address","name":"_tradeFactory","type":"address"},{"internalType":"address","name":"_proxy","type":"address"},{"internalType":"address","name":"_gauge","type":"address"}],"name":"cloneStrategyCurveBoosted","outputs":[{"internalType":"address","name":"newStrategy","type":"address"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"creditThreshold","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"crv","outputs":[{"internalType":"contract IERC20","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"curveVoter","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"delegatedAssets","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"doHealthCheck","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"emergencyExit","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"estimatedTotalAssets","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_ethAmount","type":"uint256"}],"name":"ethToWant","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"forceHarvestTriggerOnce","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"gauge","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"harvest","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"callCostinEth","type":"uint256"}],"name":"harvestTrigger","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"healthCheck","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_vault","type":"address"},{"internalType":"address","name":"_strategist","type":"address"},{"internalType":"address","name":"_rewards","type":"address"},{"internalType":"address","name":"_keeper","type":"address"},{"internalType":"address","name":"_tradeFactory","type":"address"},{"internalType":"address","name":"_proxy","type":"address"},{"internalType":"address","name":"_gauge","type":"address"}],"name":"initialize","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"isActive","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"isBaseFeeAcceptable","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"isOriginal","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"keeper","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"localKeepCRV","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"maxReportDelay","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"metadataURI","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_newStrategy","type":"address"}],"name":"migrate","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"minReportDelay","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"","type":"string"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"proxy","outputs":[{"internalType":"contract ICurveStrategyProxy","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"bool","name":"_disableTf","type":"bool"}],"name":"removeTradeFactoryPermissions","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"rewards","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"rewardsTokens","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_baseFeeOracle","type":"address"}],"name":"setBaseFeeOracle","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_creditThreshold","type":"uint256"}],"name":"setCreditThreshold","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"bool","name":"_doHealthCheck","type":"bool"}],"name":"setDoHealthCheck","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"setEmergencyExit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"bool","name":"_forceHarvestTriggerOnce","type":"bool"}],"name":"setForceHarvestTriggerOnce","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_healthCheck","type":"address"}],"name":"setHealthCheck","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_keeper","type":"address"}],"name":"setKeeper","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_keepCrv","type":"uint256"}],"name":"setLocalKeepCrv","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_delay","type":"uint256"}],"name":"setMaxReportDelay","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"string","name":"_metadataURI","type":"string"}],"name":"setMetadataURI","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_delay","type":"uint256"}],"name":"setMinReportDelay","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_rewards","type":"address"}],"name":"setRewards","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_strategist","type":"address"}],"name":"setStrategist","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_curveVoter","type":"address"}],"name":"setVoter","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"stakedBalance","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"strategist","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address","name":"_token","type":"address"}],"name":"sweep","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"tend","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"callCostInWei","type":"uint256"}],"name":"tendTrigger","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"tradeFactory","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"address[]","name":"_rewards","type":"address[]"}],"name":"updateRewards","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_newTradeFactory","type":"address"}],"name":"updateTradeFactory","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"vault","outputs":[{"internalType":"contract VaultAPI","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"want","outputs":[{"internalType":"contract IERC20","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_amountNeeded","type":"uint256"}],"name":"withdraw","outputs":[{"internalType":"uint256","name":"_loss","type":"uint256"}],"stateMutability":"nonpayable","type":"function"}] \ No newline at end of file diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index db0ecfa38..3049913bf 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -1,6 +1,6 @@ from lib2to3.pgen2 import token import logging -import time, os +import time, os, requests,json import telebot from discordwebhook import Discord from dotenv import load_dotenv @@ -28,6 +28,7 @@ inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') +ETHERSCANKEY = os.environ.get('ETHERSCAN_KEY') invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') alerts_enabled = True if env == "PROD" or env == "TEST" else False @@ -182,6 +183,8 @@ def main(dynamically_find_multi_harvest=False): interval_seconds = 25 last_reported_block, last_reported_block030 = last_harvest_block() + last_reported_block = 16418897 + last_reported_block030 = 16418897 print("latest block (v0.3.1+ API)",last_reported_block) print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) if chain.id == 1: @@ -241,14 +244,13 @@ def handle_event(event, multi_harvest): if event.address not in endorsed_vaults: # check if a vault from inverse partnership if event.address not in INVERSE_PRIVATE_VAULTS: - print("trying",event.address) print(f"skipping: not endorsed. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") return if event.address not in INVERSE_PRIVATE_VAULTS: if get_vault_endorsement_block(event.address) > event.block_number: print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") return - + print(txn_hash) tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice ts = chain[event.block_number].timestamp @@ -296,16 +298,16 @@ def handle_event(event, multi_harvest): txn_record_exists = True r.block = event.block_number r.txn_hash = txn_hash - strategy = contract(r.strategy_address) + strategy = get_contract(r.strategy_address) r.vault_api = vault.apiVersion() r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want - r.token_symbol = contract(strategy.want()).symbol() + r.token_symbol = get_contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = 0 if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": - r.want_price_at_block = magic.get_price(constants.weth, r.block) * contract(contract(r.want_token).coins(0)).getExchangeRate() / 1e18 + r.want_price_at_block = magic.get_price(constants.weth, r.block) * get_contract(get_contract(r.want_token).coins(0)).getExchangeRate() / 1e18 else: r.want_price_at_block = magic.get_price(r.want_token, r.block) @@ -500,10 +502,10 @@ def get_vault_endorsement_block(vault_address): pass registries = CHAIN_VALUES[chain.id]["REGISTRY_ADDRESSES"] height = chain.height - lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height - + v = '0x5B8C556B8b2a78696F0B9B830B3d67623122E270' for r in registries: r = contract(r) + lo, hi = CHAIN_VALUES[chain.id]["START_BLOCK"], height while hi - lo > 1: mid = lo + (hi - lo) // 2 try: @@ -515,7 +517,6 @@ def get_vault_endorsement_block(vault_address): except: lo = mid if hi < height: - print(f'🍿REGSITERD AT BLOCK {mid}') return mid return hi @@ -660,3 +661,12 @@ def dhms_from_seconds(seconds): hours, minutes = divmod(minutes, 60) days, hours = divmod(hours, 24) return (days, hours, minutes) + +def get_contract(address): + address = web3.toChecksumAddress(address) + try: + return contract(address) + except: + response = requests.get(f"https://api.etherscan.io/api?module=contract&action=getabi&address={address}&apikey={ETHERSCANKEY}").json() + return Contract.from_abi('',address, json.loads(response['result'])) + From 58c346d98d8184c166e3320a3df558af827796e3 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 27 Jan 2023 14:52:01 -0500 Subject: [PATCH 63/86] feat: emojis --- scripts/collect_reports.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 3049913bf..e19c7ccfe 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -19,6 +19,7 @@ warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") warnings.filterwarnings("ignore", ".*It has been discarded*") +warnings.filterwarnings("ignore", ".*MismatchedABI*") # mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') @@ -78,8 +79,9 @@ "LENS_DEPLOY_BLOCK": 12707450, "VAULT_ADDRESS030": "0x19D3364A399d251E894aC732651be8B0E4e85001", "VAULT_ADDRESS031": "0xdA816459F1AB5631232FE5e97a05BBBb94970c95", - "KEEPER_CALL_CONTRACT": "0x2150b45626199CFa5089368BDcA30cd0bfB152D6", + "KEEPER_CALL_CONTRACT": "0x0a61c2146A7800bdC278833F21EBf56Cd660EE2a", "KEEPER_TOKEN": "0x1cEB5cB57C4D4E2b2433641b95Dd330A33185A44", + "KEEPER_WRAPPER": "0x0D26E894C2371AB6D20d99A65E991775e3b5CAd7", "YEARN_TREASURY": "0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde", "STRATEGIST_MULTISIG": "0x16388463d60FFE0661Cf7F1f31a7D658aC790ff7", "GOVERNANCE_MULTISIG": "0xFEB4acf3df3cDEA7399794D0869ef76A6EfAff52", @@ -105,6 +107,7 @@ "VAULT_ADDRESS031": "0x637eC617c86D24E421328e6CAEa1d92114892439", "KEEPER_CALL_CONTRACT": "0x57419fb50fa588fc165acc26449b2bf4c7731458", "KEEPER_TOKEN": "", + "KEEPER_WRAPPER": "0x0D26E894C2371AB6D20d99A65E991775e3b5CAd7", "YEARN_TREASURY": "0x89716Ad7EDC3be3B35695789C475F3e7A3Deb12a", "STRATEGIST_MULTISIG": "0x72a34AbafAB09b15E7191822A679f28E067C4a16", "GOVERNANCE_MULTISIG": "0xC0E2830724C946a6748dDFE09753613cd38f6767", @@ -128,6 +131,7 @@ "VAULT_ADDRESS031": "0x239e14A19DFF93a17339DCC444f74406C17f8E67", "KEEPER_CALL_CONTRACT": "", "KEEPER_TOKEN": "", + "KEEPER_WRAPPER": "0x0D26E894C2371AB6D20d99A65E991775e3b5CAd7", "YEARN_TREASURY": "0x1DEb47dCC9a35AD454Bf7f0fCDb03c09792C08c1", "STRATEGIST_MULTISIG": "0x6346282DB8323A54E840c6C772B4399C9c655C0d", "GOVERNANCE_MULTISIG": "0xb6bc033D34733329971B938fEf32faD7e98E56aD", @@ -150,6 +154,7 @@ "VAULT_ADDRESS031": "0x0fBeA11f39be912096cEc5cE22F46908B5375c19", "KEEPER_CALL_CONTRACT": "", "KEEPER_TOKEN": "", + "KEEPER_WRAPPER": "0x0D26E894C2371AB6D20d99A65E991775e3b5CAd7", "YEARN_TREASURY": "0x84654e35E504452769757AAe5a8C7C6599cBf954", "STRATEGIST_MULTISIG": "0xea3a15df68fCdBE44Fdb0DB675B2b3A14a148b26", "GOVERNANCE_MULTISIG": "0xF5d9D6133b698cE29567a90Ab35CfB874204B3A7", @@ -183,8 +188,8 @@ def main(dynamically_find_multi_harvest=False): interval_seconds = 25 last_reported_block, last_reported_block030 = last_harvest_block() - last_reported_block = 16418897 - last_reported_block030 = 16418897 + # last_reported_block = 16482431 + # last_reported_block030 = 16482431 print("latest block (v0.3.1+ API)",last_reported_block) print("blocks behind (v0.3.1+ API)", chain.height - last_reported_block) if chain.id == 1: @@ -566,6 +571,7 @@ def format_public_telegram(r, t): sms = CHAIN_VALUES[chain.id]["STRATEGIST_MULTISIG"] gov = CHAIN_VALUES[chain.id]["GOVERNANCE_MULTISIG"] keeper = CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"] + keeper_wrapper = CHAIN_VALUES[chain.id]["KEEPER_WRAPPER"] from_indicator = "" if t.txn_to == sms or t.txn_to == gov: @@ -577,6 +583,9 @@ def format_public_telegram(r, t): elif t.keeper_called or t.txn_from == keeper or t.txn_to == keeper: from_indicator = "🤖 " + elif t.txn_to == keeper_wrapper: # Permissionlessly executed by anyone + from_indicator = "🧍‍♂️ " + message = "" message += from_indicator message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' From 15f7f9f0cae9375457844efb5632ff0e1042dcac Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 2 Feb 2023 09:13:12 -0500 Subject: [PATCH 64/86] feat: suppress alerts on 0 gain for public channels --- scripts/collect_reports.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index e19c7ccfe..341c7dacd 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -548,15 +548,17 @@ def prepare_alerts(r, t): if alerts_enabled: if r.vault_address not in INVERSE_PRIVATE_VAULTS: m = format_public_telegram(r, t) - # Send to chain specific channels - bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) - discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) - discord.post( - embeds=[{ - "title": "New harvest", - "description": m - }], - ) + + # Only send to public TG and Discord on > $0 harvests + if r.gain != 0: + bot.send_message(CHAIN_VALUES[chain.id]["TELEGRAM_CHAT_ID"], m, parse_mode="markdown", disable_web_page_preview = True) + discord = Discord(url=CHAIN_VALUES[chain.id]["DISCORD_CHAN"]) + discord.post( + embeds=[{ + "title": "New harvest", + "description": m + }], + ) # Send to dev channel m = f'Network: {CHAIN_VALUES[chain.id]["EMOJI"]} {CHAIN_VALUES[chain.id]["NETWORK_SYMBOL"]}\n\n' + m + format_dev_telegram(r, t) From f411261d1083a9df3f82911fab4b3a360498b5aa Mon Sep 17 00:00:00 2001 From: wavey0x Date: Fri, 17 Feb 2023 07:37:26 -0600 Subject: [PATCH 65/86] fix: peth exception --- scripts/collect_reports.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 341c7dacd..339957bb8 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -313,6 +313,8 @@ def handle_event(event, multi_harvest): r.want_price_at_block = 0 if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": r.want_price_at_block = magic.get_price(constants.weth, r.block) * get_contract(get_contract(r.want_token).coins(0)).getExchangeRate() / 1e18 + elif r.want_token == "0x836A808d4828586A69364065A1e064609F5078c7": + r.want_price_at_block = magic.get_price(constants.weth, r.block) else: r.want_price_at_block = magic.get_price(r.want_token, r.block) From e0e4b99c33ebcc37aa6a4cab51b410faa80fbb8a Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 20 Feb 2023 07:24:15 -0500 Subject: [PATCH 66/86] fix --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 339957bb8..9160ac075 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -313,7 +313,7 @@ def handle_event(event, multi_harvest): r.want_price_at_block = 0 if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": r.want_price_at_block = magic.get_price(constants.weth, r.block) * get_contract(get_contract(r.want_token).coins(0)).getExchangeRate() / 1e18 - elif r.want_token == "0x836A808d4828586A69364065A1e064609F5078c7": + elif r.want_token == "0x9848482da3Ee3076165ce6497eDA906E66bB85C5": r.want_price_at_block = magic.get_price(constants.weth, r.block) else: r.want_price_at_block = magic.get_price(r.want_token, r.block) From 938b9efddfdca148f9fe0b1f1dbcaa21639f8c98 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 22 Feb 2023 20:23:15 -0500 Subject: [PATCH 67/86] feat: sdcrv --- scripts/collect_reports.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 9160ac075..ad409f285 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -311,10 +311,14 @@ def handle_event(event, multi_harvest): r.token_symbol = get_contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = 0 - if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": + print(f'Want token = {r.want_token}') + if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": # Rocket Pool ETH r.want_price_at_block = magic.get_price(constants.weth, r.block) * get_contract(get_contract(r.want_token).coins(0)).getExchangeRate() / 1e18 - elif r.want_token == "0x9848482da3Ee3076165ce6497eDA906E66bB85C5": + elif (r.want_token == "0x9848482da3Ee3076165ce6497eDA906E66bB85C5" or + r.want_token == "0x836A808d4828586A69364065A1e064609F5078c7"): # pETH r.want_price_at_block = magic.get_price(constants.weth, r.block) + elif r.want_token == '0xf7b55C3732aD8b2c2dA7c24f30A69f55c54FB717': + r.want_price_at_block = magic.get_price(crv, r.block) else: r.want_price_at_block = magic.get_price(r.want_token, r.block) From 7857d70264b806e26dc6bd0611b3a1e547cbf8bc Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 22 Feb 2023 20:44:23 -0500 Subject: [PATCH 68/86] feat: sdcrv --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index ad409f285..5e02f788f 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -318,7 +318,7 @@ def handle_event(event, multi_harvest): r.want_token == "0x836A808d4828586A69364065A1e064609F5078c7"): # pETH r.want_price_at_block = magic.get_price(constants.weth, r.block) elif r.want_token == '0xf7b55C3732aD8b2c2dA7c24f30A69f55c54FB717': - r.want_price_at_block = magic.get_price(crv, r.block) + r.want_price_at_block = magic.get_price('0xD533a949740bb3306d119CC777fa900bA034cd52', r.block) else: r.want_price_at_block = magic.get_price(r.want_token, r.block) From ec382201d6694ac7a9ed99070fedffe1a431d2e5 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Sun, 26 Feb 2023 20:44:45 -0700 Subject: [PATCH 69/86] feat: opti new registry --- scripts/collect_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 5e02f788f..2839a9ea6 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -146,9 +146,9 @@ "EMOJI": "🔴", "START_DATE": datetime(2022, 8, 6, tzinfo=timezone.utc), "START_BLOCK": 24097341, - "REGISTRY_ADDRESSES": ["0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0"], + "REGISTRY_ADDRESSES": ["0x1ba4eB0F44AB82541E56669e18972b0d6037dfE0", "0x79286Dd38C9017E5423073bAc11F53357Fc5C128"], "REGISTRY_DEPLOY_BLOCK": 18097341, - "REGISTRY_HELPER_ADDRESS": "0x0983b4899a3168c2509569faf1e4e75c57b4aba6", + "REGISTRY_HELPER_ADDRESS": "0x2222aaf54Fe3B10937E91A0C2B8a92c18A636D05", "LENS_ADDRESS": "0xD3A93C794ee2798D8f7906493Cd3c2A835aa0074", "VAULT_ADDRESS030": "0x0fBeA11f39be912096cEc5cE22F46908B5375c19", "VAULT_ADDRESS031": "0x0fBeA11f39be912096cEc5cE22F46908B5375c19", From 1c28acd75f25073d3e407f378b405cf848cf04b1 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Sun, 26 Feb 2023 22:22:37 -0700 Subject: [PATCH 70/86] feat: disable alerts locally --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 2839a9ea6..7e1ea82b7 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -32,7 +32,7 @@ ETHERSCANKEY = os.environ.get('ETHERSCAN_KEY') invbot = telebot.TeleBot(inv_telegram_key) env = os.environ.get('ENVIRONMENT') -alerts_enabled = True if env == "PROD" or env == "TEST" else False +alerts_enabled = True if env == "PROD" else False #or env == "TEST" else False test_channel = os.environ.get('TELEGRAM_CHANNEL_TEST') if env == "TEST": From 66b8c293c06e682af60b5fa44efc6baa529e814d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Sat, 4 Mar 2023 11:57:29 -0500 Subject: [PATCH 71/86] chore: keeper indicator --- scripts/collect_reports.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 7e1ea82b7..8aa78108b 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -10,7 +10,9 @@ from brownie import chain, web3, Contract, ZERO_ADDRESS, interface from web3._utils.events import construct_event_topic_set from yearn.utils import contract, contract_creation_block -from yearn.prices import magic, constants + +from yearn.prices import constants +from ypricemagic import magic from yearn.db.models import Reports, Event, Transactions, Session, engine, select from sqlalchemy import desc, asc from yearn.networks import Network @@ -312,15 +314,7 @@ def handle_event(event, multi_harvest): r.want_token = strategy.want() r.want_price_at_block = 0 print(f'Want token = {r.want_token}') - if r.want_token == "0x447Ddd4960d9fdBF6af9a790560d0AF76795CB08": # Rocket Pool ETH - r.want_price_at_block = magic.get_price(constants.weth, r.block) * get_contract(get_contract(r.want_token).coins(0)).getExchangeRate() / 1e18 - elif (r.want_token == "0x9848482da3Ee3076165ce6497eDA906E66bB85C5" or - r.want_token == "0x836A808d4828586A69364065A1e064609F5078c7"): # pETH - r.want_price_at_block = magic.get_price(constants.weth, r.block) - elif r.want_token == '0xf7b55C3732aD8b2c2dA7c24f30A69f55c54FB717': - r.want_price_at_block = magic.get_price('0xD533a949740bb3306d119CC777fa900bA034cd52', r.block) - else: - r.want_price_at_block = magic.get_price(r.want_token, r.block) + r.want_price_at_block = magic.get_price(r.want_token, r.block) r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() @@ -588,7 +582,12 @@ def format_public_telegram(r, t): elif t.txn_from == r.strategist and t.txn_to != sms: from_indicator = "🧠 " - elif t.keeper_called or t.txn_from == keeper or t.txn_to == keeper: + elif ( + t.keeper_called or + t.txn_from == keeper or + t.txn_to == keeper or + t.txn_to == '0xf4F748D45E03a70a9473394B28c3C7b5572DfA82' # ETH public harvest job + ): from_indicator = "🤖 " elif t.txn_to == keeper_wrapper: # Permissionlessly executed by anyone From f560f3c549c279d3224978111e421628c54667e3 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 30 Mar 2023 20:13:45 -0600 Subject: [PATCH 72/86] chore: temp price fix --- scripts/collect_reports.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 8aa78108b..964934af7 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -314,7 +314,10 @@ def handle_event(event, multi_harvest): r.want_token = strategy.want() r.want_price_at_block = 0 print(f'Want token = {r.want_token}') - r.want_price_at_block = magic.get_price(r.want_token, r.block) + if r.vault_address == '0x9E0E0AF468FbD041Cab7883c5eEf16D1A99a47C3': + r.want_price_at_block = 1 + else: + r.want_price_at_block = magic.get_price(r.want_token, r.block) r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() From 86c02f4705396026c3635d6791fb0d53a374de75 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 6 Apr 2023 00:37:57 -0400 Subject: [PATCH 73/86] chore: backout get_contract stuff --- scripts/collect_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 964934af7..62c25ddfb 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -305,12 +305,12 @@ def handle_event(event, multi_harvest): txn_record_exists = True r.block = event.block_number r.txn_hash = txn_hash - strategy = get_contract(r.strategy_address) + strategy = Contract(r.strategy_address) r.vault_api = vault.apiVersion() r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want - r.token_symbol = get_contract(strategy.want()).symbol() + r.token_symbol = Contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = 0 print(f'Want token = {r.want_token}') From 7aaf6622e8b493004f9acbfbad144617e0d6629a Mon Sep 17 00:00:00 2001 From: wavey0x Date: Thu, 6 Apr 2023 00:47:00 -0400 Subject: [PATCH 74/86] chore: print token --- scripts/collect_reports.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 62c25ddfb..ca25ed16d 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -305,12 +305,13 @@ def handle_event(event, multi_harvest): txn_record_exists = True r.block = event.block_number r.txn_hash = txn_hash - strategy = Contract(r.strategy_address) + print("ETHERSCAN_TOKEN: ", os.environ.get('ETHERSCAN_TOKEN')) + strategy = contract(r.strategy_address) r.vault_api = vault.apiVersion() r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want - r.token_symbol = Contract(strategy.want()).symbol() + r.token_symbol = contract(strategy.want()).symbol() r.want_token = strategy.want() r.want_price_at_block = 0 print(f'Want token = {r.want_token}') From ddd9c3544099696915c2997c12327a025d329f25 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 15 May 2023 15:10:38 -0400 Subject: [PATCH 75/86] feat: ypm --- scripts/collect_reports.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index ca25ed16d..c24705aa9 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -10,9 +10,8 @@ from brownie import chain, web3, Contract, ZERO_ADDRESS, interface from web3._utils.events import construct_event_topic_set from yearn.utils import contract, contract_creation_block - from yearn.prices import constants -from ypricemagic import magic +from y import get_price from yearn.db.models import Reports, Event, Transactions, Session, engine, select from sqlalchemy import desc, asc from yearn.networks import Network @@ -23,7 +22,7 @@ warnings.filterwarnings("ignore", ".*It has been discarded*") warnings.filterwarnings("ignore", ".*MismatchedABI*") - +logging.basicConfig(level=logging.DEBUG) # mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') # ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') # discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') @@ -284,11 +283,11 @@ def handle_event(event, multi_harvest): t.txn_from = tx["from"] t.txn_gas_used = tx.gasUsed t.txn_gas_price = gas_price / 1e9 # Use gwei - t.eth_price_at_block = magic.get_price(constants.weth, t.block) + t.eth_price_at_block = get_price(constants.weth, t.block) t.call_cost_eth = gas_price * tx.gasUsed / 1e18 t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth if chain.id == 1: - t.kp3r_price_at_block = magic.get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) + t.kp3r_price_at_block = get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) t.kp3r_paid = get_keeper_payment(tx) / 1e18 t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block t.keeper_called = t.kp3r_paid > 0 @@ -318,7 +317,7 @@ def handle_event(event, multi_harvest): if r.vault_address == '0x9E0E0AF468FbD041Cab7883c5eEf16D1A99a47C3': r.want_price_at_block = 1 else: - r.want_price_at_block = magic.get_price(r.want_token, r.block) + r.want_price_at_block = get_price(r.want_token, r.block) r.want_gain_usd = r.gain * r.want_price_at_block r.vault_name = vault.name() @@ -345,7 +344,7 @@ def handle_event(event, multi_harvest): _from, _to, _val = tfr.args.values() if tfr.address == crv and _from == r.strategy_address and (_to == voter or _to == treasury): r.keep_crv = _val / 1e18 - r.crv_price_usd = magic.get_price(crv, r.block) + r.crv_price_usd = get_price(crv, r.block) r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd if r.keep_crv > 0: From 8f0fefff448ee7c508d2ce5ddc3946aa6a75ba8d Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 23 May 2023 22:13:53 -0400 Subject: [PATCH 76/86] chore: change to 10 minute timer --- scripts/collect_reports.py | 68 +++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index c24705aa9..78d96edf3 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -200,48 +200,48 @@ def main(dynamically_find_multi_harvest=False): if chain.id == 1: event_filter_v030 = web3.eth.filter({'topics': topics_v030, "fromBlock": last_reported_block030 + 1}) - while True: # Keep this as a long-running script - events_to_process = [] - transaction_hashes = [] - if dynamically_find_multi_harvest: - # The code below is used to populate the "multi_harvest" property # - for strategy_report_event in decode_logs(event_filter.get_new_entries()): - e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + # while True: # Keep this as a long-running script # <--- disabled this since ypm issues + events_to_process = [] + transaction_hashes = [] + if dynamically_find_multi_harvest: + # The code below is used to populate the "multi_harvest" property # + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(0, len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + + if chain.id == 1: # No old vaults deployed anywhere other than mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) if e.txn_hash in transaction_hashes: e.multi_harvest = True for i in range(0, len(events_to_process)): - if e.txn_hash == events_to_process[i].txn_hash: - events_to_process[i].multi_harvest = True + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True else: transaction_hashes.append(strategy_report_event.transaction_hash.hex()) events_to_process.append(e) - - if chain.id == 1: # No old vaults deployed anywhere other than mainnet - for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): - e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) - if e.txn_hash in transaction_hashes: - e.multi_harvest = True - for i in range(0, len(events_to_process)): - if e.txn_hash == events_to_process[i].txn_hash: - events_to_process[i].multi_harvest = True - else: - transaction_hashes.append(strategy_report_event.transaction_hash.hex()) - events_to_process.append(e) - - for e in events_to_process: - handle_event(e.event, e.multi_harvest) - time.sleep(interval_seconds) - else: - for strategy_report_event in decode_logs(event_filter.get_new_entries()): - e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + + for e in events_to_process: + handle_event(e.event, e.multi_harvest) + # time.sleep(interval_seconds) + else: + for strategy_report_event in decode_logs(event_filter.get_new_entries()): + e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) + handle_event(e.event, e.multi_harvest) + + if chain.id == 1: # Old vault API exists only on Ethereum mainnet + for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): + e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) handle_event(e.event, e.multi_harvest) - - if chain.id == 1: # Old vault API exists only on Ethereum mainnet - for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): - e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) - handle_event(e.event, e.multi_harvest) - time.sleep(interval_seconds) + # time.sleep(interval_seconds) def handle_event(event, multi_harvest): # exception because skeletor didnt verify contract From df902884263ee46570adc6ffd08f55e0eb305370 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Wed, 12 Jul 2023 09:41:15 -0400 Subject: [PATCH 77/86] chore: add vault exceptions list --- scripts/collect_reports.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 78d96edf3..eaa75868a 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -21,6 +21,7 @@ warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") warnings.filterwarnings("ignore", ".*It has been discarded*") warnings.filterwarnings("ignore", ".*MismatchedABI*") +logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.DEBUG) # mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') @@ -28,6 +29,9 @@ # discord_mainnet = os.environ.get('DISCORD_CHANNEL_1') # discord_ftm = os.environ.get('DISCORD_CHANNEL_250') +VAULT_EXCEPTIONS = [ + '0xcd68c3fC3e94C5AcC10366556b836855D96bfa93', # yvCurve-dETH-f +] inv_telegram_key = os.environ.get('WAVEY_ALERTS_BOT_KEY') ETHERSCANKEY = os.environ.get('ETHERSCAN_KEY') @@ -244,9 +248,10 @@ def main(dynamically_find_multi_harvest=False): # time.sleep(interval_seconds) def handle_event(event, multi_harvest): - # exception because skeletor didnt verify contract endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) txn_hash = event.transaction_hash.hex() + if event.address in VAULT_EXCEPTIONS: + return if event.address not in endorsed_vaults: # check if a vault from inverse partnership if event.address not in INVERSE_PRIVATE_VAULTS: @@ -256,6 +261,7 @@ def handle_event(event, multi_harvest): if get_vault_endorsement_block(event.address) > event.block_number: print(f"skipping: not endorsed yet. txn hash {txn_hash}. chain id {chain.id} sync {event.block_number} / {chain.height}.") return + print(txn_hash) tx = web3.eth.getTransactionReceipt(txn_hash) gas_price = web3.eth.getTransaction(txn_hash).gasPrice From fa4e77e93c4a0e108cad3e501a0f9057e58f0053 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 9 Oct 2023 10:14:59 -0400 Subject: [PATCH 78/86] rekt pool fix --- scripts/collect_reports.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index eaa75868a..3e79e90b4 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -322,6 +322,10 @@ def handle_event(event, multi_harvest): print(f'Want token = {r.want_token}') if r.vault_address == '0x9E0E0AF468FbD041Cab7883c5eEf16D1A99a47C3': r.want_price_at_block = 1 + if r.want_token in [ + '0xC4C319E2D4d66CcA4464C0c2B32c9Bd23ebe784e', # rekt alETH + ]: + r.want_price_at_block = 0 else: r.want_price_at_block = get_price(r.want_token, r.block) From a2d1b1810f7243454e5d97c593198713bfd8c229 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Mon, 9 Oct 2023 10:15:01 -0400 Subject: [PATCH 79/86] rekt pool fix --- scripts/collect_reports.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 3e79e90b4..bda355828 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -324,6 +324,8 @@ def handle_event(event, multi_harvest): r.want_price_at_block = 1 if r.want_token in [ '0xC4C319E2D4d66CcA4464C0c2B32c9Bd23ebe784e', # rekt alETH + '0x9848482da3Ee3076165ce6497eDA906E66bB85C5', # rekt jPegd pETH + '0xEd4064f376cB8d68F770FB1Ff088a3d0F3FF5c4d', # rekt crvETH ]: r.want_price_at_block = 0 else: From 7f40f867973595e840f146adc59c24ab895cca00 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 10 Oct 2023 03:06:27 -0400 Subject: [PATCH 80/86] decimal float multiply --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index bda355828..a33a15afa 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -291,7 +291,7 @@ def handle_event(event, multi_harvest): t.txn_gas_price = gas_price / 1e9 # Use gwei t.eth_price_at_block = get_price(constants.weth, t.block) t.call_cost_eth = gas_price * tx.gasUsed / 1e18 - t.call_cost_usd = t.eth_price_at_block * t.call_cost_eth + t.call_cost_usd = float(t.eth_price_at_block) * float(t.call_cost_eth) if chain.id == 1: t.kp3r_price_at_block = get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) t.kp3r_paid = get_keeper_payment(tx) / 1e18 From c10b50afdc0e9c4e25fa9bcd448e2ef97a2d10cf Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 10 Oct 2023 03:07:49 -0400 Subject: [PATCH 81/86] decimal float multiply --- scripts/collect_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index a33a15afa..3a9e928aa 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -295,7 +295,7 @@ def handle_event(event, multi_harvest): if chain.id == 1: t.kp3r_price_at_block = get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) t.kp3r_paid = get_keeper_payment(tx) / 1e18 - t.kp3r_paid_usd = t.kp3r_paid * t.kp3r_price_at_block + t.kp3r_paid_usd = float(t.kp3r_paid) * float(t.kp3r_price_at_block) t.keeper_called = t.kp3r_paid > 0 else: if t.txn_to == CHAIN_VALUES[chain.id]["KEEPER_CALL_CONTRACT"]: From 27c73185ae5c1411dbb519c61088030fab3279c6 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 10 Oct 2023 03:09:57 -0400 Subject: [PATCH 82/86] decimal float multiply --- scripts/collect_reports.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 3a9e928aa..e4b96c151 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -331,7 +331,7 @@ def handle_event(event, multi_harvest): else: r.want_price_at_block = get_price(r.want_token, r.block) - r.want_gain_usd = r.gain * r.want_price_at_block + r.want_gain_usd = r.gain * float(r.want_price_at_block) r.vault_name = vault.name() r.strategy_name = strategy.name() r.strategy_api = strategy.apiVersion() @@ -357,7 +357,7 @@ def handle_event(event, multi_harvest): if tfr.address == crv and _from == r.strategy_address and (_to == voter or _to == treasury): r.keep_crv = _val / 1e18 r.crv_price_usd = get_price(crv, r.block) - r.keep_crv_value_usd = r.keep_crv * r.crv_price_usd + r.keep_crv_value_usd = r.keep_crv * float(r.crv_price_usd) if r.keep_crv > 0: yvecrv_token = web3.eth.contract(yvecrv, abi=token_abi) @@ -613,7 +613,7 @@ def format_public_telegram(r, t): message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' message += f'📅 {r.date_string} UTC \n\n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) - net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) + net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * float(r.want_price_at_block)) sym = r.token_symbol.replace('_','-') message += f'💰 Net profit: {net_profit_want} {sym} (${net_profit_usd})\n\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) From 82203a06afa8812912c4c90cbc805e5b91d38517 Mon Sep 17 00:00:00 2001 From: wavey0x Date: Tue, 17 Oct 2023 00:10:29 -0400 Subject: [PATCH 83/86] float cast --- scripts/collect_reports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index e4b96c151..208e3d1d8 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -613,7 +613,7 @@ def format_public_telegram(r, t): message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n\n' message += f'📅 {r.date_string} UTC \n\n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) - net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * float(r.want_price_at_block)) + net_profit_usd = "{:,.2f}".format(float(r.gain - r.loss) * float(r.want_price_at_block)) sym = r.token_symbol.replace('_','-') message += f'💰 Net profit: {net_profit_want} {sym} (${net_profit_usd})\n\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) @@ -634,7 +634,7 @@ def format_public_telegram_inv(r, t): message += f' [{r.vault_name}]({explorer}address/{r.vault_address}) -- [{r.strategy_name}]({explorer}address/{r.strategy_address})\n' message += f'{r.date_string} UTC \n' net_profit_want = "{:,.2f}".format(r.gain - r.loss) - net_profit_usd = "{:,.2f}".format((r.gain - r.loss) * r.want_price_at_block) + net_profit_usd = "{:,.2f}".format(float(r.gain - r.loss) * r.want_price_at_block) sym = r.token_symbol.replace('_','-') message += f'Net profit: {net_profit_want} {sym} (${net_profit_usd})\n' txn_cost_str = "${:,.2f}".format(t.call_cost_usd) From 60e1efeb9bcc2350df23362816b89c16274644ad Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 8 Feb 2024 14:58:06 -0500 Subject: [PATCH 84/86] feat: asyncify and refactor --- scripts/collect_reports.py | 177 ++++++++++++++++++++----------------- 1 file changed, 98 insertions(+), 79 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 208e3d1d8..472ec4e7e 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -12,6 +12,7 @@ from yearn.utils import contract, contract_creation_block from yearn.prices import constants from y import get_price +from y.utils.dank_mids import dank_w3 from yearn.db.models import Reports, Event, Transactions, Session, engine, select from sqlalchemy import desc, asc from yearn.networks import Network @@ -185,12 +186,12 @@ vault_v030.events.StrategyReported().abi, web3.codec, {} ) -def main(dynamically_find_multi_harvest=False): - if chain.id == 1: - ycrv = interface.YCRV('0xFCc5c47bE19d06BF83eB04298b026F81069ff65b') - y = Contract.from_abi('','0xFCc5c47bE19d06BF83eB04298b026F81069ff65b',interface.YCRV.abi) + +def main(): + asyncio.get_event_loop().run_until_complete(_main()) + +async def _main(dynamically_find_multi_harvest=False): print(f"dynamic multi_harvest detection is enabled: {dynamically_find_multi_harvest}") - interval_seconds = 25 last_reported_block, last_reported_block030 = last_harvest_block() # last_reported_block = 16482431 @@ -200,55 +201,25 @@ def main(dynamically_find_multi_harvest=False): if chain.id == 1: print("latest block (v0.3.0 API)",last_reported_block030) print("blocks behind (v0.3.0 API)", chain.height - last_reported_block030) - event_filter = web3.eth.filter({'topics': topics, "fromBlock": last_reported_block + 1}) + + filters = [StrategyReportedEvents(dynamically_find_multi_harvest, from_block=last_reported_block+1)] if chain.id == 1: - event_filter_v030 = web3.eth.filter({'topics': topics_v030, "fromBlock": last_reported_block030 + 1}) + # No old vaults deployed anywhere other than mainnet + filters.append(StrategyReportedEventsV030(dynamically_find_multi_harvest, from_block=last_reported_block030+1)) # while True: # Keep this as a long-running script # <--- disabled this since ypm issues - events_to_process = [] - transaction_hashes = [] - if dynamically_find_multi_harvest: - # The code below is used to populate the "multi_harvest" property # - for strategy_report_event in decode_logs(event_filter.get_new_entries()): - e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) - if e.txn_hash in transaction_hashes: - e.multi_harvest = True - for i in range(0, len(events_to_process)): - if e.txn_hash == events_to_process[i].txn_hash: - events_to_process[i].multi_harvest = True - else: - transaction_hashes.append(strategy_report_event.transaction_hash.hex()) - events_to_process.append(e) - - if chain.id == 1: # No old vaults deployed anywhere other than mainnet - for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): - e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) - if e.txn_hash in transaction_hashes: - e.multi_harvest = True - for i in range(0, len(events_to_process)): - if e.txn_hash == events_to_process[i].txn_hash: - events_to_process[i].multi_harvest = True - else: - transaction_hashes.append(strategy_report_event.transaction_hash.hex()) - events_to_process.append(e) - - for e in events_to_process: - handle_event(e.event, e.multi_harvest) - # time.sleep(interval_seconds) - else: - for strategy_report_event in decode_logs(event_filter.get_new_entries()): - e = Event(False, strategy_report_event, strategy_report_event.transaction_hash.hex()) - handle_event(e.event, e.multi_harvest) - - if chain.id == 1: # Old vault API exists only on Ethereum mainnet - for strategy_report_event in decode_logs(event_filter_v030.get_new_entries()): - e = Event(True, strategy_report_event, strategy_report_event.transaction_hash.hex()) - handle_event(e.event, e.multi_harvest) - - # time.sleep(interval_seconds) - -def handle_event(event, multi_harvest): - endorsed_vaults = list(contract(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]).getVaults()) + tasks = [] + async for strategy_report_event in a_sync.as_yielded(*filters): + asyncio.create_task(handle_event(strategy_report_event.event, strategy_report_event.multi_harvest)) + +@alru_cache(maxsize=1) +async def get_vaults() -> List[str]: + registry_helper = await Contract.coroutine(CHAIN_VALUES[chain.id]["REGISTRY_HELPER_ADDRESS"]) + return list(await registry_helper.getVaults.coroutine()) + +@a_sync.a_sync(default='sync') +async def handle_event(event, multi_harvest): + endorsed_vaults = await get_vaults() txn_hash = event.transaction_hash.hex() if event.address in VAULT_EXCEPTIONS: return @@ -263,19 +234,25 @@ def handle_event(event, multi_harvest): return print(txn_hash) - tx = web3.eth.getTransactionReceipt(txn_hash) - gas_price = web3.eth.getTransaction(txn_hash).gasPrice - ts = chain[event.block_number].timestamp + block, tx, tx_receipt = await asyncio.gather( + dank_w3.eth.getBlock(event.block_number), + dank_w3.eth.getTransaction(txn_hash), + dank_w3.eth.getTransactionReceipt(txn_hash), + ) + gas_price = tx.gasPrice + ts = block.timestamp dt = datetime.utcfromtimestamp(ts).strftime("%m/%d/%Y, %H:%M:%S") r = Reports() r.multi_harvest = multi_harvest r.chain_id = chain.id r.vault_address = event.address try: - vault = contract(r.vault_address) + vault = await Contract.coroutine(r.vault_address) except ValueError: return - r.vault_decimals = vault.decimals() + # now we cache this so we don't need to call it for every event with `vault.decimals()` + v = ERC20(vault.address, asynchronous=True) + r.vault_decimals = await v.decimals r.strategy_address, r.gain, r.loss, r.debt_paid, r.total_gain, r.total_loss, r.total_debt, r.debt_added, r.debt_ratio = normalize_event_values(event.values(), r.vault_decimals) txn_record_exists = False @@ -285,16 +262,16 @@ def handle_event(event, multi_harvest): t.chain_id = chain.id t.txn_hash = txn_hash t.block = event.block_number - t.txn_to = tx.to - t.txn_from = tx["from"] - t.txn_gas_used = tx.gasUsed + t.txn_to = tx_receipt.to + t.txn_from = tx_receipt["from"] + t.txn_gas_used = tx_receipt.gasUsed t.txn_gas_price = gas_price / 1e9 # Use gwei - t.eth_price_at_block = get_price(constants.weth, t.block) - t.call_cost_eth = gas_price * tx.gasUsed / 1e18 + t.eth_price_at_block = await get_price(constants.weth, t.block, sync=False) + t.call_cost_eth = gas_price * tx_receipt.gasUsed / 1e18 t.call_cost_usd = float(t.eth_price_at_block) * float(t.call_cost_eth) if chain.id == 1: - t.kp3r_price_at_block = get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block) - t.kp3r_paid = get_keeper_payment(tx) / 1e18 + t.kp3r_price_at_block = await get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block, sync=False) + t.kp3r_paid = get_keeper_payment(tx_receipt) / 1e18 t.kp3r_paid_usd = float(t.kp3r_paid) * float(t.kp3r_price_at_block) t.keeper_called = t.kp3r_paid > 0 else: @@ -311,13 +288,15 @@ def handle_event(event, multi_harvest): r.block = event.block_number r.txn_hash = txn_hash print("ETHERSCAN_TOKEN: ", os.environ.get('ETHERSCAN_TOKEN')) - strategy = contract(r.strategy_address) + strategy = await Contract.coroutine(r.strategy_address) - r.vault_api = vault.apiVersion() - r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) + r.vault_api, r.want_token = await asyncio.gather( + vault.apiVersion.coroutine(), + strategy.want.coroutine(), + ) + r.gov_fee_in_want, r.strategist_fee_in_want = parse_fees(tx_receipt, r.vault_address, r.strategy_address, r.vault_decimals, r.gain, r.vault_api) r.gain_post_fees = r.gain - r.loss - r.strategist_fee_in_want - r.gov_fee_in_want - r.token_symbol = contract(strategy.want()).symbol() - r.want_token = strategy.want() + r.token_symbol = await ERC20(r.want_token, asynchronous=True).symbol r.want_price_at_block = 0 print(f'Want token = {r.want_token}') if r.vault_address == '0x9E0E0AF468FbD041Cab7883c5eEf16D1A99a47C3': @@ -329,14 +308,23 @@ def handle_event(event, multi_harvest): ]: r.want_price_at_block = 0 else: - r.want_price_at_block = get_price(r.want_token, r.block) + r.want_price_at_block = await get_price(r.want_token, r.block, sync=False) r.want_gain_usd = r.gain * float(r.want_price_at_block) - r.vault_name = vault.name() - r.strategy_name = strategy.name() - r.strategy_api = strategy.apiVersion() - r.strategist = strategy.strategist() - r.vault_symbol = vault.symbol() + + ( + r.vault_name, + r.strategy_name, + r.strategy_api, + r.strategist, + r.vault_symbol, + ) = await asyncio.gather( + v.name, + ERC20(strategy, asynchronous=True).name, + strategy.apiVersion.coroutine(), + strategy.strategist.coroutine(), + v.symbol, + ) r.date = datetime.utcfromtimestamp(ts) r.date_string = dt r.timestamp = ts @@ -348,22 +336,23 @@ def handle_event(event, multi_harvest): yvecrv = '0xc5bDdf9843308380375a611c18B50Fb9341f502A' voter = '0xF147b8125d2ef93FB6965Db97D6746952a133934' treasury = '0x93A62dA5a14C80f265DAbC077fCEE437B1a0Efde' - token_abi = Contract(crv).abi + crv_contract = await Contract.coroutine(crv) + token_abi = crv_contract.abi crv_token = web3.eth.contract(crv, abi=token_abi) - decoded_events = crv_token.events.Transfer().processReceipt(tx) + decoded_events = crv_token.events.Transfer().processReceipt(tx_receipt) r.keep_crv = 0 for tfr in decoded_events: _from, _to, _val = tfr.args.values() if tfr.address == crv and _from == r.strategy_address and (_to == voter or _to == treasury): r.keep_crv = _val / 1e18 - r.crv_price_usd = get_price(crv, r.block) + r.crv_price_usd = await get_price(crv, r.block, sync=False) r.keep_crv_value_usd = r.keep_crv * float(r.crv_price_usd) if r.keep_crv > 0: yvecrv_token = web3.eth.contract(yvecrv, abi=token_abi) - decoded_events = yvecrv_token.events.Transfer().processReceipt(tx) + decoded_events = yvecrv_token.events.Transfer().processReceipt(tx_receipt) try: - r.keep_crv_percent = strategy.keepCRV() + r.keep_crv_percent = await strategy.keepCRV.coroutine() except: pass for tfr in decoded_events: @@ -701,3 +690,33 @@ def get_contract(address): response = requests.get(f"https://api.etherscan.io/api?module=contract&action=getabi&address={address}&apikey={ETHERSCANKEY}").json() return Contract.from_abi('',address, json.loads(response['result'])) + +class _StrategyReportedEvents(ProcessedEvents): + is_v030: bool + def __init__(self, topics: List, from_block: int, dynamically_find_multi_harvest: bool) -> None: + super().__init__(topics=topics, from_block=from_block) + self.dynamically_find_multi_harvest = dynamically_find_multi_harvest + + async def _process_event(self, strategy_report_event: _EventItem) -> Event: + e = Event(self.is_v030, strategy_report_event, strategy_report_event.transaction_hash.hex()) + if self.dynamically_find_multi_harvest: + # The code below is used to populate the "multi_harvest" property # + if e.txn_hash in transaction_hashes: + e.multi_harvest = True + for i in range(len(events_to_process)): + if e.txn_hash == events_to_process[i].txn_hash: + events_to_process[i].multi_harvest = True + else: + transaction_hashes.append(strategy_report_event.transaction_hash.hex()) + events_to_process.append(e) + return e + +class StrategyReportedEvents(_StrategyReportedEvents): + is_v030 = False + def __init__(self, from_block: int, dynamically_find_multi_harvest: bool) -> None: + super().__init__(topics, from_block, dynamically_find_multi_harvest) + +class StrategyReportedEventsV030(_StrategyReportedEvents): + is_v030 = True + def __init__(self, from_block: int, dynamically_find_multi_harvest: bool) -> None: + super().__init__(topics_v030, from_block, dynamically_find_multi_harvest) From af207de40a0a6a431384b089641bb453430d5afb Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:08:40 -0500 Subject: [PATCH 85/86] feat: asyncify and refactor more --- scripts/collect_reports.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 472ec4e7e..8a62dd59c 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -270,8 +270,11 @@ async def handle_event(event, multi_harvest): t.call_cost_eth = gas_price * tx_receipt.gasUsed / 1e18 t.call_cost_usd = float(t.eth_price_at_block) * float(t.call_cost_eth) if chain.id == 1: - t.kp3r_price_at_block = await get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block, sync=False) - t.kp3r_paid = get_keeper_payment(tx_receipt) / 1e18 + t.kp3r_price_at_block, t.kp3r_paid = await asyncio.gather( + get_price(CHAIN_VALUES[chain.id]["KEEPER_TOKEN"], t.block, sync=False), + get_keeper_payment(tx_receipt) + ) + t.kp3r_paid /= 10 ** 18 t.kp3r_paid_usd = float(t.kp3r_paid) * float(t.kp3r_price_at_block) t.keeper_called = t.kp3r_paid > 0 else: @@ -368,7 +371,7 @@ async def handle_event(event, multi_harvest): if previous_report != None: previous_report_id = previous_report.id r.previous_report_id = previous_report_id - r.rough_apr_pre_fee, r.rough_apr_post_fee = compute_apr(r, previous_report) + r.rough_apr_pre_fee, r.rough_apr_post_fee = await compute_apr(r, previous_report) # Insert to database insert_success = False try: @@ -414,11 +417,13 @@ def last_harvest_block(): return result1, result2 -def get_keeper_payment(tx): +async def get_keeper_payment(tx): kp3r_token = CHAIN_VALUES[chain.id]["KEEPER_TOKEN"] - token = contract(kp3r_token) - denominator = 10 ** token.decimals() - token = web3.eth.contract(str(kp3r_token), abi=token.abi) + kp3r_contract, denominator = await asyncio.gather( + Contract.coroutine(kp3r_token), + await ERC20(kp3r_token, asynchronous=True).scale, + ) + token = web3.eth.contract(str(kp3r_token), abi=kp3r_contract.abi) decoded_events = token.events.Transfer().processReceipt(tx) amount = 0 for e in decoded_events: @@ -429,19 +434,23 @@ def get_keeper_payment(tx): amount = token_amount return amount -def compute_apr(report, previous_report): +async def compute_apr(report, previous_report): SECONDS_IN_A_YEAR = 31557600 seconds_between_reports = report.timestamp - previous_report.timestamp pre_fee_apr = 0 post_fee_apr = 0 if report.vault_address == '0x27B5739e22ad9033bcBf192059122d163b60349D': - vault = Contract(report.vault_address) - if vault.totalAssets() == 0 or seconds_between_reports == 0: + vault, vault_scale = await asyncio.gather( + Contract.coroutine(report.vault_address), + ERC20(report.vault_address, asynchronous=True).scale, + ) + total_assets = await vault.totalAssets.coroutine() + if total_assets == 0 or seconds_between_reports == 0: return 0, 0 - pre_fee_apr = report.gain / int(vault.totalAssets()/10**vault.decimals()) * (SECONDS_IN_A_YEAR / seconds_between_reports) + pre_fee_apr = report.gain / int(total_assets/vault_scale) * (SECONDS_IN_A_YEAR / seconds_between_reports) if report.gain_post_fees != 0: - post_fee_apr = report.gain_post_fees / int(vault.totalAssets()/10**vault.decimals()) * (SECONDS_IN_A_YEAR / seconds_between_reports) + post_fee_apr = report.gain_post_fees / int(total_assets/vault_scale) * (SECONDS_IN_A_YEAR / seconds_between_reports) else: if int(previous_report.total_debt) == 0 or seconds_between_reports == 0: return 0, 0 From c930be239349966805f9646008238222b3941ca0 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Thu, 8 Feb 2024 15:14:34 -0500 Subject: [PATCH 86/86] fix: broken import --- scripts/collect_reports.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/collect_reports.py b/scripts/collect_reports.py index 8a62dd59c..6e60b53c6 100644 --- a/scripts/collect_reports.py +++ b/scripts/collect_reports.py @@ -7,16 +7,15 @@ from yearn.cache import memory import pandas as pd from datetime import datetime, timezone -from brownie import chain, web3, Contract, ZERO_ADDRESS, interface +from brownie import chain, web3, ZERO_ADDRESS, interface from web3._utils.events import construct_event_topic_set from yearn.utils import contract, contract_creation_block from yearn.prices import constants -from y import get_price +from y import Contract, Network, get_price from y.utils.dank_mids import dank_w3 +from y.utils.events import ProcessedEvents from yearn.db.models import Reports, Event, Transactions, Session, engine, select from sqlalchemy import desc, asc -from yearn.networks import Network -from yearn.events import decode_logs import warnings warnings.filterwarnings("ignore", ".*Class SelectOfScalar will not make use of SQL compilation caching.*") warnings.filterwarnings("ignore", ".*Locally compiled and on-chain*") @@ -24,7 +23,6 @@ warnings.filterwarnings("ignore", ".*MismatchedABI*") logging.basicConfig(level=logging.DEBUG) -logging.basicConfig(level=logging.DEBUG) # mainnet_public_channel = os.environ.get('TELEGRAM_CHANNEL_1_PUBLIC') # ftm_public_channel = os.environ.get('TELEGRAM_CHANNEL_250_PUBLIC') # discord_mainnet = os.environ.get('DISCORD_CHANNEL_1')