Skip to content

Commit

Permalink
Init Balancer V3 spells and macros (duneanalytics#7274)
Browse files Browse the repository at this point in the history 
* commit v3 macros (sans liquidity)

* commit new spells

* add pools_fees

* ange poolbalancechanged to liquidityadded/removed

* update sources and schemas

* add aave token pricing

* improve schemas and add aave tokens prices to macros

* introduce incremental aave models

* fix typo on gnosis trades

* add new spells and schemas

* remove potentially troublesome incremental filter from token_data

* 1st round of review on macros

* 1st round of review on dex subproject

* update static tokens to include morpho

* replace atokens for erc4626 tokens

* update sources and schemas

* update sources and schema

* again

* again

* again

* again2

* fix names

* again

* again

* add pool labels to sources

* fix macros

* fix liquidity macro

* fix protocol_fee

* a

* fix errors

* again

* fix new errors

* fix copypasta

* also fix on sources

* fix references

* fix pools_tokens_weights

* update token_data for varbinary

* fix erc4626 token prices

* improvements for erc4626_token_prices

* replicate changes on gnosis

* again

* fix labels

* last fix erc4626 prices

* fix

* again

* fix§

* again

* fix group by

* add gnosis seed

* yay

* fix bpt_supply_macro

* fix liquidity and bpt_prices

* final round (hopefully)

* fix errors

* fix errors

* fix trades

* fix incremental errors

* update ethereum seed

* last tweaks

* gain

* again

* fix typo in schema name

* add incremental filter to prices

* add incremental filter to price join

---------

Co-authored-by: Huang Geyang <[email protected]>
Co-authored-by: jeff-dude <[email protected]>
  • Loading branch information
@viniabussafi @Hosuke @jeff-dude
3 people authored Dec 11, 2024
1 parent fd8b1de commit ed7bfd7
Show file tree
Hide file tree
Showing 92 changed files with 3,865 additions and 118 deletions.
275 changes: 275 additions & 0 deletions dbt_macros/shared/balancer/balancer_bpt_prices_macro.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -349,3 +349,278 @@ WITH pool_labels AS (
WHERE supply > 0

{% endmacro %}

{# ######################################################################### #}

{% macro
balancer_v3_compatible_bpt_prices_macro(
blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
)
%}

WITH pool_labels AS (
SELECT
address AS pool_id,
name AS pool_symbol,
pool_type
FROM {{ pool_labels_spell }}
WHERE blockchain = '{{blockchain}}'
),

token_data AS (
SELECT
pool,
ARRAY_AGG(FROM_HEX(json_extract_scalar(token, '$.token')) ORDER BY token_index) AS tokens
FROM (
SELECT
pool,
tokenConfig,
SEQUENCE(1, CARDINALITY(tokenConfig)) AS token_index_array
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_PoolRegistered') }}
) AS pool_data
CROSS JOIN UNNEST(tokenConfig, token_index_array) AS t(token, token_index)
GROUP BY 1
),

-- liquidity formulation, with a few simplifications, compared to liquidity spell

prices AS (
SELECT
date_trunc('day', minute) AS day,
contract_address AS token,
decimals,
AVG(price) AS price
FROM {{ source('prices', 'usd') }}
WHERE blockchain = '{{blockchain}}'
GROUP BY 1, 2, 3
),

erc4626_prices AS(
SELECT
date_trunc('day', minute) AS day,
wrapped_token AS token,
decimals,
APPROX_PERCENTILE(median_price, 0.5) AS price,
DATE_TRUNC ('day', next_change) AS next_change
FROM {{ ref('balancer_v3_erc4626_token_prices') }}
WHERE blockchain = '{{blockchain}}'
GROUP BY 1, 2, 3, 5
),

swaps_changes AS (
SELECT
day,
pool_id,
token,
SUM(COALESCE(delta, INT256 '0')) AS delta
FROM
(
SELECT
date_trunc('day', evt_block_time) AS day,
pool AS pool_id,
tokenIn AS token,
CAST(amountIn AS INT256) AS delta
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}

UNION ALL

SELECT
date_trunc('day', evt_block_time) AS day,
pool AS pool_id,
tokenOut AS token,
-CAST(amountOut AS INT256) AS delta
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
) swaps
GROUP BY 1, 2, 3
),

balance_changes AS(
SELECT
evt_block_time,
pool_id,
category,
deltas,
swapFeeAmountsRaw
FROM
(
SELECT
evt_block_time,
pool AS pool_id,
'add' AS category,
amountsAddedRaw AS deltas,
swapFeeAmountsRaw
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityAdded') }}

UNION ALL

SELECT
evt_block_time,
pool AS pool_id,
'remove' AS category,
amountsRemovedRaw AS deltas,
swapFeeAmountsRaw
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_LiquidityRemoved') }}
) adds_and_removes
),

zipped_balance_changes AS (
SELECT
date_trunc('day', evt_block_time) AS day,
pool_id,
t.tokens,
CASE WHEN b.category = 'add'
THEN d.deltas
WHEN b.category = 'remove'
THEN -d.deltas
END AS deltas,
p.swapFeeAmountsRaw
FROM balance_changes b
JOIN token_data td ON b.pool_id = td.pool
CROSS JOIN UNNEST (td.tokens) WITH ORDINALITY as t(tokens,i)
CROSS JOIN UNNEST (b.deltas) WITH ORDINALITY as d(deltas,i)
CROSS JOIN UNNEST (b.swapFeeAmountsRaw) WITH ORDINALITY as p(swapFeeAmountsRaw,i)
WHERE t.i = d.i
AND d.i = p.i
ORDER BY 1,2,3
),

balances_changes AS (
SELECT
day,
pool_id,
tokens AS token,
deltas - CAST(swapFeeAmountsRaw as int256) AS delta
FROM zipped_balance_changes
ORDER BY 1, 2, 3
),

daily_delta_balance AS (
SELECT
day,
pool_id,
token,
SUM(COALESCE(amount, INT256 '0')) AS amount
FROM
(
SELECT
day,
pool_id,
token,
SUM(COALESCE(delta, INT256 '0')) AS amount
FROM balances_changes
GROUP BY 1, 2, 3

UNION ALL

SELECT
day,
pool_id,
token,
delta AS amount
FROM
swaps_changes
) balance
GROUP BY 1, 2, 3
),


cumulative_balance AS (
SELECT
DAY,
pool_id,
token,
LEAD(DAY, 1, NOW()) OVER (PARTITION BY token, pool_id ORDER BY DAY) AS day_of_next_change,
SUM(amount) OVER (PARTITION BY pool_id, token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumulative_amount
FROM daily_delta_balance
),

calendar AS (
SELECT date_sequence AS day
FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence)
),

cumulative_usd_balance AS (
SELECT
c.day,
'{{blockchain}}' as blockchain,
b.pool_id,
b.token,
cumulative_amount / POWER(10, COALESCE(t.decimals, p1.decimals, p4.decimals)) * COALESCE(p1.price, p4.price, 0) AS protocol_liquidity_usd
FROM calendar c
LEFT JOIN cumulative_balance b ON b.day <= c.day
AND c.day < b.day_of_next_change
LEFT JOIN {{ source('tokens', 'erc20') }} t ON t.contract_address = b.token
AND blockchain = '{{blockchain}}'
LEFT JOIN prices p1 ON p1.day = b.day
AND p1.token = b.token
LEFT JOIN erc4626_prices p4 ON p4.day <= c.day
AND c.day < p4.next_change
AND p4.token = b.token
WHERE b.token != BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
),

weighted_pool_liquidity_estimates AS (
SELECT
b.day,
b.pool_id,
q.name,
pool_type,
ROW_NUMBER() OVER (partition by b.day, b.pool_id ORDER BY SUM(b.protocol_liquidity_usd) ASC) AS pricing_count, --to avoid double count in pools with multiple pricing assets
SUM(b.protocol_liquidity_usd) / COALESCE(SUM(w.normalized_weight), 1) AS protocol_liquidity
FROM cumulative_usd_balance b
LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id
AND b.token = w.token_address
AND b.protocol_liquidity_usd > 0
LEFT JOIN {{ source('balancer','token_whitelist') }} q ON b.token = q.address
AND b.blockchain = q.chain
LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(b.pool_id, 1, 20)
WHERE q.name IS NOT NULL
AND p.pool_type IN ('weighted') -- filters for weighted pools with pricing assets
AND w.blockchain = '{{blockchain}}'
AND w.version = '{{version}}'
GROUP BY 1, 2, 3, 4
),

weighted_pool_liquidity_estimates_2 AS(
SELECT e.day,
e.pool_id,
SUM(e.protocol_liquidity) / MAX(e.pricing_count) AS protocol_liquidity
FROM weighted_pool_liquidity_estimates e
GROUP BY 1,2
),

tvl AS(
SELECT
c.day,
BYTEARRAY_SUBSTRING(c.pool_id, 1, 20) AS pool_address,
'{{version}}' AS version,
'{{blockchain}}' AS blockchain,
SUM(COALESCE(b.protocol_liquidity * w.normalized_weight, c.protocol_liquidity_usd)) AS liquidity
FROM cumulative_usd_balance c
FULL OUTER JOIN weighted_pool_liquidity_estimates_2 b ON c.day = b.day
AND c.pool_id = b.pool_id
LEFT JOIN {{ ref(base_spells_namespace + '_pools_tokens_weights') }} w ON b.pool_id = w.pool_id
AND w.blockchain = '{{blockchain}}'
AND w.version = '{{version}}'
AND w.token_address = c.token
LEFT JOIN pool_labels p ON p.pool_id = BYTEARRAY_SUBSTRING(c.pool_id, 1, 20)
GROUP BY 1, 2, 3, 4
)

SELECT
l.day,
l.blockchain,
l.version,
18 AS decimals,
l.pool_address AS contract_address,
pl.pool_type,
l.liquidity / s.supply AS bpt_price
FROM tvl l
LEFT JOIN {{ ref(base_spells_namespace + '_bpt_supply') }} s ON l.pool_address = s.token_address
AND l.blockchain = s.blockchain
AND l.version = s.version
AND l.day = s.day
LEFT JOIN pool_labels pl ON pl.pool_id = l.pool_address
WHERE supply > 0

{% endmacro %}
100 changes: 100 additions & 0 deletions dbt_macros/shared/balancer/balancer_bpt_supply_macro.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,103 @@ WITH pool_labels AS (
HAVING SUM(b.supply - COALESCE(preminted_bpts, 0) + COALESCE(adelta, 0)) >= 0 --simple filter to remove outliers

{% endmacro %}

{# ######################################################################### #}

{% macro
balancer_v3_compatible_bpt_supply_macro(
blockchain, version, project_decoded_as, base_spells_namespace, pool_labels_spell
)
%}

WITH pool_labels AS (
SELECT * FROM (
SELECT
address,
name,
pool_type,
ROW_NUMBER() OVER (PARTITION BY address ORDER BY MAX(updated_at) DESC) AS num
FROM {{ pool_labels_spell }}
WHERE blockchain = '{{blockchain}}'
GROUP BY 1, 2, 3)
WHERE num = 1
),

-- Extract mints and burns from transfers
transfers AS (
SELECT
block_date AS day,
contract_address AS token,
COALESCE(SUM(CASE WHEN t."from" = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS mints,
COALESCE(SUM(CASE WHEN t.to = 0x0000000000000000000000000000000000000000 THEN value / POWER(10, 18) ELSE 0 END), 0) AS burns
FROM {{ ref(base_spells_namespace + '_transfers_bpt') }} t
WHERE blockchain = '{{blockchain}}'
AND version = '{{version}}'
GROUP BY 1, 2
),

-- Calculate token balances over time
balances AS (
SELECT
day,
token,
LEAD(DAY, 1, NOW()) OVER (PARTITION BY token ORDER BY DAY) AS day_of_next_change,
SUM(COALESCE(mints, 0) - COALESCE(burns, 0)) OVER (PARTITION BY token ORDER BY DAY ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS supply
FROM transfers
),

-- Calculating Joins(mint) and Exits(burn) via Swap
joins AS (
SELECT
DATE_TRUNC('day', evt_block_time) AS block_date,
tokenOut,
pool_type,
SUM(amountOut / POWER(10, 18)) AS ajoins
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
LEFT JOIN pool_labels ON pool = address
WHERE tokenOut = pool
GROUP BY 1, 2, 3
),

exits AS (
SELECT
DATE_TRUNC('day', evt_block_time) AS block_date,
tokenIn,
pool_type,
SUM(amountIn / POWER(10, 18)) AS aexits
FROM {{ source(project_decoded_as + '_' + blockchain, 'Vault_evt_Swap') }}
LEFT JOIN pool_labels ON pool = address
WHERE tokenIn = pool
GROUP BY 1, 2, 3
),

joins_and_exits AS (
SELECT
j.block_date,
j.tokenOut AS bpt,
SUM(COALESCE(ajoins, 0) - COALESCE(aexits, 0)) OVER (PARTITION BY j.tokenOut ORDER BY j.block_date ASC) AS adelta
FROM joins j
FULL OUTER JOIN exits e ON j.block_date = e.block_date AND e.tokenIn = j.tokenOut
),

calendar AS (
SELECT
date_sequence AS day
FROM unnest(sequence(date('2024-12-01'), date(now()), interval '1' day)) as t(date_sequence)
)

SELECT
c.day,
l.pool_type,
'{{version}}' as version,
'{{blockchain}}' as blockchain,
b.token AS token_address,
COALESCE(SUM(b.supply + COALESCE(adelta, 0)),0) AS supply
FROM calendar c
LEFT JOIN balances b ON b.day <= c.day AND c.day < b.day_of_next_change
LEFT JOIN joins_and_exits j ON c.day = j.block_date AND b.token = j.bpt
LEFT JOIN pool_labels l ON b.token = l.address
GROUP BY 1, 2, 3, 4, 5
HAVING SUM(b.supply + COALESCE(adelta, 0)) >= 0 --simple filter to remove outliers

{% endmacro %}
Loading

0 comments on commit ed7bfd7

Please sign in to comment.