From 5bbc0fa9b56b9ba06e625eb42d9688f6c5df062c Mon Sep 17 00:00:00 2001 From: azurwastaken Date: Tue, 8 Oct 2024 19:40:11 +0200 Subject: [PATCH 01/10] Create devnet-script-dispatch.js --- .../pragma-devnet/devnet-script-dispatch.js | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 examples/pragma/pragma-devnet/devnet-script-dispatch.js diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js new file mode 100644 index 0000000..3cf1616 --- /dev/null +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -0,0 +1,116 @@ +import { hash, shortString } from "https://esm.run/starknet@5.14"; +import * as ethers from "https://esm.run/ethers"; + +const filter = { + // Only request header if any event matches. + header: { + weak: true, + }, + events: [ + { + fromAddress: + "0x41c20175af14a0bfebfc9ae2f3bda29230a0bceb551844197d9f46faf76d6da", + keys: [hash.getSelectorFromName("Dispatch")], + includeTransaction: true, + includeReceipt: false, + }, + ], +}; + +function escapeInvalidCharacters(str) { + return str.replace(/^[\x00-\x1F]+/, ""); +} + + +// number of udpate u16 (4 char) +// Unique() +// feedID u256 +// ts u64 +// num source u16 +// decimal u8 +// price u256 +// volume u256  +// 256 + 64 + 16 + 8 + 256 + 256 = 856 / 4 = 214 +function decodeHyperlaneMessageBody(hexData) { + const uniqueSize = 214; + let data = hexData.map(hex => { + // Remove '0x' prefix + let trimmed = hex.replace(/^0x/, ''); + + // Remove the first 32 characters + trimmed = trimmed.slice(32); + + // If the string became empty after processing, return an empty string + return trimmed === '' ? '' : trimmed; + }); + data = data.join(''); + let numberOfUpdate = Number(data.slice(0,4)); + data = data.slice(4); + // parse unique + let feedId = data.slice(0,64); + let timestamp = data.slice(64, 80); + let num_source = data.slice(80,84); + let decimal = data.slice(84,86); + let price = data.slice(86,86+64); + let feedTypeId = Number(data.slice(4,5)); + let sizeData = uniqueSize; //TODO conditioné par type de donné + + let res = { + feedId: feedId, + feedTypeId: feedTypeId, + sizeData : sizeData, + timestamp: timestamp, + num_source, + decimal, + price, + }; + // concat data + + console.log(res); +} + +function decodeTransfersInBlock({ header, events }) { + const { blockNumber, blockHash, timestamp } = header; + + return events.flatMap(({ event, transaction }) => { + const transactionHash = transaction.meta.hash; + + console.log(event.data); + + const nonce = event.data[6]; + + // retrieve body + // decode body + // recuperer tout les feedId a l'interieur + let messageBody = event.data.slice(15); + let decoded = decodeHyperlaneMessageBody(messageBody); + console.log(decoded); + // Convert to snake_case because it works better with postgres. + return { + network: "pragma-devnet", + block_hash: blockHash, + block_number: +blockNumber, + block_timestamp: timestamp, + transaction_hash: transactionHash, + }; + }); +} + +// Configure indexer for streaming PragmaGix data starting at the specified block. +export const config = { + streamUrl: "https://devnet.pragma.a5a.ch", + startingBlock: Number(220_840), + network: "starknet", + filter, + batchSize: 1, + finality: "DATA_STATUS_PENDING", + sinkType: "console", + sinkOptions: { + // Send data as returned by `transform`. + // When `raw = false`, the data is sent together with the starting and end cursor. + raw: true, + }, +}; + +// Transform each block using the function defined in starknet.js. +export default decodeTransfersInBlock; From 1d85f289e8efc2475029681a6e4ed4ae02c6119b Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:31:34 +0200 Subject: [PATCH 02/10] feat(handle-pragma-devnet): Indexing --- .../pragma-devnet/devnet-script-dispatch.js | 106 ++++++++---------- 1 file changed, 46 insertions(+), 60 deletions(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 3cf1616..3633ed2 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -1,5 +1,8 @@ -import { hash, shortString } from "https://esm.run/starknet@5.14"; -import * as ethers from "https://esm.run/ethers"; +import { hash } from "https://esm.run/starknet@5.14"; + +const HYPERLANE_MAILBOX_CONTRACT = + "0x41c20175af14a0bfebfc9ae2f3bda29230a0bceb551844197d9f46faf76d6da"; +const FEED_ID_SIZE = 64; const filter = { // Only request header if any event matches. @@ -8,8 +11,7 @@ const filter = { }, events: [ { - fromAddress: - "0x41c20175af14a0bfebfc9ae2f3bda29230a0bceb551844197d9f46faf76d6da", + fromAddress: HYPERLANE_MAILBOX_CONTRACT, keys: [hash.getSelectorFromName("Dispatch")], includeTransaction: true, includeReceipt: false, @@ -17,81 +19,65 @@ const filter = { ], }; -function escapeInvalidCharacters(str) { - return str.replace(/^[\x00-\x1F]+/, ""); +function decodeFeedId(feedIdHex) { + const feedId = BigInt(`0x${feedIdHex}`); + const assetClass = Number((feedId >> BigInt(232)) & BigInt(0xffff)); + const feedType = Number((feedId >> BigInt(216)) & BigInt(0xffff)); + const pairId = feedId & BigInt((1n << 216n) - 1n); + + return { assetClass, feedType, pairId }; } +function getFeedSize(feedType) { + const mainType = feedType >> 8; + switch (mainType) { + case 0: // Unique + return 214; // 856 bits / 4 = 214 hex characters + case 1: // Twap + return 470; // 1880 bits / 4 = 470 hex characters + default: + throw new Error(`Unknown feed type: ${feedType}`); + } +} -// number of udpate u16 (4 char) -// Unique() -// feedID u256 -// ts u64 -// num source u16 -// decimal u8 -// price u256 -// volume u256  -// 256 + 64 + 16 + 8 + 256 + 256 = 856 / 4 = 214 -function decodeHyperlaneMessageBody(hexData) { - const uniqueSize = 214; - let data = hexData.map(hex => { - // Remove '0x' prefix - let trimmed = hex.replace(/^0x/, ''); - - // Remove the first 32 characters - trimmed = trimmed.slice(32); - - // If the string became empty after processing, return an empty string - return trimmed === '' ? '' : trimmed; +function decodeFeedsUpdatedFromHyperlaneMessage(hexData) { + let data = hexData.map((hex) => { + let trimmed = hex.replace(/^0x/, ""); + trimmed = trimmed.slice(32); + return trimmed === "" ? "" : trimmed; }); - data = data.join(''); - let numberOfUpdate = Number(data.slice(0,4)); - data = data.slice(4); - // parse unique - let feedId = data.slice(0,64); - let timestamp = data.slice(64, 80); - let num_source = data.slice(80,84); - let decimal = data.slice(84,86); - let price = data.slice(86,86+64); - let feedTypeId = Number(data.slice(4,5)); - let sizeData = uniqueSize; //TODO conditioné par type de donné + data = data.join(""); - let res = { - feedId: feedId, - feedTypeId: feedTypeId, - sizeData : sizeData, - timestamp: timestamp, - num_source, - decimal, - price, - }; - // concat data + const numberOfUpdates = Number(data.slice(0, 4)); + data = data.slice(4); - console.log(res); + const feedIdsUpdated = []; + for (let i = 0; i < numberOfUpdates; i++) { + const feedIdHex = data.slice(0, FEED_ID_SIZE); + const { feedType } = decodeFeedId(feedIdHex); + feedIdsUpdated.push(`0x${feedIdHex}`); + data = data.slice(getFeedSize(feedType)); + } + return feedIdsUpdated; } -function decodeTransfersInBlock({ header, events }) { +export function decodeTransfersInBlock({ header, events }) { const { blockNumber, blockHash, timestamp } = header; - + return events.flatMap(({ event, transaction }) => { const transactionHash = transaction.meta.hash; - - console.log(event.data); - const nonce = event.data[6]; - - // retrieve body - // decode body - // recuperer tout les feedId a l'interieur let messageBody = event.data.slice(15); - let decoded = decodeHyperlaneMessageBody(messageBody); - console.log(decoded); - // Convert to snake_case because it works better with postgres. + let feedsUpdated = decodeFeedsUpdatedFromHyperlaneMessage(messageBody); + return { network: "pragma-devnet", block_hash: blockHash, block_number: +blockNumber, block_timestamp: timestamp, transaction_hash: transactionHash, + nonce: nonce, + feeds_updated: feedsUpdated, }; }); } From de1c556c90fe710e70ffa3ce9e50b0dd2754e72f Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:32:37 +0200 Subject: [PATCH 03/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 4 ++-- examples/pragma/pragma-devnet/devnet-script-spot.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 3633ed2..1ab7c67 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -85,12 +85,12 @@ export function decodeTransfersInBlock({ header, events }) { // Configure indexer for streaming PragmaGix data starting at the specified block. export const config = { streamUrl: "https://devnet.pragma.a5a.ch", - startingBlock: Number(220_840), + startingBlock: Number(0), network: "starknet", filter, batchSize: 1, finality: "DATA_STATUS_PENDING", - sinkType: "console", + sinkType: "postgres", sinkOptions: { // Send data as returned by `transform`. // When `raw = false`, the data is sent together with the starting and end cursor. diff --git a/examples/pragma/pragma-devnet/devnet-script-spot.js b/examples/pragma/pragma-devnet/devnet-script-spot.js index 68898a2..6147e89 100644 --- a/examples/pragma/pragma-devnet/devnet-script-spot.js +++ b/examples/pragma/pragma-devnet/devnet-script-spot.js @@ -60,7 +60,7 @@ function decodeTransfersInBlock({ header, events }) { }); } -// Configure indexer for streaming Starknet Goerli data starting at the specified block. +// Configure indexer for streaming PragmaGix data starting at the specified block. export const config = { streamUrl: "https://devnet.pragma.a5a.ch", startingBlock: Number(0), From 9513501bf183c652d06f1ca3589be005d1ddd0c1 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:33:56 +0200 Subject: [PATCH 04/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 1ab7c67..31cc341 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -66,7 +66,7 @@ export function decodeTransfersInBlock({ header, events }) { return events.flatMap(({ event, transaction }) => { const transactionHash = transaction.meta.hash; - const nonce = event.data[6]; + const hyperlaneMessageNonce = event.data[6]; let messageBody = event.data.slice(15); let feedsUpdated = decodeFeedsUpdatedFromHyperlaneMessage(messageBody); @@ -76,7 +76,7 @@ export function decodeTransfersInBlock({ header, events }) { block_number: +blockNumber, block_timestamp: timestamp, transaction_hash: transactionHash, - nonce: nonce, + hyperlane_message_nonce: hyperlaneMessageNonce, feeds_updated: feedsUpdated, }; }); From 83c29fb66e0906dfff90bcad71b09f83e1c144a1 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:39:55 +0200 Subject: [PATCH 05/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 31cc341..fb501e3 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -85,7 +85,8 @@ export function decodeTransfersInBlock({ header, events }) { // Configure indexer for streaming PragmaGix data starting at the specified block. export const config = { streamUrl: "https://devnet.pragma.a5a.ch", - startingBlock: Number(0), + // We don't have any Dispatch before the 180_000'th block + startingBlock: Number(180_000), network: "starknet", filter, batchSize: 1, From af2e41056a85d3783a861b879f3832084947111f Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:46:40 +0200 Subject: [PATCH 06/10] feat(handle-pragma-devnet): --- .../pragma-devnet/devnet-script-dispatch.js | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index fb501e3..c7b7b79 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -28,15 +28,20 @@ function decodeFeedId(feedIdHex) { return { assetClass, feedType, pairId }; } -function getFeedSize(feedType) { +function getFeedSize(assetClass, feedType) { const mainType = feedType >> 8; - switch (mainType) { - case 0: // Unique - return 214; // 856 bits / 4 = 214 hex characters - case 1: // Twap - return 470; // 1880 bits / 4 = 470 hex characters + switch (assetClass) { + case 0: // Crypto + switch (mainType) { + case 0: // Unique + return 214; // 856 bits / 4 = 214 hex characters + case 1: // Twap + return 470; // 1880 bits / 4 = 470 hex characters + default: + throw new Error(`Unknown feed type: ${feedType}`); + } default: - throw new Error(`Unknown feed type: ${feedType}`); + throw new Error(`Unknown asset class: ${feedType}`); } } @@ -54,9 +59,9 @@ function decodeFeedsUpdatedFromHyperlaneMessage(hexData) { const feedIdsUpdated = []; for (let i = 0; i < numberOfUpdates; i++) { const feedIdHex = data.slice(0, FEED_ID_SIZE); - const { feedType } = decodeFeedId(feedIdHex); + const { assetClass, feedType } = decodeFeedId(feedIdHex); feedIdsUpdated.push(`0x${feedIdHex}`); - data = data.slice(getFeedSize(feedType)); + data = data.slice(getFeedSize(assetClass, feedType)); } return feedIdsUpdated; } @@ -91,7 +96,7 @@ export const config = { filter, batchSize: 1, finality: "DATA_STATUS_PENDING", - sinkType: "postgres", + sinkType: "console", sinkOptions: { // Send data as returned by `transform`. // When `raw = false`, the data is sent together with the starting and end cursor. From a9149fbb20d64af2a37f18d8f67abfb20a280cc4 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:48:43 +0200 Subject: [PATCH 07/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index c7b7b79..01850a4 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -72,8 +72,8 @@ export function decodeTransfersInBlock({ header, events }) { return events.flatMap(({ event, transaction }) => { const transactionHash = transaction.meta.hash; const hyperlaneMessageNonce = event.data[6]; - let messageBody = event.data.slice(15); - let feedsUpdated = decodeFeedsUpdatedFromHyperlaneMessage(messageBody); + const messageBody = event.data.slice(15); + const feedsUpdated = decodeFeedsUpdatedFromHyperlaneMessage(messageBody); return { network: "pragma-devnet", From b8ae6c679fba9e58cee2658f1ddeddc129702057 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:48:56 +0200 Subject: [PATCH 08/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 01850a4..40727c6 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -96,7 +96,7 @@ export const config = { filter, batchSize: 1, finality: "DATA_STATUS_PENDING", - sinkType: "console", + sinkType: "postgres", sinkOptions: { // Send data as returned by `transform`. // When `raw = false`, the data is sent together with the starting and end cursor. From b1cc81c89f8a3d3c2d33354bd91aa7bb5b6a40a7 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 04:51:07 +0200 Subject: [PATCH 09/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 40727c6..3ea63c0 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -71,7 +71,7 @@ export function decodeTransfersInBlock({ header, events }) { return events.flatMap(({ event, transaction }) => { const transactionHash = transaction.meta.hash; - const hyperlaneMessageNonce = event.data[6]; + const hyperlaneMessageNonce = parseInt(event.data[6], 16); const messageBody = event.data.slice(15); const feedsUpdated = decodeFeedsUpdatedFromHyperlaneMessage(messageBody); From ed5a3d5b0f7b506a06647a6ce2e59ad855aa7a05 Mon Sep 17 00:00:00 2001 From: akhercha Date: Wed, 9 Oct 2024 05:52:02 +0200 Subject: [PATCH 10/10] feat(handle-pragma-devnet): --- examples/pragma/pragma-devnet/devnet-script-dispatch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pragma/pragma-devnet/devnet-script-dispatch.js b/examples/pragma/pragma-devnet/devnet-script-dispatch.js index 3ea63c0..b0ca580 100644 --- a/examples/pragma/pragma-devnet/devnet-script-dispatch.js +++ b/examples/pragma/pragma-devnet/devnet-script-dispatch.js @@ -41,7 +41,7 @@ function getFeedSize(assetClass, feedType) { throw new Error(`Unknown feed type: ${feedType}`); } default: - throw new Error(`Unknown asset class: ${feedType}`); + throw new Error(`Unknown asset class: ${assetClass}`); } }