Fix pagination (#244)

* Improve tests

* Fix pagination

* Fix pagination

* Add tests and fix pagination

Cargo.toml

@@ -7,7 +7,7 @@ name = "photon-indexer"
 publish = true
 readme = "README.md"
 repository = "https://github.com/helius-labs/photon"
-version = "0.46.0"
+version = "0.47.0"
 
 [[bin]]
 name = "photon"

src/api/method/get_compressed_mint_token_holders.rs

@@ -50,11 +50,12 @@ pub async fn get_compressed_mint_token_holders(
         limit,
     } = request;
     let mut filter = token_owner_balances::Column::Mint.eq::<Vec<u8>>(mint.into());
+
     if let Some(cursor) = cursor {
         let bytes = cursor.0;
         let expected_cursor_length = 40;
         let (balance, owner) = if bytes.len() == expected_cursor_length {
-            let (balance, owner) = bytes.split_at(expected_cursor_length);
+            let (balance, owner) = bytes.split_at(8);
             (balance, owner)
         } else {
             return Err(PhotonApiError::ValidationError(format!(
@@ -64,11 +65,12 @@ pub async fn get_compressed_mint_token_holders(
             )));
         };
         let balance = LittleEndian::read_u64(&balance);
+
         filter = filter.and(
             token_owner_balances::Column::Amount.lt(balance).or(
                 token_owner_balances::Column::Amount
                     .eq(balance)
-                    .and(token_owner_balances::Column::Owner.gt::<Vec<u8>>(owner.into())),
+                    .and(token_owner_balances::Column::Owner.lt::<Vec<u8>>(owner.into())),
             ),
         );
     }
@@ -77,7 +79,7 @@ pub async fn get_compressed_mint_token_holders(
     let items = token_owner_balances::Entity::find()
         .filter(filter)
         .order_by_desc(token_owner_balances::Column::Amount)
-        .order_by_asc(token_owner_balances::Column::Mint)
+        .order_by_desc(token_owner_balances::Column::Owner)
         .limit(limit)
         .all(conn)
         .await?

tests/integration_tests/mock_tests.rs

@@ -334,6 +334,8 @@ async fn test_persist_token_data(
     let mint3 = SerializablePubkey::new_unique();
     let owner1 = SerializablePubkey::new_unique();
     let owner2 = SerializablePubkey::new_unique();
+    let owner3 = SerializablePubkey::new_unique();
+    let owner4 = SerializablePubkey::new_unique();
     let delegate1 = SerializablePubkey::new_unique();
     let delegate2 = SerializablePubkey::new_unique();
 
@@ -385,10 +387,56 @@ async fn test_persist_token_data(
         state: AccountState::frozen,
         tlv: None,
     };
-    let all_token_data = vec![token_data1, token_data2, token_data3, token_data4];
+    let token_data5 = TokenData {
+        mint: mint1,
+        owner: owner3,
+        amount: UnsignedInteger(4),
+        delegate: Some(delegate1),
+        state: AccountState::frozen,
+        tlv: None,
+    };
+    let token_data6 = TokenData {
+        mint: mint1,
+        owner: owner4,
+        amount: UnsignedInteger(6),
+        delegate: Some(delegate1),
+        state: AccountState::frozen,
+        tlv: None,
+    };
+    let token_data7 = TokenData {
+        mint: mint1,
+        owner: owner2,
+        amount: UnsignedInteger(4),
+        delegate: Some(delegate1),
+        state: AccountState::frozen,
+        tlv: None,
+    };
+    let all_token_data = vec![
+        token_data1,
+        token_data2,
+        token_data3,
+        token_data4,
+        token_data5,
+        token_data6,
+        token_data7,
+    ];
+    let hashes = all_token_data
+        .iter()
+        .map(|_| Hash::new_unique())
+        .collect::<HashSet<Hash>>();
+
+    let all_token_data: Vec<TokenDataWithHash> = all_token_data
+        .iter()
+        .zip(hashes.iter())
+        .map(|(token_data, hash)| TokenDataWithHash {
+            token_data: token_data.clone(),
+            hash: hash.clone(),
+        })
+        .collect();
 
     let mut mint_to_owner_to_balance = HashMap::new();
     for token_data in all_token_data.clone() {
+        let token_data = token_data.token_data;
         let mint = token_data.mint;
         let owner = token_data.owner;
         let mint_owner_balances = mint_to_owner_to_balance
@@ -406,7 +454,8 @@ async fn test_persist_token_data(
 
     for (i, token_data) in all_token_data.iter().enumerate() {
         let slot = 11;
-        let hash = Hash::new_unique();
+        let hash = token_data.hash.clone();
+        let token_data = token_data.token_data.clone();
         let model = accounts::ActiveModel {
             hash: Set(hash.clone().into()),
             address: Set(Some(Pubkey::new_unique().to_bytes().to_vec())),
@@ -434,7 +483,7 @@ async fn test_persist_token_data(
 
     let owner_tlv = all_token_data
         .iter()
-        .filter(|x| x.owner == owner1 && x.mint == mint1)
+        .filter(|x| x.token_data.owner == owner1 && x.token_data.mint == mint1)
         .map(Clone::clone)
         .collect();
 
@@ -448,12 +497,12 @@ async fn test_persist_token_data(
         .await
         .unwrap()
         .value;
-    verify_responses_match_tlv_data(res.clone(), owner_tlv);
+    verify_response_matches_input_token_data(res.clone(), owner_tlv);
 
     for owner in [owner2] {
         let owner_tlv = all_token_data
             .iter()
-            .filter(|x| x.owner == owner)
+            .filter(|x| x.token_data.owner == owner)
             .map(Clone::clone)
             .collect();
         let res = setup
@@ -512,7 +561,7 @@ async fn test_persist_token_data(
             assert_eq!(res.token_balances[0].balance.0, *balance);
         }
 
-        verify_responses_match_tlv_data(res.clone(), owner_tlv);
+        verify_response_matches_input_token_data(res.clone(), owner_tlv);
         for token_account in res.items {
             let request = CompressedAccountRequest {
                 address: None,
@@ -532,7 +581,7 @@ async fn test_persist_token_data(
         let delegate_tlv = all_token_data
             .clone()
             .into_iter()
-            .filter(|x| x.delegate == Some(delegate))
+            .filter(|x| x.token_data.delegate == Some(delegate))
             .collect();
         let res = setup
             .api
@@ -565,7 +614,7 @@ async fn test_persist_token_data(
             }
         }
         assert_eq!(paginated_res, res.items);
-        verify_responses_match_tlv_data(res, delegate_tlv)
+        verify_response_matches_input_token_data(res, delegate_tlv);
     }
 
     for (mint, owner_to_balance) in mint_to_owner_to_balance.iter() {
@@ -577,7 +626,7 @@ async fn test_persist_token_data(
                 .api
                 .get_compressed_mint_token_holders(GetCompressedMintTokenHoldersRequest {
                     mint: mint.clone(),
-                    limit: Some(photon_indexer::api::method::utils::Limit::new(100).unwrap()),
+                    limit: Some(photon_indexer::api::method::utils::Limit::new(1).unwrap()),
                     cursor,
                 })
                 .await

tests/integration_tests/utils.rs

@@ -1,5 +1,8 @@
 use std::{env, path::Path, str::FromStr, sync::Mutex};
 
+use once_cell::sync::Lazy;
+use photon_indexer::common::typedefs::hash::Hash;
+use photon_indexer::migration::{Migrator, MigratorTrait};
 use photon_indexer::{
     api::{api::PhotonApi, method::utils::TokenAccountList},
     common::{
@@ -12,9 +15,6 @@ use photon_indexer::{
         typedefs::block_info::{parse_ui_confirmed_blocked, BlockInfo, TransactionInfo},
     },
 };
-
-use once_cell::sync::Lazy;
-use photon_indexer::migration::{Migrator, MigratorTrait};
 pub use sea_orm::DatabaseBackend;
 use sea_orm::{
     ConnectionTrait, DatabaseConnection, DbBackend, DbErr, ExecResult, SqlxPostgresConnector,
@@ -306,20 +306,27 @@ pub fn trim_test_name(name: &str) -> String {
         .to_string()
 }
 
-fn order_token_datas(mut token_datas: Vec<TokenData>) -> Vec<TokenData> {
-    let mut without_duplicates = token_datas.clone();
-    without_duplicates.dedup_by(|a, b| a.mint == b.mint);
-    if without_duplicates.len() != token_datas.len() {
-        panic!(
-            "Duplicate mint in token_datas: {:?}. Need hashes to further order token tlv data.",
-            token_datas
-        );
-    }
-    token_datas.sort_by(|a, b| a.mint.0.cmp(&b.mint.0));
+#[derive(Clone, Debug)]
+pub struct TokenDataWithHash {
+    pub token_data: TokenData,
+    pub hash: Hash,
+}
+
+fn order_token_datas(mut token_datas: Vec<TokenDataWithHash>) -> Vec<TokenDataWithHash> {
+    token_datas.sort_by(|a, b| {
+        a.token_data
+            .mint
+            .0
+            .cmp(&b.token_data.mint.0)
+            .then_with(|| a.hash.to_vec().cmp(&b.hash.to_vec()))
+    });
     token_datas
 }
 
-pub fn verify_responses_match_tlv_data(response: TokenAccountList, tlvs: Vec<TokenData>) {
+pub fn verify_response_matches_input_token_data(
+    response: TokenAccountList,
+    tlvs: Vec<TokenDataWithHash>,
+) {
     if response.items.len() != tlvs.len() {
         panic!(
             "Mismatch in number of accounts. Expected: {}, Actual: {}",
@@ -331,10 +338,13 @@ pub fn verify_responses_match_tlv_data(response: TokenAccountList, tlvs: Vec<Tok
     let token_accounts = response.items;
     for (account, tlv) in token_accounts.iter().zip(order_token_datas(tlvs).iter()) {
         let account = account.clone();
-        assert_eq!(account.token_data.mint, tlv.mint);
-        assert_eq!(account.token_data.owner, tlv.owner);
-        assert_eq!(account.token_data.amount, tlv.amount);
-        assert_eq!(account.token_data.delegate, tlv.delegate.map(Into::into));
+        assert_eq!(account.token_data.mint, tlv.token_data.mint);
+        assert_eq!(account.token_data.owner, tlv.token_data.owner);
+        assert_eq!(account.token_data.amount, tlv.token_data.amount);
+        assert_eq!(
+            account.token_data.delegate,
+            tlv.token_data.delegate.map(Into::into)
+        );
     }
 }
 pub fn assert_account_response_list_matches_input(