Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Exp: Early Stoppage #605

Draft
wants to merge 10 commits into
base: main
Choose a base branch
from
Draft

Exp: Early Stoppage #605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
868ece9
add distance based stoppage
Nov 12, 2024
c0fd427
update early stoppage
Nov 21, 2024
f063269
add early stop results
Nov 21, 2024
08aa75d
add early stop results
Nov 25, 2024
1f66698
add some results
Dec 3, 2024
0c83bb9
update reruns
Dec 3, 2024
8c4d91f
remove results
Dec 3, 2024
7c9521f
add more results
Dec 4, 2024
5dc2eea
add plots
Dec 6, 2024
131cf19
Merge branch 'microsoft:main' into amey-earlystop
yemaedahrav Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions apps/search_memory_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,9 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
}
else
{
std::cout << std::setw(4) << "Ls" << std::setw(12) << qps_title << std::setw(18) << "Avg dist cmps"
std::cout << std::setw(4) << "Ls" << std::setw(12) << qps_title << std::setw(18) << "Avg dist cmps" << std::setw(18) << "Avg hops"
<< std::setw(20) << "Mean Latency (mus)" << std::setw(15) << "99.9 Latency";
table_width += 4 + 12 + 18 + 20 + 15;
table_width += 4 + 12 + 18 + 18+ 20 + 15;
}
uint32_t recalls_to_print = 0;
const uint32_t first_recall = print_all_recalls ? 1 : recall_at;
Expand All @@ -131,9 +131,11 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
std::vector<std::vector<float>> query_result_dists(Lvec.size());
std::vector<float> latency_stats(query_num, 0);
std::vector<uint32_t> cmp_stats;
std::vector<uint32_t> hop_stats;
if (not tags || filtered_search)
{
cmp_stats = std::vector<uint32_t>(query_num, 0);
hop_stats = std::vector<uint32_t>(query_num, 0);
}

std::vector<TagT> query_result_tags;
Expand Down Expand Up @@ -171,6 +173,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
query_result_ids[test_id].data() + i * recall_at,
query_result_dists[test_id].data() + i * recall_at);
cmp_stats[i] = retval.second;
hop_stats[i] = retval.first;
}
else if (metric == diskann::FAST_L2)
{
Expand Down Expand Up @@ -203,6 +206,10 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
->search(query + i * query_aligned_dim, recall_at, L,
query_result_ids[test_id].data() + i * recall_at)
.second;
hop_stats[i] = index
->search(query + i * query_aligned_dim, recall_at, L,
query_result_ids[test_id].data() + i * recall_at)
.first;
}
auto qe = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> diff = qe - qs;
Expand Down Expand Up @@ -231,6 +238,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
std::accumulate(latency_stats.begin(), latency_stats.end(), 0.0) / static_cast<float>(query_num);

float avg_cmps = (float)std::accumulate(cmp_stats.begin(), cmp_stats.end(), 0) / (float)query_num;
float avg_hops = (float)std::accumulate(hop_stats.begin(), hop_stats.end(), 0) / (float)query_num;

if (tags && !filtered_search)
{
Expand All @@ -239,7 +247,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
}
else
{
std::cout << std::setw(4) << L << std::setw(12) << displayed_qps << std::setw(18) << avg_cmps
std::cout << std::setw(4) << L << std::setw(12) << displayed_qps << std::setw(18) << avg_cmps << std::setw(18) << avg_hops
<< std::setw(20) << (float)mean_latency << std::setw(15)
<< (float)latency_stats[(uint64_t)(0.999 * query_num)];
}
Expand Down
2 changes: 1 addition & 1 deletion include/index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ template <typename T, typename TagT = uint32_t, typename LabelT = uint32_t> clas

// The query to use is placed in scratch->aligned_query
std::pair<uint32_t, uint32_t> iterate_to_fixed_point(InMemQueryScratch<T> *scratch, const uint32_t Lindex,
const std::vector<uint32_t> &init_ids, bool use_filter,
const size_t K, const std::vector<uint32_t> &init_ids, bool use_filter,
const std::vector<LabelT> &filters, bool search_invocation);

void search_for_point_and_prune(int location, uint32_t Lindex, std::vector<uint32_t> &pruned_list,
Expand Down
32 changes: 31 additions & 1 deletion include/neighbor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class NeighborPriorityQueue
// item in the set. The set cursor that is used to pop() the
// next item will be set to the lowest index of an uncheck item
void insert(const Neighbor &nbr)
{
{
if (_size == _capacity && _data[_size - 1] < nbr)
{
return;
Expand Down Expand Up @@ -93,6 +93,36 @@ class NeighborPriorityQueue
}
}

int check_in_top_k(const Neighbor &nbr, size_t K)
{
if (_size == _capacity && _data[_size - 1] < nbr)
{
return 0;
}

size_t lo = 0, hi = _size;
while (lo < hi)
{
size_t mid = (lo + hi) >> 1;
if (nbr < _data[mid])
{
hi = mid;
}
else if (_data[mid].id == nbr.id)
{
return 0;
}
else
{
lo = mid + 1;
}
}
// if(lo < K){
// return 1;
// }
return 1;
}

Neighbor closest_unexpanded()
{
_data[_cur].expanded = true;
Expand Down
84 changes: 84 additions & 0 deletions old_baseline/baseline_r64_k10.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/gt100_prec40M ...
Metadata: #pts = 13265, #dims = 100...
Using AVX2 distance computation DistanceL2Int8.
Resizing took: 1.40021s
From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0
Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341
Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@10
=================================================================================
10 51557.93 1324.67 913.18 9792.77 43.13
20 33640.81 2021.14 1420.26 3249.95 58.38
30 25210.59 2617.97 1896.59 4469.27 66.24
40 20547.19 3168.06 2326.82 5450.12 71.25
50 17328.33 3687.22 2760.19 6599.82 74.44
60 14980.50 4195.96 3193.32 7739.67 76.98
70 13341.03 4685.40 3588.47 8306.07 78.78
80 11462.47 5171.15 4176.58 11172.33 80.21
90 10723.69 5654.48 4462.36 10865.74 81.53
100 9782.57 6129.37 4892.21 12162.22 82.58
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_10_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_10_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_20_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_20_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 10, size = 530608B
Finished writing bin.
21 changes: 21 additions & 0 deletions old_baseline/baseline_r64_k100.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/gt100_prec40M ...
Metadata: #pts = 13265, #dims = 100...
Using AVX2 distance computation DistanceL2Int8.
Resizing took: 1.40809s
From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0
Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341
Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@100
=================================================================================
100 10331.70 6129.37 4626.05 11214.14 66.81
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 100, size = 5306008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 100, size = 5306008B
Finished writing bin.
70 changes: 70 additions & 0 deletions old_baseline/baseline_r64_k25.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/gt100_prec40M ...
Metadata: #pts = 13265, #dims = 100...
Using AVX2 distance computation DistanceL2Int8.
Resizing took: 1.48129s
From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0
Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341
Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609
Index loaded
Using 48 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@25
=================================================================================
30 25253.53 2617.97 1885.01 10497.99 60.18
40 20654.60 3168.06 2316.21 5201.40 65.38
50 17413.17 3687.22 2747.47 6720.12 68.82
60 15010.45 4195.96 3187.87 7331.08 71.60
70 13300.10 4685.40 3599.56 8146.48 73.66
80 11990.02 5171.15 3991.75 8909.45 75.35
90 10918.86 5654.48 4385.05 9846.92 76.87
100 9976.54 6129.37 4799.29 10763.89 78.14
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_30_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_30_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_40_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_40_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 25, size = 1326508B
Finished writing bin.
56 changes: 56 additions & 0 deletions old_baseline/baseline_r64_k50.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
Reading (with alignment) bin file /nvmessd1/fbv4/queries384d.bin ...Metadata: #pts = 13265, #dims = 384, aligned_dim = 384... allocating aligned memory of 5093760 bytes... done. Copying data to mem_aligned buffer... done.
Opened: /nvmessd1/fbv4/gt100_prec40M, size: 10612008, cache_size: 10612008
Reading truthset file /nvmessd1/fbv4/gt100_prec40M ...
Metadata: #pts = 13265, #dims = 100...
Using AVX2 distance computation DistanceL2Int8.
Resizing took: 1.39346s
From graph header, expected_file_size: 7438757948, _max_observed_degree: 64, _start: 11890341, file_frozen_pts: 0
Loading vamana graph /nvmessd1/fbv4/avarhade/prec40M_memory_index_r64_l100.......done. Index has 40469609 nodes and 1819219872 out-edges, _start is set to 11890341
Num frozen points:0 _nd: 40469609 _start: 11890341 size(_location_to_tag): 0 size(_tag_to_location):0 Max points: 40469609
Index loaded
Using 1 threads to search
Ls QPS Avg dist cmps Mean Latency (mus) 99.9 Latency Recall@50
=================================================================================
50 750.71 3687.22 1332.03 15081.17 62.96
60 664.89 4195.96 1503.96 13729.61 65.92
70 595.61 4685.40 1678.89 12513.98 68.19
80 539.72 5171.15 1852.77 12737.32 70.07
90 493.28 5654.48 2027.21 12803.06 71.76
100 453.69 6129.37 2204.12 15871.16 73.17
Done searching. Now saving results
Writing bin: /home/rakri/avarhade/Dump/tmp_50_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_50_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_60_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_70_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_80_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_90_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_idx_uint32.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Writing bin: /home/rakri/avarhade/Dump/tmp_100_dists_float.bin
bin: #pts = 13265, #dims = 50, size = 2653008B
Finished writing bin.
Loading
Loading