diff --git a/tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h b/tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h index 2576df29c..bdbb7e007 100644 --- a/tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h +++ b/tests/benchmark/bm_initialization/bm_basics_svs_initialize_fp32.h @@ -40,3 +40,17 @@ BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_TriggerUpdateTiered)) {2, 4, 8}}) ->ArgNames({"update_threshold", "thread_count"}) ->MeasureProcessCPUTime(); + +// Add vectors to reach training threshold in tiered index, and then add more vectors in parallel to +// backend training job. Measure time to add new vectors in this scenario. +BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddVectorsDuringTraining), + DATA_TYPE_INDEX_T) +(benchmark::State &st) { AddVectorsDuringTraining(st); } +BENCHMARK_REGISTER_F(BM_VecSimSVS, BM_FUNC_NAME(BM_AddVectorsDuringTraining)) + ->Unit(benchmark::kMillisecond) + ->Iterations(1) + ->ArgsProduct({{static_cast(BM_VecSimGeneral::block_size), 5000, + static_cast(10 * BM_VecSimGeneral::block_size)}, + {2, 4}}) + ->ArgNames({"training_threshold", "thread_count"}) + ->UseRealTime(); diff --git a/tests/benchmark/bm_vecsim_svs.h b/tests/benchmark/bm_vecsim_svs.h index 71a0c9f1c..fe6c92493 100644 --- a/tests/benchmark/bm_vecsim_svs.h +++ b/tests/benchmark/bm_vecsim_svs.h @@ -47,6 +47,10 @@ class BM_VecSimSVS : public BM_VecSimGeneral { // index. void TriggerUpdateTiered(benchmark::State &st); + // Add vectors to reach training threshold in tiered index, and then add more vectors in + // parallel to backend training job. Measure time to add new vectors in this scenario. + void AddVectorsDuringTraining(benchmark::State &st); + // Deletes an amount of labels from the index that triggers inplace consolidation. void RunGC(benchmark::State &st); @@ -403,6 +407,44 @@ void BM_VecSimSVS::TriggerUpdateTiered(benchmark::State &st) { .str()); } +template +inline void BM_VecSimSVS::AddVectorsDuringTraining(benchmark::State &st) { + // ensure mode is async + ASSERT_EQ(VecSimIndexInterface::asyncWriteMode, VecSim_WriteAsync); + + auto training_threshold = st.range(0); + int unsigned num_threads = st.range(1); + + if (num_threads > std::thread::hardware_concurrency()) { + GTEST_SKIP() << "Not enough threads available, skipping test..."; + } + + // Ensure we have enough vectors to train. + ASSERT_GE(N_QUERIES, training_threshold + 1000); // need extra vectors to add during training + + // In each iteration create a new index + auto mock_thread_pool = tieredIndexMock(num_threads); + ASSERT_EQ(mock_thread_pool.thread_pool_size, num_threads); + auto *tiered_index = CreateTieredSVSIndex( + mock_thread_pool, training_threshold, + 1 << 30); // set very high update threshold to avoid updates during this test + + // Add vectors to reach training threshold and trigger training. + for (size_t i = 0; i < training_threshold; ++i) { + VecSimIndex_AddVector(tiered_index, test_vectors[i].data(), i); + } + mock_thread_pool.init_threads(); + size_t label = training_threshold; + + for (auto _ : st) { + // While the backend is training, keep adding vectors in parallel to the training job. + for (size_t i = 0; i < 1000 && label < N_QUERIES; i++, label++) { + VecSimIndex_AddVector(tiered_index, test_vectors[label].data(), label); + } + } + mock_thread_pool.thread_pool_join(); +} + template void BM_VecSimSVS::RunGC(benchmark::State &st) {