[TST] Move rust benchmark crate to test crate (#2900)

## Description of changes

*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
	 - Move the dataset crate to a new test crate that aims to contain all relevant information for testing.
 - New functionality
	 - N/A

## Test plan
*How are these changes tested?*

- [x] Tests pass locally with `pytest` for python, `yarn test` for js, `cargo test` for rust

## Documentation Changes
*Are all docstrings for user-facing APIs updated if required? Do we need to make documentation changes in the [docs repository](https://github.com/chroma-core/docs)?*
N/A

Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 
 members = [
-   "rust/benchmark-datasets",
    "rust/blockstore",
    "rust/cache",
    "rust/chroma",
@@ -13,6 +12,7 @@ members = [
    "rust/storage",
    "rust/types",
    "rust/worker",
+   "rust/test",
 ]
 
 [workspace.dependencies]
@@ -44,7 +44,7 @@ chroma-cache = { path = "rust/cache" }
 chroma-types = { path = "rust/types" }
 chroma-index = { path = "rust/index" }
 chroma-distance = { path = "rust/distance" }
-chroma-benchmark-datasets = { path = "rust/benchmark-datasets" }
+chroma-test = { path = "rust/test" }
 worker = { path = "rust/worker" }
 
 # Dev dependencies

rust/index/Cargo.toml

@@ -32,7 +32,7 @@ chroma-distance = { workspace = true }
 tempfile = { workspace = true }
 rayon = { workspace = true }
 criterion = { workspace = true }
-chroma-benchmark-datasets = { workspace = true }
+chroma-test = { workspace = true }
 indicatif = "0.17.8"
 anyhow = "1.0.89"
 

rust/index/benches/dataset_utilities.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use chroma_benchmark_datasets::types::{FrozenQuerySubset, QueryDataset, RecordDataset};
+use chroma_test::datasets::types::{FrozenQuerySubset, QueryDataset, RecordDataset};
 use futures::TryFutureExt;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 

rust/index/benches/full_text.rs

@@ -1,10 +1,6 @@
 use std::sync::Arc;
 
 use anyhow::Result;
-use chroma_benchmark_datasets::{
-    datasets::{ms_marco_queries::MicrosoftMarcoQueriesDataset, scidocs::SciDocsDataset},
-    types::RecordDataset,
-};
 use chroma_blockstore::{arrow::provider::ArrowBlockfileProvider, provider::BlockfileProvider};
 use chroma_cache::{
     cache::Cache,
@@ -15,6 +11,9 @@ use chroma_index::fulltext::{
     types::{FullTextIndexReader, FullTextIndexWriter},
 };
 use chroma_storage::{local::LocalStorage, Storage};
+use chroma_test::datasets::{
+    ms_marco_queries::MicrosoftMarcoQueriesDataset, scidocs::SciDocsDataset, types::RecordDataset,
+};
 use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput};
 use futures::{StreamExt, TryStreamExt};
 use tantivy::tokenizer::NgramTokenizer;

rust/benchmark-datasets/Cargo.toml → rust/test/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "chroma-benchmark-datasets"
+name = "chroma-test"
 version = "0.1.0"
 edition = "2021"
 

rust/benchmark-datasets/src/datasets/mod.rs → rust/test/src/datasets/mod.rs

@@ -1,3 +1,6 @@
+pub mod types;
+pub mod util;
+
 pub mod ms_marco_queries;
 pub mod scidocs;
 pub mod wikipedia;

rust/benchmark-datasets/src/datasets/ms_marco_queries.rs → rust/test/src/datasets/ms_marco_queries.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, path::PathBuf};
 
-use crate::{
+use super::{
     types::{QueryDataset, Record, RecordDataset},
     util::get_or_populate_cached_dataset_file,
 };

rust/benchmark-datasets/src/datasets/scidocs.rs → rust/test/src/datasets/scidocs.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, path::PathBuf};
 
-use crate::{
+use super::{
     types::{Record, RecordDataset},
     util::get_or_populate_cached_dataset_file,
 };

rust/benchmark-datasets/src/types.rs → rust/test/src/datasets/types.rs

@@ -13,7 +13,7 @@ use tantivy::{
 use tokio::io::AsyncWriteExt;
 use tokio_stream::{Stream, StreamExt};
 
-use crate::util::{get_dir_for_persistent_dataset_files, get_or_populate_cached_dataset_file};
+use super::util::{get_dir_for_persistent_dataset_files, get_or_populate_cached_dataset_file};
 
 #[derive(Debug, Clone)]
 pub struct Record {

rust/benchmark-datasets/src/datasets/wikipedia.rs → rust/test/src/datasets/wikipedia.rs

@@ -8,7 +8,7 @@ use tokio::{fs::File, io::AsyncBufReadExt};
 use tokio_stream::wrappers::LinesStream;
 use tokio_util::io::StreamReader;
 
-use crate::{
+use super::{
     types::{Record, RecordDataset},
     util::get_or_populate_cached_dataset_file,
 };

rust/test/src/lib.rs

@@ -0,0 +1 @@
+pub mod datasets;