Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/bpe/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ aneubeck-daachorse = "1.1.1"
base64 = { version = "0.22", optional = true }
fnv = "1.0"
itertools = "0.14"
rand = { version = "0.9", optional = true }
rand = { version = "0.10", optional = true }
serde = { version = "1", features = ["derive"] }

[dev-dependencies]
Expand Down
2 changes: 1 addition & 1 deletion crates/bpe/benchmarks/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ test = true
bpe = { path = "../../bpe", features = ["rand", "tiktoken"] }
bpe-openai = { path = "../../bpe-openai" }
criterion = "0.8"
rand = "0.9"
rand = "0.10"
tiktoken-rs = "0.9"
tokenizers = { version = "0.22", features = ["http"] }
4 changes: 1 addition & 3 deletions crates/bpe/benchmarks/performance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ use bpe_benchmarks::*;
use criterion::{
criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration,
};
use rand::rngs::StdRng;
use rand::SeedableRng;
use rand::{rng, Rng};
use rand::{rng, RngExt};

fn counting_benchmark(c: &mut Criterion) {
for (name, bpe, _, _) in TOKENIZERS.iter() {
Expand Down
10 changes: 5 additions & 5 deletions crates/bpe/src/byte_pair_encoding.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ pub fn find_hash_factor_for_tiktoken(data: &str) -> Result<u64, base64::DecodeEr
pub fn find_hash_factor_for_dictionary(tokens: impl IntoIterator<Item = Vec<u8>>) -> u64 {
use std::collections::HashSet;

use rand::Rng;
use rand::RngExt;

let all_tokens = tokens.into_iter().collect_vec();
let mut rnd = rand::rng();
Expand Down Expand Up @@ -573,7 +573,7 @@ impl BytePairEncoding {
// and hence may include previously discarded token later down the byte stream. At the sentence level though we don't expect it to make much difference.
// Also, this implementation of BPE constructs merges on the fly from the set of tokens, hence might come up with a different set of merges with the same dictionary.
#[cfg(feature = "rand")]
pub fn encode_minimal_dropout<R: rand::Rng>(
pub fn encode_minimal_dropout<R: rand::RngExt>(
&self,
text: &[u8],
dropout: f32,
Expand Down Expand Up @@ -627,7 +627,7 @@ pub fn create_test_string_with_predicate(
min_bytes: usize,
predicate: impl Fn(&str) -> bool,
) -> String {
use rand::{rng, Rng};
use rand::{rng, RngExt};
// the string we accumulated thus far
let mut result = String::new();
// the tokens we added so we can backtrack
Expand Down Expand Up @@ -662,7 +662,7 @@ pub fn create_test_string_with_predicate(

#[cfg(feature = "rand")]
pub fn select_test_string(text: &str, min_bytes: usize) -> &str {
use rand::{rng, Rng};
use rand::{rng, RngExt};
let mut start = rng().random_range(0..text.len() - min_bytes);
while !text.is_char_boundary(start) {
start -= 1;
Expand All @@ -677,7 +677,7 @@ pub fn select_test_string(text: &str, min_bytes: usize) -> &str {
/// Generate test bytes by concatenating random tokens.
#[cfg(feature = "rand")]
pub fn create_test_bytes(bpe: &BytePairEncoding, min_bytes: usize) -> Vec<u8> {
use rand::{rng, Rng};
use rand::{rng, RngExt};
let mut result = Vec::new();
while result.len() < min_bytes {
let i = rng().random_range(0..bpe.num_tokens());
Expand Down
2 changes: 1 addition & 1 deletion crates/bpe/tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ edition = "2021"
bpe = { path = "../../bpe", features = ["rand"] }
bpe-openai = { path = "../../bpe-openai" }
itertools = "0.14"
rand = "0.9"
rand = "0.10"
tiktoken-rs = "0.9"
2 changes: 1 addition & 1 deletion crates/bpe/tests/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#[cfg(test)]
mod tests {
use itertools::Itertools;
use rand::{rng, Rng};
use rand::{rng, RngExt};
use tiktoken_rs::cl100k_base_singleton;

use bpe::appendable_encoder::AppendableEncoder;
Expand Down
8 changes: 4 additions & 4 deletions crates/geo_filters/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@ fnv = "1.0"
hyperloglogplus = { version = "0.4", optional = true }
itertools = "0.14"
once_cell = "1.18"
rand = { version = "0.9", optional = true }
rand = { version = "0.10", optional = true }
rayon = { version = "1.7", optional = true }
regex = { version = "1", optional = true }
serde = { version = "1.0", default-features = false, optional = true }
rand_chacha = { version = "0.9", optional = true }
rand_chacha = { version = "0.10", optional = true }

[dev-dependencies]
criterion = "0.8"
geo_filters = { path = ".", features = ["evaluation"] }
rand = "0.9"
rand_chacha = "0.9"
rand = "0.10"
rand_chacha = "0.10"
rayon = "1.7"

[[bench]]
Expand Down
2 changes: 1 addition & 1 deletion crates/geo_filters/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ pub(crate) fn take_ref<I: Iterator>(iter: &mut I, n: usize) -> impl Iterator<Ite

#[cfg(test)]
pub(crate) mod tests {
use rand::RngCore;
use rand::Rng as RngCore;
use rand_chacha::ChaCha12Rng;

use crate::{Count, Method};
Expand Down
2 changes: 1 addition & 1 deletion crates/geo_filters/src/config/lookup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ impl HashToBucketLookup {

#[cfg(test)]
mod tests {
use rand::RngCore;
use rand::Rng as RngCore;
use rand_chacha::ChaCha12Rng;

use crate::{
Expand Down
5 changes: 2 additions & 3 deletions crates/geo_filters/src/diff_count.rs
Original file line number Diff line number Diff line change
Expand Up @@ -362,8 +362,7 @@ impl<'a, C: GeoConfig<Diff>> GeoDiffCount<'a, C> {
/// the resulting geofilter should always be the same.
#[cfg(any(test, feature = "test-support"))]
pub fn pseudorandom_filter_with_config(config: C, items: usize) -> Self {
use rand::RngCore;
use rand_chacha::rand_core::SeedableRng;
use rand_chacha::rand_core::{Rng, SeedableRng};

let mut rng = rand_chacha::ChaCha12Rng::seed_from_u64(items as u64);
let mut filter = Self::new(config);
Expand Down Expand Up @@ -468,7 +467,7 @@ mod tests {
use std::io::Write;

use itertools::Itertools;
use rand::{seq::IteratorRandom, RngCore};
use rand::{seq::IteratorRandom, Rng as RngCore};
use rand_chacha::ChaCha12Rng;

use crate::{
Expand Down
2 changes: 1 addition & 1 deletion crates/geo_filters/src/distinct_count.rs
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ fn or<C: GeoConfig<Distinct>>(
#[cfg(test)]
mod tests {
use itertools::Itertools;
use rand::RngCore;
use rand::Rng as RngCore;

use crate::build_hasher::UnstableDefaultBuildHasher;
use crate::config::{iter_ones, tests::test_estimate, FixedConfig, VariableConfig};
Expand Down
4 changes: 2 additions & 2 deletions crates/geo_filters/src/evaluation/simulation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::io::Write;
use std::time::Instant;

use itertools::Itertools;
use rand::{RngCore, SeedableRng};
use rand::Rng as RngCore;
use rayon::prelude::{IntoParallelIterator, ParallelIterator};

use crate::config::GeoConfig;
Expand Down Expand Up @@ -181,7 +181,7 @@ pub fn simulate<F: Fn() -> Box<dyn SimulationCount> + Send + Sync>(
.map(|_| {
let mut t = f();
let mut last_set_size = 0;
let mut rnd = rand::rngs::StdRng::from_os_rng();
let mut rnd: rand::rngs::StdRng = rand::make_rng();
set_sizes
.iter()
.map(move |set_size| {
Expand Down
4 changes: 2 additions & 2 deletions crates/string-offsets/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ wasm = ["wasm-bindgen"]
wasm-bindgen = { version = "0.2", optional = true }

[dev-dependencies]
rand = "0.9"
rand_chacha = "0.9"
rand = "0.10"
rand_chacha = "0.10"
criterion = "0.8"

[[bench]]
Expand Down
2 changes: 1 addition & 1 deletion crates/string-offsets/benchmarks/performance.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use rand::{rng, Rng};
use rand::{rng, RngExt};
use string_offsets::{AllConfig, OnlyLines, StringOffsets};

fn only_lines_construction_benchmark(c: &mut Criterion) {
Expand Down
Loading