Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions algorithms/linfa-kernel/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,13 @@ sprs = { version = "=0.11.2", default-features = false }

linfa = { version = "0.8.1", path = "../.." }
linfa-nn = { version = "0.8.1", path = "../linfa-nn" }

[dev-dependencies]
criterion = "0.5"
linfa = { version = "0.8.1", path = "../..", features = ["benchmarks"] }
rand_xoshiro = "0.6"
ndarray-rand = "0.15"

[[bench]]
name = "gaussian"
harness = false
54 changes: 54 additions & 0 deletions algorithms/linfa-kernel/benches/gaussian.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
//! Dense Gaussian kernel construction:
//!
//! cargo bench -p linfa-kernel --bench gaussian

use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use linfa::benchmarks::config;
use linfa::traits::Transformer;
use linfa::Float;
use linfa_kernel::{Kernel, KernelMethod, KernelType};
use ndarray::Array2;
use ndarray_rand::{rand::SeedableRng, rand_distr::Uniform, RandomExt};
use rand_xoshiro::Xoshiro256Plus;

fn bench_type<F: Float>(c: &mut Criterion, ty: &str) {
let mut rng = Xoshiro256Plus::seed_from_u64(42);

for n in [500, 2_000] {
let mut group = c.benchmark_group(format!("Gaussian kernel {ty} n={n}"));
config::set_default_benchmark_configs(&mut group);
group.throughput(Throughput::Elements((n * n) as u64));

for d in [2, 16, 64, 120] {
let x: Array2<F> =
Array2::random_using((n, d), Uniform::new(0., 1.), &mut rng).mapv(F::cast);
// The mean squared distance between uniform points in [0, 1)^d is d/6: with
// eps = d/3 the exponents average -1/2, a typical bandwidth.
let params = Kernel::params()
.kind(KernelType::Dense)
.method(KernelMethod::Gaussian(F::cast(d as f64 / 3.0)));

group.bench_with_input(BenchmarkId::from_parameter(d), &x, |b, x| {
b.iter(|| params.transform(x.view()))
});
}
group.finish();
}
}

fn bench(c: &mut Criterion) {
bench_type::<f32>(c, "f32");
bench_type::<f64>(c, "f64");
}

#[cfg(not(target_os = "windows"))]
criterion_group! {
name = benches;
config = config::get_default_profiling_configs();
targets = bench
}

#[cfg(target_os = "windows")]
criterion_group!(benches, bench);

criterion_main!(benches);
104 changes: 95 additions & 9 deletions algorithms/linfa-kernel/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,15 +272,7 @@ pub enum KernelMethod<F> {
impl<F: Float> KernelMethod<F> {
pub fn distance(&self, a: ArrayView1<F>, b: ArrayView1<F>) -> F {
match *self {
KernelMethod::Gaussian(eps) => {
let distance = a
.iter()
.zip(b.iter())
.map(|(x, y)| (*x - *y) * (*x - *y))
.sum::<F>();

(-distance / eps).exp()
}
KernelMethod::Gaussian(eps) => gaussian_exponent(a, b, eps).exp(),
KernelMethod::Linear => a.mul(&b).sum(),
KernelMethod::Polynomial(c, d) => (a.mul(&b).sum() + c).powf(d),
}
Expand Down Expand Up @@ -477,10 +469,36 @@ impl<
}
}

/// The exponent of the Gaussian kernel, `-‖a - b‖² / eps`.
#[inline]
fn gaussian_exponent<F: Float>(a: ArrayView1<F>, b: ArrayView1<F>, eps: F) -> F {
let distance = match (a.as_slice(), b.as_slice()) {
// Contiguous rows, the usual case: a plain slice loop compiles to a tight body whatever
// the surrounding code, where ndarray's strided iterators do not reliably. Same order
// of summation, so the same result.
(Some(a), Some(b)) => a
.iter()
.zip(b)
.map(|(x, y)| (*x - *y) * (*x - *y))
.sum::<F>(),
_ => a
.iter()
.zip(b.iter())
.map(|(x, y)| (*x - *y) * (*x - *y))
.sum::<F>(),
};

-distance / eps
}

fn dense_from_fn<F: Float, D: Data<Elem = F>>(
dataset: &ArrayBase<D, Ix2>,
method: &KernelMethod<F>,
) -> Array2<F> {
if let KernelMethod::Gaussian(eps) = *method {
return dense_gaussian(dataset, eps);
}

let n_observations = dataset.len_of(Axis(0));
let mut similarity = Array2::eye(n_observations);

Expand All @@ -496,6 +514,38 @@ fn dense_from_fn<F: Float, D: Data<Elem = F>>(
similarity
}

/// Dense Gaussian kernel matrix, built row by row with the distances vectorised across points.
///
/// Bit-identical to calling [`KernelMethod::distance`] for every pair.
fn dense_gaussian<F: Float, D: Data<Elem = F>>(dataset: &ArrayBase<D, Ix2>, eps: F) -> Array2<F> {
let n_observations = dataset.len_of(Axis(0));
// The dataset feature by feature: row `k` holds feature `k` of every point, contiguously.
let features = dataset.t().as_standard_layout().into_owned();
let mut similarity = Array2::<F>::zeros((n_observations, n_observations));

for (i, mut row) in similarity.rows_mut().into_iter().enumerate() {
// The rows of a new standard-layout matrix are contiguous.
let row = row.as_slice_mut().unwrap();

// Row `i` of the squared distances, one feature at a time across all points, which
// vectorises across points. Each distance is still summed feature by feature in order,
// as `gaussian_exponent` does, so the result is the same.
for (feature, &x_i) in features.rows().into_iter().zip(dataset.row(i)) {
let feature = feature.as_slice().unwrap();
for (sum, &x_j) in row.iter_mut().zip(feature) {
let diff = x_j - x_i;
*sum += diff * diff;
}
}

for v in row.iter_mut() {
*v = (-*v / eps).exp();
}
}

similarity
}

fn sparse_from_fn<F: Float, D: Data<Elem = F>, N: NearestNeighbour>(
dataset: &ArrayBase<D, Ix2>,
k: usize,
Expand Down Expand Up @@ -614,6 +664,42 @@ mod tests {
}
}

#[test]
fn dense_gaussian_matches_distance() {
fn check<F: Float>() {
// Deterministic points in [0, 1)^d.
let mut s = 0x9E37_79B9_7F4A_7C15u64;
for &d in &[1, 2, 7, 33] {
let x = Array2::from_shape_simple_fn((61, d), || {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
F::cast((s >> 11) as f64 / (1u64 << 53) as f64)
});
let method = KernelMethod::Gaussian(F::cast(d as f64 / 3.0));
let k = dense_from_fn(&x, &method);
for i in 0..x.nrows() {
for j in 0..x.nrows() {
let expected = method.distance(x.row(i), x.row(j));
// Same exponent, same `exp`: exactly the same value.
assert!(
k[(i, j)] == expected,
"({}, {}), d = {}: {} != {}",
i,
j,
d,
k[(i, j)],
expected
);
}
}
}
}

check::<f32>();
check::<f64>();
}

#[test]
fn gaussian_test() {
let gauss_1 = KernelMethod::Gaussian(1.);
Expand Down
Loading