More benchmarks

Cargo.toml

@@ -84,10 +84,12 @@ name = "ironcore_alloy"
 [[bin]]
 name = "uniffi-bindgen"
 path = "uniffi-bindgen.rs"
+bench = false
 
 [[bin]]
 name = "uniffi-bindgen-java"
 path = "uniffi-bindgen-java.rs"
+bench = false
 
 # used to create the smallest cdylib binary we can to ship with the library in each ecosystem.
 # 6.9M vs 1.5M in initial testing. Can further have `strip` (the Unix utility) run on it to save ~0.2 MB more.

benches/README.md

@@ -90,23 +90,55 @@ There are also benchmarks available in [Kotlin](https://github.com/IronCoreLabs/
 
 ## Results
 
-The following benchmarking run was done on May 7, 2024 on a MacBook Pro (2023) with an Apple M2 Max chip. It uses a locally-built TSP running with the configuration from `demo-tsp.conf`.
+The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU. It uses a locally-built TSP running with the configuration from `demo-tsp.conf`.
 
 ```
+Standalone - vector_encrypt d=384
+                        time:   [17.430 µs 17.573 µs 17.735 µs]
+Standalone - vector_encrypt d=768
+                        time:   [32.334 µs 32.547 µs 32.741 µs]
+Standalone - vector_encrypt d=1536
+                        time:   [60.878 µs 61.498 µs 62.147 µs]
+Standalone - vector_encrypt d=2048
+                        time:   [87.224 µs 88.124 µs 89.011 µs]
+Standalone - vector_roundtrip d=384
+                        time:   [39.432 µs 39.517 µs 39.622 µs]
+Standalone - vector_roundtrip d=768
+                        time:   [80.038 µs 80.114 µs 80.199 µs]
+Standalone - vector_roundtrip d=1536
+                        time:   [190.09 µs 191.63 µs 193.30 µs]
+Standalone - vector_roundtrip d=2048
+                        time:   [249.74 µs 251.00 µs 252.41 µs]
 Standalone - roundtrip 10B
-                        time:   [11.515 µs 11.536 µs 11.558 µs]
+                        time:   [7.3951 µs 7.4709 µs 7.5491 µs]
 Standalone - roundtrip 10KB
-                        time:   [141.92 µs 142.34 µs 142.75 µs]
+                        time:   [23.303 µs 23.426 µs 23.548 µs]
 Standalone - roundtrip 100KB
-                        time:   [1.3312 ms 1.3350 ms 1.3387 ms]
-TSP - encrypt 1B        time:   [55.808 µs 56.149 µs 56.715 µs]
-TSP - encrypt 100B      time:   [56.793 µs 57.152 µs 57.713 µs]
-TSP - encrypt 10KB      time:   [127.61 µs 129.40 µs 131.57 µs]
-TSP - encrypt 1MB       time:   [7.1328 ms 7.1438 ms 7.1613 ms]
-TSP - decrypt 1B        time:   [59.172 µs 59.508 µs 59.865 µs]
-TSP - decrypt 100B      time:   [60.218 µs 60.462 µs 60.695 µs]
-TSP - decrypt 10KB      time:   [132.08 µs 132.43 µs 132.80 µs]
-TSP - decrypt 1MB       time:   [6.8972 ms 6.9199 ms 6.9441 ms]
+                        time:   [179.20 µs 181.46 µs 183.65 µs]
+TSP - vector_encrypt d=384
+                        time:   [142.69 µs 152.19 µs 161.07 µs]
+TSP - vector_encrypt d=768
+                        time:   [172.72 µs 176.22 µs 181.64 µs]
+TSP - vector_encrypt d=1536
+                        time:   [218.59 µs 235.90 µs 248.05 µs]
+TSP - vector_encrypt d=2048
+                        time:   [270.12 µs 286.51 µs 298.61 µs]
+TSP - vector_roundtrip d=384
+                        time:   [312.78 µs 321.93 µs 333.69 µs]
+TSP - vector_roundtrip d=768
+                        time:   [383.62 µs 395.26 µs 409.41 µs]
+TSP - vector_roundtrip d=1536
+                        time:   [529.15 µs 545.48 µs 562.23 µs]
+TSP - vector_roundtrip d=2048
+                        time:   [651.78 µs 662.87 µs 675.18 µs]
+TSP - encrypt 1B        time:   [126.73 µs 128.40 µs 130.31 µs]
+TSP - encrypt 100B      time:   [130.53 µs 133.51 µs 135.96 µs]
+TSP - encrypt 10KB      time:   [140.84 µs 143.25 µs 146.06 µs]
+TSP - encrypt 1MB       time:   [1.2828 ms 1.3429 ms 1.4088 ms]
+TSP - decrypt 1B        time:   [136.40 µs 144.30 µs 149.06 µs]
+TSP - decrypt 100B      time:   [136.97 µs 141.28 µs 148.68 µs]
+TSP - decrypt 10KB      time:   [159.37 µs 165.90 µs 175.01 µs]
+TSP - decrypt 1MB       time:   [1.5501 ms 1.5733 ms 1.6051 ms]
 TSP - batch encrypt 10 documents, 10 fields, 10B
-                        time:   [414.47 µs 416.00 µs 417.20 µs]
+                        time:   [412.43 µs 417.72 µs 421.34 µs]
 ```

benches/ironcore_alloy_bench.rs

@@ -47,7 +47,7 @@ fn benches(c: &mut Criterion) {
     .into();
     let vector_secrets = [(
         SecretPath("secret_path".to_string()),
-        VectorSecret::new(1.23, rotatable_secret),
+        VectorSecret::new(2.5, rotatable_secret),
     )]
     .into();
 
@@ -57,17 +57,77 @@ fn benches(c: &mut Criterion) {
     let metadata = AlloyMetadata::new_simple(TenantId("tenant".to_string()));
 
     let range = Uniform::from(-1.0..1.0);
-    c.bench_function("vector_encrypt d=1k", |b| {
+    let encrypt_vector = |values| async {
+        let vector = PlaintextVector {
+            plaintext_vector: values,
+            secret_path: SecretPath("secret_path".to_string()),
+            derivation_path: DerivationPath("derivation_path".to_string()),
+        };
+        sdk.vector().encrypt(vector, &metadata).await.unwrap();
+    };
+    c.bench_function("Standalone - vector_encrypt d=384", |b| {
         b.to_async(Runtime::new().unwrap()).iter_batched(
-            || rng.clone().sample_iter(&range).take(1000).collect_vec(),
-            |values| async {
-                let vector = PlaintextVector {
-                    plaintext_vector: values,
-                    secret_path: SecretPath("secret_path".to_string()),
-                    derivation_path: DerivationPath("derivation_path".to_string()),
-                };
-                sdk.vector().encrypt(vector, &metadata).await.unwrap();
-            },
+            || rng.clone().sample_iter(&range).take(384).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_encrypt d=768", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(768).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_encrypt d=1536", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(1536).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_encrypt d=2048", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(2048).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+
+    let roundtrip_vector = |values| async {
+        let vector = PlaintextVector {
+            plaintext_vector: values,
+            secret_path: SecretPath("secret_path".to_string()),
+            derivation_path: DerivationPath("derivation_path".to_string()),
+        };
+        let encrypted = sdk.vector().encrypt(vector, &metadata).await.unwrap();
+        sdk.vector().decrypt(encrypted, &metadata).await.unwrap();
+    };
+    c.bench_function("Standalone - vector_roundtrip d=384", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(384).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_roundtrip d=768", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(768).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_roundtrip d=1536", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(1536).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("Standalone - vector_roundtrip d=2048", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(2048).collect_vec(),
+            roundtrip_vector,
             BatchSize::SmallInput,
         )
     });
@@ -126,7 +186,8 @@ fn tsp_benches(c: &mut Criterion) {
     let api_key = env::var("API_KEY").unwrap_or("0WUaXesNgbTAuLwn".to_string());
 
     let config =
-        SaasShieldConfiguration::new(format!("{tsp_uri}:{tsp_port}"), api_key, true, None).unwrap();
+        SaasShieldConfiguration::new(format!("{tsp_uri}:{tsp_port}"), api_key, true, Some(2.5))
+            .unwrap();
     let sdk = SaasShield::new(&config);
     let metadata = AlloyMetadata::new_simple(TenantId(tenant_id));
 
@@ -138,6 +199,82 @@ fn tsp_benches(c: &mut Criterion) {
         sdk.standard().decrypt(encrypted, &metadata).await.unwrap()
     };
 
+    let range = Uniform::from(-1.0..1.0);
+    let encrypt_vector = |values| async {
+        let vector = PlaintextVector {
+            plaintext_vector: values,
+            secret_path: SecretPath("secret_path".to_string()),
+            derivation_path: DerivationPath("derivation_path".to_string()),
+        };
+        sdk.vector().encrypt(vector, &metadata).await.unwrap();
+    };
+    c.bench_function("TSP - vector_encrypt d=384", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(384).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_encrypt d=768", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(768).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_encrypt d=1536", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(1536).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_encrypt d=2048", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(2048).collect_vec(),
+            encrypt_vector,
+            BatchSize::SmallInput,
+        )
+    });
+
+    let roundtrip_vector = |values| async {
+        let vector = PlaintextVector {
+            plaintext_vector: values,
+            secret_path: SecretPath("secret_path".to_string()),
+            derivation_path: DerivationPath("derivation_path".to_string()),
+        };
+        let encrypted = sdk.vector().encrypt(vector, &metadata).await.unwrap();
+        sdk.vector().decrypt(encrypted, &metadata).await.unwrap();
+    };
+    c.bench_function("TSP - vector_roundtrip d=384", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(384).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_roundtrip d=768", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(768).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_roundtrip d=1536", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(1536).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+    c.bench_function("TSP - vector_roundtrip d=2048", |b| {
+        b.to_async(Runtime::new().unwrap()).iter_batched(
+            || rng.clone().sample_iter(&range).take(2048).collect_vec(),
+            roundtrip_vector,
+            BatchSize::SmallInput,
+        )
+    });
+
     c.bench_function("TSP - encrypt 1B", |b| {
         b.to_async(Runtime::new().unwrap()).iter_batched(
             || generate_plaintext(1, 1, &mut rng),

kotlin/benchmarks/src/StandaloneBenchmark.kt

@@ -3,6 +3,7 @@ package test
 import com.ironcorelabs.ironcore_alloy.*
 import java.util.Base64
 import java.util.concurrent.*
+import java.util.Random
 import kotlin.ByteArray
 import kotlin.system.*
 import kotlinx.coroutines.*
@@ -23,7 +24,7 @@ class StandaloneBenchmark {
     var largeWord: ByteArray = "".toByteArray()
 
     val keyByteArray = "hJdwvEeg5mxTu9qWcWrljfKs1ga4MpQ9MzXgLxtlkwX//yA=".base64ToByteArray()
-    val approximationFactor = 1.1f
+    val approximationFactor = 2.5f
     val standardSecrets = StandardSecrets(10, listOf(StandaloneSecret(10, Secret(keyByteArray))))
     val deterministicSecrets =
             mapOf(
@@ -62,6 +63,18 @@ class StandaloneBenchmark {
         largeWord = randomWord(100).toByteArray()
     }
 
+    @State(Scope.Thread)
+    open class Vector384State {
+        var vector: List<Float> = (1..384).map { Random().nextFloat() }
+    }
+
+    @Benchmark
+    fun standaloneVectorEncrypt384d(s: Vector384State) {
+        runBlocking {
+            standaloneSdk.vector().encrypt(PlaintextVector(s.vector, "", ""), metadata)
+        }
+    }
+
     @Benchmark
     fun standaloneRoundtripStandard10B() {
         runBlocking {

python/ironcore-alloy/BENCHMARKS.md

@@ -0,0 +1,33 @@
+# Python Benchmarks
+
+See `project_root/benches/README.md` for more general information about benchmarks and interpretation, along with links to benchmarks in the other languages.
+
+## Usage
+
+`hatch run bench:bench` to run the benchmarks. Python benchmarks are all Standalone right now, if you'd like to see TSP benchmarks in Python open up an issue (or a PR :)).
+`hatch run bench:stats` to view the in-depth statistics of the most recent run.
+
+## Results
+
+The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU.
+
+```
+.....................
+vector encrypt d=384: Mean +- std dev: 2.66 ms +- 0.14 ms
+.....................
+vector encrypt d=768: Mean +- std dev: 5.28 ms +- 0.25 ms
+.....................
+vector encrypt d=1536: Mean +- std dev: 10.3 ms +- 0.6 ms
+.....................
+vector encrypt d=2048: Mean +- std dev: 13.6 ms +- 0.7 ms
+.....................
+vector batch (100) encrypt d=768: Mean +- std dev: 416 ms +- 25 ms
+.....................
+vector batch (1000) encrypt d=768: Mean +- std dev: 4.14 sec +- 0.19 sec
+.....................
+standard_roundtrip_small: Mean +- std dev: 616 us +- 23 us
+.....................
+standard_roundtrip_medium: Mean +- std dev: 13.1 ms +- 0.6 ms
+.....................
+standard_roundtrip_large: Mean +- std dev: 124 ms +- 6 ms 
+```