From ac4ba3a58421121da36db2909934237223f5b1d1 Mon Sep 17 00:00:00 2001 From: Murph Murphy Date: Fri, 30 Aug 2024 12:42:10 -0600 Subject: [PATCH 1/5] Add some vector benches --- Cargo.toml | 2 + benches/README.md | 58 +++++++++--- benches/ironcore_alloy_bench.rs | 161 +++++++++++++++++++++++++++++--- 3 files changed, 196 insertions(+), 25 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 899a583..653a242 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,10 +84,12 @@ name = "ironcore_alloy" [[bin]] name = "uniffi-bindgen" path = "uniffi-bindgen.rs" +bench = false [[bin]] name = "uniffi-bindgen-java" path = "uniffi-bindgen-java.rs" +bench = false # used to create the smallest cdylib binary we can to ship with the library in each ecosystem. # 6.9M vs 1.5M in initial testing. Can further have `strip` (the Unix utility) run on it to save ~0.2 MB more. diff --git a/benches/README.md b/benches/README.md index ee5b6bd..7dcab3d 100644 --- a/benches/README.md +++ b/benches/README.md @@ -90,23 +90,55 @@ There are also benchmarks available in [Kotlin](https://github.com/IronCoreLabs/ ## Results -The following benchmarking run was done on May 7, 2024 on a MacBook Pro (2023) with an Apple M2 Max chip. It uses a locally-built TSP running with the configuration from `demo-tsp.conf`. +The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU. It uses a locally-built TSP running with the configuration from `demo-tsp.conf`. ``` +Standalone - vector_encrypt d=384 + time: [17.430 µs 17.573 µs 17.735 µs] +Standalone - vector_encrypt d=768 + time: [32.334 µs 32.547 µs 32.741 µs] +Standalone - vector_encrypt d=1536 + time: [60.878 µs 61.498 µs 62.147 µs] +Standalone - vector_encrypt d=2048 + time: [87.224 µs 88.124 µs 89.011 µs] +Standalone - vector_roundtrip d=384 + time: [39.432 µs 39.517 µs 39.622 µs] +Standalone - vector_roundtrip d=768 + time: [80.038 µs 80.114 µs 80.199 µs] +Standalone - vector_roundtrip d=1536 + time: [190.09 µs 191.63 µs 193.30 µs] +Standalone - vector_roundtrip d=2048 + time: [249.74 µs 251.00 µs 252.41 µs] Standalone - roundtrip 10B - time: [11.515 µs 11.536 µs 11.558 µs] + time: [7.3951 µs 7.4709 µs 7.5491 µs] Standalone - roundtrip 10KB - time: [141.92 µs 142.34 µs 142.75 µs] + time: [23.303 µs 23.426 µs 23.548 µs] Standalone - roundtrip 100KB - time: [1.3312 ms 1.3350 ms 1.3387 ms] -TSP - encrypt 1B time: [55.808 µs 56.149 µs 56.715 µs] -TSP - encrypt 100B time: [56.793 µs 57.152 µs 57.713 µs] -TSP - encrypt 10KB time: [127.61 µs 129.40 µs 131.57 µs] -TSP - encrypt 1MB time: [7.1328 ms 7.1438 ms 7.1613 ms] -TSP - decrypt 1B time: [59.172 µs 59.508 µs 59.865 µs] -TSP - decrypt 100B time: [60.218 µs 60.462 µs 60.695 µs] -TSP - decrypt 10KB time: [132.08 µs 132.43 µs 132.80 µs] -TSP - decrypt 1MB time: [6.8972 ms 6.9199 ms 6.9441 ms] + time: [179.20 µs 181.46 µs 183.65 µs] +TSP - vector_encrypt d=384 + time: [142.69 µs 152.19 µs 161.07 µs] +TSP - vector_encrypt d=768 + time: [172.72 µs 176.22 µs 181.64 µs] +TSP - vector_encrypt d=1536 + time: [218.59 µs 235.90 µs 248.05 µs] +TSP - vector_encrypt d=2048 + time: [270.12 µs 286.51 µs 298.61 µs] +TSP - vector_roundtrip d=384 + time: [312.78 µs 321.93 µs 333.69 µs] +TSP - vector_roundtrip d=768 + time: [383.62 µs 395.26 µs 409.41 µs] +TSP - vector_roundtrip d=1536 + time: [529.15 µs 545.48 µs 562.23 µs] +TSP - vector_roundtrip d=2048 + time: [651.78 µs 662.87 µs 675.18 µs] +TSP - encrypt 1B time: [126.73 µs 128.40 µs 130.31 µs] +TSP - encrypt 100B time: [130.53 µs 133.51 µs 135.96 µs] +TSP - encrypt 10KB time: [140.84 µs 143.25 µs 146.06 µs] +TSP - encrypt 1MB time: [1.2828 ms 1.3429 ms 1.4088 ms] +TSP - decrypt 1B time: [136.40 µs 144.30 µs 149.06 µs] +TSP - decrypt 100B time: [136.97 µs 141.28 µs 148.68 µs] +TSP - decrypt 10KB time: [159.37 µs 165.90 µs 175.01 µs] +TSP - decrypt 1MB time: [1.5501 ms 1.5733 ms 1.6051 ms] TSP - batch encrypt 10 documents, 10 fields, 10B - time: [414.47 µs 416.00 µs 417.20 µs] + time: [412.43 µs 417.72 µs 421.34 µs] ``` diff --git a/benches/ironcore_alloy_bench.rs b/benches/ironcore_alloy_bench.rs index 0c5db91..4609ed5 100644 --- a/benches/ironcore_alloy_bench.rs +++ b/benches/ironcore_alloy_bench.rs @@ -47,7 +47,7 @@ fn benches(c: &mut Criterion) { .into(); let vector_secrets = [( SecretPath("secret_path".to_string()), - VectorSecret::new(1.23, rotatable_secret), + VectorSecret::new(2.5, rotatable_secret), )] .into(); @@ -57,17 +57,77 @@ fn benches(c: &mut Criterion) { let metadata = AlloyMetadata::new_simple(TenantId("tenant".to_string())); let range = Uniform::from(-1.0..1.0); - c.bench_function("vector_encrypt d=1k", |b| { + let encrypt_vector = |values| async { + let vector = PlaintextVector { + plaintext_vector: values, + secret_path: SecretPath("secret_path".to_string()), + derivation_path: DerivationPath("derivation_path".to_string()), + }; + sdk.vector().encrypt(vector, &metadata).await.unwrap(); + }; + c.bench_function("Standalone - vector_encrypt d=384", |b| { b.to_async(Runtime::new().unwrap()).iter_batched( - || rng.clone().sample_iter(&range).take(1000).collect_vec(), - |values| async { - let vector = PlaintextVector { - plaintext_vector: values, - secret_path: SecretPath("secret_path".to_string()), - derivation_path: DerivationPath("derivation_path".to_string()), - }; - sdk.vector().encrypt(vector, &metadata).await.unwrap(); - }, + || rng.clone().sample_iter(&range).take(384).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_encrypt d=768", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(768).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_encrypt d=1536", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(1536).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_encrypt d=2048", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(2048).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + + let roundtrip_vector = |values| async { + let vector = PlaintextVector { + plaintext_vector: values, + secret_path: SecretPath("secret_path".to_string()), + derivation_path: DerivationPath("derivation_path".to_string()), + }; + let encrypted = sdk.vector().encrypt(vector, &metadata).await.unwrap(); + sdk.vector().decrypt(encrypted, &metadata).await.unwrap(); + }; + c.bench_function("Standalone - vector_roundtrip d=384", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(384).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_roundtrip d=768", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(768).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_roundtrip d=1536", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(1536).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("Standalone - vector_roundtrip d=2048", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(2048).collect_vec(), + roundtrip_vector, BatchSize::SmallInput, ) }); @@ -126,7 +186,8 @@ fn tsp_benches(c: &mut Criterion) { let api_key = env::var("API_KEY").unwrap_or("0WUaXesNgbTAuLwn".to_string()); let config = - SaasShieldConfiguration::new(format!("{tsp_uri}:{tsp_port}"), api_key, true, None).unwrap(); + SaasShieldConfiguration::new(format!("{tsp_uri}:{tsp_port}"), api_key, true, Some(2.5)) + .unwrap(); let sdk = SaasShield::new(&config); let metadata = AlloyMetadata::new_simple(TenantId(tenant_id)); @@ -138,6 +199,82 @@ fn tsp_benches(c: &mut Criterion) { sdk.standard().decrypt(encrypted, &metadata).await.unwrap() }; + let range = Uniform::from(-1.0..1.0); + let encrypt_vector = |values| async { + let vector = PlaintextVector { + plaintext_vector: values, + secret_path: SecretPath("secret_path".to_string()), + derivation_path: DerivationPath("derivation_path".to_string()), + }; + sdk.vector().encrypt(vector, &metadata).await.unwrap(); + }; + c.bench_function("TSP - vector_encrypt d=384", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(384).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_encrypt d=768", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(768).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_encrypt d=1536", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(1536).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_encrypt d=2048", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(2048).collect_vec(), + encrypt_vector, + BatchSize::SmallInput, + ) + }); + + let roundtrip_vector = |values| async { + let vector = PlaintextVector { + plaintext_vector: values, + secret_path: SecretPath("secret_path".to_string()), + derivation_path: DerivationPath("derivation_path".to_string()), + }; + let encrypted = sdk.vector().encrypt(vector, &metadata).await.unwrap(); + sdk.vector().decrypt(encrypted, &metadata).await.unwrap(); + }; + c.bench_function("TSP - vector_roundtrip d=384", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(384).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_roundtrip d=768", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(768).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_roundtrip d=1536", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(1536).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - vector_roundtrip d=2048", |b| { + b.to_async(Runtime::new().unwrap()).iter_batched( + || rng.clone().sample_iter(&range).take(2048).collect_vec(), + roundtrip_vector, + BatchSize::SmallInput, + ) + }); + c.bench_function("TSP - encrypt 1B", |b| { b.to_async(Runtime::new().unwrap()).iter_batched( || generate_plaintext(1, 1, &mut rng), From fc800163578c6772defb2e5cd73d7951a37f10a5 Mon Sep 17 00:00:00 2001 From: Murph Murphy Date: Fri, 30 Aug 2024 13:57:14 -0600 Subject: [PATCH 2/5] Add vector and vector batch python benches --- python/ironcore-alloy/bench.py | 107 +++++++++++++++++++++++++++++++-- 1 file changed, 103 insertions(+), 4 deletions(-) diff --git a/python/ironcore-alloy/bench.py b/python/ironcore-alloy/bench.py index 55709d2..f441ae9 100644 --- a/python/ironcore-alloy/bench.py +++ b/python/ironcore-alloy/bench.py @@ -1,4 +1,4 @@ -import ironcore_alloy +from ironcore_alloy import * import time, random, string, asyncio, statistics import pyperf @@ -6,15 +6,14 @@ # Review stats with `hatch run bench:stats` +## STANDALONE def create_sdk_shared_key(key_bytes: bytearray) -> ironcore_alloy.Standalone: secret = ironcore_alloy.Secret(key_bytes) standalone_secret = ironcore_alloy.StandaloneSecret(1, secret) rotatable_secret = ironcore_alloy.RotatableSecret(standalone_secret, None) standard_secrets = ironcore_alloy.StandardSecrets(1, [standalone_secret]) deterministic_secrets = {"secret_path": rotatable_secret} - vector_secrets = { - "secret_path": ironcore_alloy.VectorSecret(1.23, rotatable_secret) - } + vector_secrets = {"secret_path": ironcore_alloy.VectorSecret(2.5, rotatable_secret)} config = ironcore_alloy.StandaloneConfiguration( standard_secrets, deterministic_secrets, vector_secrets ) @@ -36,13 +35,113 @@ async def standard_roundtrip( await sdk.standard().decrypt(encrypted, metadata) +async def vector_encrypt( + sdk: ironcore_alloy.Standalone, + vector: list[float], + metadata: ironcore_alloy.AlloyMetadata, +): + plaintext_vector = ironcore_alloy.PlaintextVector( + plaintext_vector=vector, + secret_path="secret_path", + derivation_path="derivation_path", + ) + await sdk.vector().encrypt(plaintext_vector, metadata) + + +async def vector_batch_encrypt( + sdk: ironcore_alloy.Standalone, + plaintext_vectors: dict[ironcore_alloy.FieldId, ironcore_alloy.PlaintextField], + metadata: ironcore_alloy.AlloyMetadata, +): + await sdk.vector().encrypt_batch(plaintext_vectors, metadata) + + +async def vector_roundtrip( + sdk: ironcore_alloy.Standalone, + vector: list[float], + metadata: ironcore_alloy.AlloyMetadata, +): + plaintext_vector = ironcore_alloy.PlaintextVector( + plaintext_vector=vector, + secret_path="secret_path", + derivation_path="derivation_path", + ) + encrypted_vector = await sdk.vector().encrypt(plaintext_vector, metadata) + await sdk.vector().decrypt(encrypted_vector, metadata) + + +def random_floats(dimensions: int): + return [random.uniform(-1.0, 1.0) for _ in range(dimensions)] + + metadata = ironcore_alloy.AlloyMetadata.new_simple("tenant") key_bytes = "awholelotoftotallyrandomdatathatcanbeusedasasecurecryptokey".encode( "utf-8" ) sdk = create_sdk_shared_key(key_bytes) + runner = pyperf.Runner() runner.metadata["description"] = "Run the IronCore Alloy benchmarks." + +### standalone vector +runner.bench_async_func( + "vector encrypt d=384", + vector_encrypt, + sdk, + random_floats(384), + metadata, +) +runner.bench_async_func( + "vector encrypt d=768", + vector_encrypt, + sdk, + random_floats(768), + metadata, +) +runner.bench_async_func( + "vector encrypt d=1536", + vector_encrypt, + sdk, + random_floats(1536), + metadata, +) +runner.bench_async_func( + "vector encrypt d=2048", + vector_encrypt, + sdk, + random_floats(2048), + metadata, +) +runner.bench_async_func( + "vector batch (100) encrypt d=768", + vector_batch_encrypt, + sdk, + { + str(idx): ironcore_alloy.PlaintextVector( + plaintext_vector=vector, + secret_path="secret_path", + derivation_path="derivation_path", + ) + for idx, vector in enumerate([random_floats(768) for _ in range(100)]) + }, + metadata, +) +runner.bench_async_func( + "vector batch (1000) encrypt d=768", + vector_batch_encrypt, + sdk, + { + str(idx): ironcore_alloy.PlaintextVector( + plaintext_vector=vector, + secret_path="secret_path", + derivation_path="derivation_path", + ) + for idx, vector in enumerate([random_floats(768) for _ in range(1000)]) + }, + metadata, +) + +### standalone standard random_small_word = random_word(10).encode("utf-8") runner.bench_async_func( "standard_roundtrip_small", From 7c5007ca500d9308f6c87b5c7c23ae8dea21ffda Mon Sep 17 00:00:00 2001 From: Murph Murphy Date: Fri, 30 Aug 2024 14:02:52 -0600 Subject: [PATCH 3/5] Add python benchmarks doc with basics --- BENCHMARKS.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 BENCHMARKS.md diff --git a/BENCHMARKS.md b/BENCHMARKS.md new file mode 100644 index 0000000..d71d1cd --- /dev/null +++ b/BENCHMARKS.md @@ -0,0 +1,33 @@ +# Python Benchmarks + +See `project_root/benches/README.md` for more general information about benchmarks and interpretation, along with links to benchmarks in the other languages. + +## Usage + +`hatch run bench:bench` to run the benchmarks. Python benchmarks are all Standalone right now, if you'd like to see TSP benchmarks in Python open up an issue (or a PR :)). +`hatch run bench:stats` to view the in-depth statistics of the most recent run. + +## Results + +The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU. + +``` +..................... +vector encrypt d=384: Mean +- std dev: 2.66 ms +- 0.14 ms +..................... +vector encrypt d=768: Mean +- std dev: 5.28 ms +- 0.25 ms +..................... +vector encrypt d=1536: Mean +- std dev: 10.3 ms +- 0.6 ms +..................... +vector encrypt d=2048: Mean +- std dev: 13.6 ms +- 0.7 ms +..................... +vector batch (100) encrypt d=768: Mean +- std dev: 416 ms +- 25 ms +..................... +vector batch (1000) encrypt d=768: Mean +- std dev: 4.14 sec +- 0.19 sec +..................... +standard_roundtrip_small: Mean +- std dev: 616 us +- 23 us +..................... +standard_roundtrip_medium: Mean +- std dev: 13.1 ms +- 0.6 ms +..................... +standard_roundtrip_large: Mean +- std dev: 124 ms +- 6 ms +``` From 20aa9bb0a9510f3d927c83cb12c147d92a955528 Mon Sep 17 00:00:00 2001 From: Murph Murphy Date: Fri, 30 Aug 2024 17:13:39 -0600 Subject: [PATCH 4/5] Moved python doc into python folder --- BENCHMARKS.md | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 BENCHMARKS.md diff --git a/BENCHMARKS.md b/BENCHMARKS.md deleted file mode 100644 index d71d1cd..0000000 --- a/BENCHMARKS.md +++ /dev/null @@ -1,33 +0,0 @@ -# Python Benchmarks - -See `project_root/benches/README.md` for more general information about benchmarks and interpretation, along with links to benchmarks in the other languages. - -## Usage - -`hatch run bench:bench` to run the benchmarks. Python benchmarks are all Standalone right now, if you'd like to see TSP benchmarks in Python open up an issue (or a PR :)). -`hatch run bench:stats` to view the in-depth statistics of the most recent run. - -## Results - -The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU. - -``` -..................... -vector encrypt d=384: Mean +- std dev: 2.66 ms +- 0.14 ms -..................... -vector encrypt d=768: Mean +- std dev: 5.28 ms +- 0.25 ms -..................... -vector encrypt d=1536: Mean +- std dev: 10.3 ms +- 0.6 ms -..................... -vector encrypt d=2048: Mean +- std dev: 13.6 ms +- 0.7 ms -..................... -vector batch (100) encrypt d=768: Mean +- std dev: 416 ms +- 25 ms -..................... -vector batch (1000) encrypt d=768: Mean +- std dev: 4.14 sec +- 0.19 sec -..................... -standard_roundtrip_small: Mean +- std dev: 616 us +- 23 us -..................... -standard_roundtrip_medium: Mean +- std dev: 13.1 ms +- 0.6 ms -..................... -standard_roundtrip_large: Mean +- std dev: 124 ms +- 6 ms -``` From 300a253f2809c89cee882447f743eb7749e19e52 Mon Sep 17 00:00:00 2001 From: Murph Murphy Date: Fri, 30 Aug 2024 17:52:32 -0600 Subject: [PATCH 5/5] Python bench doc --- kotlin/benchmarks/src/StandaloneBenchmark.kt | 15 ++++++++- python/ironcore-alloy/BENCHMARKS.md | 33 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 python/ironcore-alloy/BENCHMARKS.md diff --git a/kotlin/benchmarks/src/StandaloneBenchmark.kt b/kotlin/benchmarks/src/StandaloneBenchmark.kt index 9b059fb..434a5e8 100644 --- a/kotlin/benchmarks/src/StandaloneBenchmark.kt +++ b/kotlin/benchmarks/src/StandaloneBenchmark.kt @@ -3,6 +3,7 @@ package test import com.ironcorelabs.ironcore_alloy.* import java.util.Base64 import java.util.concurrent.* +import java.util.Random import kotlin.ByteArray import kotlin.system.* import kotlinx.coroutines.* @@ -23,7 +24,7 @@ class StandaloneBenchmark { var largeWord: ByteArray = "".toByteArray() val keyByteArray = "hJdwvEeg5mxTu9qWcWrljfKs1ga4MpQ9MzXgLxtlkwX//yA=".base64ToByteArray() - val approximationFactor = 1.1f + val approximationFactor = 2.5f val standardSecrets = StandardSecrets(10, listOf(StandaloneSecret(10, Secret(keyByteArray)))) val deterministicSecrets = mapOf( @@ -62,6 +63,18 @@ class StandaloneBenchmark { largeWord = randomWord(100).toByteArray() } + @State(Scope.Thread) + open class Vector384State { + var vector: List = (1..384).map { Random().nextFloat() } + } + + @Benchmark + fun standaloneVectorEncrypt384d(s: Vector384State) { + runBlocking { + standaloneSdk.vector().encrypt(PlaintextVector(s.vector, "", ""), metadata) + } + } + @Benchmark fun standaloneRoundtripStandard10B() { runBlocking { diff --git a/python/ironcore-alloy/BENCHMARKS.md b/python/ironcore-alloy/BENCHMARKS.md new file mode 100644 index 0000000..d71d1cd --- /dev/null +++ b/python/ironcore-alloy/BENCHMARKS.md @@ -0,0 +1,33 @@ +# Python Benchmarks + +See `project_root/benches/README.md` for more general information about benchmarks and interpretation, along with links to benchmarks in the other languages. + +## Usage + +`hatch run bench:bench` to run the benchmarks. Python benchmarks are all Standalone right now, if you'd like to see TSP benchmarks in Python open up an issue (or a PR :)). +`hatch run bench:stats` to view the in-depth statistics of the most recent run. + +## Results + +The following benchmarking run was done on August 30th, 2024 on a Lenovo Thinkpad X1 Extreme 2nd Gen with an i9-9880H CPU. + +``` +..................... +vector encrypt d=384: Mean +- std dev: 2.66 ms +- 0.14 ms +..................... +vector encrypt d=768: Mean +- std dev: 5.28 ms +- 0.25 ms +..................... +vector encrypt d=1536: Mean +- std dev: 10.3 ms +- 0.6 ms +..................... +vector encrypt d=2048: Mean +- std dev: 13.6 ms +- 0.7 ms +..................... +vector batch (100) encrypt d=768: Mean +- std dev: 416 ms +- 25 ms +..................... +vector batch (1000) encrypt d=768: Mean +- std dev: 4.14 sec +- 0.19 sec +..................... +standard_roundtrip_small: Mean +- std dev: 616 us +- 23 us +..................... +standard_roundtrip_medium: Mean +- std dev: 13.1 ms +- 0.6 ms +..................... +standard_roundtrip_large: Mean +- std dev: 124 ms +- 6 ms +```