From 2ec9256db104760f3538c086a56cc385f1347f7c Mon Sep 17 00:00:00 2001 From: martyall Date: Thu, 12 Dec 2024 15:13:38 -0800 Subject: [PATCH 1/5] rust-impure shell loads with 1.74 --- nix/rust.nix | 6 ++---- .../kimchi_bindings/stubs/rust-toolchain.toml | 2 +- .../stubs/src/caml/caml_pointer.rs | 15 ++++++++------- .../stubs/src/pasta_fp_plonk_proof.rs | 4 ++-- .../stubs/src/pasta_fq_plonk_proof.rs | 2 +- .../kimchi_bindings/wasm/rust-toolchain.toml | 2 +- src/lib/crypto/kimchi_bindings/wasm/src/lib.rs | 1 - src/lib/crypto/proof-systems | 2 +- 8 files changed, 16 insertions(+), 18 deletions(-) diff --git a/nix/rust.nix b/nix/rust.nix index dad90f34a49..b1d5824c3d1 100644 --- a/nix/rust.nix +++ b/nix/rust.nix @@ -8,9 +8,7 @@ let # override stdenv.targetPlatform here, if neccesary }; toolchainHashes = { - "1.72" = "sha256-dxE7lmCFWlq0nl/wKcmYvpP9zqQbBitAQgZ1zx9Ooik="; - "nightly-2023-09-01" = - "sha256-zek9JAnRaoX8V0U2Y5ssXVe9tvoQ0ERGXfUCUGYdrMA="; + "1.74" = "sha256-PjvuouwTsYfNKW5Vi5Ye7y+lL7SsWGBxCtBOOm2z14c="; # copy the placeholder line with the correct toolchain name when adding a new toolchain # That is, # 1. Put the correct version name; @@ -19,7 +17,7 @@ let # error: hash mismatch in fixed-output derivation '/nix/store/XXXXX' # specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= # got: sha256-Q9UgzzvxLi4x9aWUJTn+/5EXekC98ODRU1TwhUs9RnY= - "placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + # "placeholder" = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; }; # rust-toolchain.toml -> { rustc, cargo, rust-analyzer, ... } rustChannelFromToolchainFileOf = file: diff --git a/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml b/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml index 5a82cfe5ab4..c2547d7095b 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml +++ b/src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml @@ -11,4 +11,4 @@ # 4. figure out the hashes of the (now obsolete) docker images used in CI rules that are failing, grep for these hashes and replace them with the new hashes [toolchain] -channel = "1.72" +channel = "1.74" diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs b/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs index ac4a1cac9bb..2a4e28260da 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs @@ -1,7 +1,7 @@ macro_rules! impl_caml_pointer { ($name: ident => $typ: ty) => { #[derive(std::fmt::Debug, Clone, ::ocaml_gen::CustomType)] - pub struct $name(pub ::std::rc::Rc<$typ>); + pub struct $name(pub ::std::rc::Rc>); impl $name { extern "C" fn caml_pointer_finalize(v: ocaml::Raw) { @@ -32,7 +32,7 @@ macro_rules! impl_caml_pointer { impl $name { pub fn create(x: $typ) -> $name { - $name(::std::rc::Rc::new(x)) + $name(::std::rc::Rc::new(std::cell::UnsafeCell::new(x))) } } @@ -40,7 +40,9 @@ macro_rules! impl_caml_pointer { type Target = $typ; fn deref(&self) -> &Self::Target { - &*self.0 + unsafe { + &*self.0.get() + } } } @@ -49,16 +51,15 @@ macro_rules! impl_caml_pointer { unsafe { // Wholely unsafe, Batman! // We would use [`get_mut_unchecked`] here, but it is nightly-only. - // Instead, we get coerce our constant pointer to a mutable - // pointer, in the knowledge that + // Instead, we use UnsafeCell in the knowledge that // * all of our mutations called from OCaml are blocking, so - // we won't have multiple live mutable references live + // we won't have multiple live mutable references // simultaneously, and // * the underlying pointer is in the correct state to be // mutable, since we can call [`get_mut_unchecked`] in // nightly, or can call [`get_mut`] and unwrap if this is // the only live reference. - &mut *(((&*self.0) as *const Self::Target) as *mut Self::Target) + &mut *self.0.get() } } } diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs index 71f7c2841a4..c0ee7346c34 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs @@ -72,7 +72,7 @@ pub fn caml_pasta_fp_plonk_proof_create( .collect() }; - let witness: Vec> = witness.iter().map(|x| (*x.0).clone()).collect(); + let witness: Vec> = witness.iter().map(|x| (**x).clone()).collect(); let witness: [Vec<_>; COLUMNS] = witness .try_into() .map_err(|_| ocaml::Error::Message("the witness should be a column of 15 vectors"))?; @@ -140,7 +140,7 @@ pub fn caml_pasta_fp_plonk_proof_create_and_verify( .collect() }; - let witness: Vec> = witness.iter().map(|x| (*x.0).clone()).collect(); + let witness: Vec> = witness.iter().map(|x| (**x).clone()).collect(); let witness: [Vec<_>; COLUMNS] = witness .try_into() .map_err(|_| ocaml::Error::Message("the witness should be a column of 15 vectors"))?; diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs index 8a6a2a3110b..34beeacc045 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs @@ -66,7 +66,7 @@ pub fn caml_pasta_fq_plonk_proof_create( .collect() }; - let witness: Vec> = witness.iter().map(|x| (*x.0).clone()).collect(); + let witness: Vec> = witness.iter().map(|x| (**x).clone()).collect(); let witness: [Vec<_>; COLUMNS] = witness .try_into() .expect("the witness should be a column of 15 vectors"); diff --git a/src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml b/src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml index b04d9639e65..e26dbd4c0ba 100644 --- a/src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml +++ b/src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml @@ -3,4 +3,4 @@ # This should stay in line with the versions in # - kimchi_bindings/js/node_js/build.sh # - kimchi_bindings/js/web/build.sh -channel = "nightly-2023-09-01" # roughly matches 1.72 +channel = "1.74" diff --git a/src/lib/crypto/kimchi_bindings/wasm/src/lib.rs b/src/lib/crypto/kimchi_bindings/wasm/src/lib.rs index 3f994cc2caa..89d527afa57 100644 --- a/src/lib/crypto/kimchi_bindings/wasm/src/lib.rs +++ b/src/lib/crypto/kimchi_bindings/wasm/src/lib.rs @@ -1,4 +1,3 @@ -#![feature(get_mut_unchecked)] //! The Marlin_plonk_stubs crate exports some functionalities //! and structures from the following the Rust crates to OCaml: //! diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index c54f985d5fe..bab4def927f 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit c54f985d5fe28dc5f34a275884bee486e0861521 +Subproject commit bab4def927f37e814418a196cf9f20a5cfa2ca14 From 1b667e57d13e7da424617f54b2a556edb1d3d0c3 Mon Sep 17 00:00:00 2001 From: martyall Date: Thu, 12 Dec 2024 16:45:16 -0800 Subject: [PATCH 2/5] remove refs to 1.72, change to 1.74 --- buildkite/src/Constants/ContainerImages.dhall | 6 +++--- dockerfiles/stages/1-build-deps | 7 +------ src/lib/crypto/kimchi_bindings/js/node_js/build.sh | 2 +- src/lib/crypto/kimchi_bindings/js/web/build.sh | 2 +- src/lib/crypto/proof-systems | 2 +- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/buildkite/src/Constants/ContainerImages.dhall b/buildkite/src/Constants/ContainerImages.dhall index b42dfd5ecae..5b5cebecc99 100644 --- a/buildkite/src/Constants/ContainerImages.dhall +++ b/buildkite/src/Constants/ContainerImages.dhall @@ -4,11 +4,11 @@ -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI { toolchainBase = "codaprotocol/ci-toolchain-base:v3" , minaToolchainBullseye = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a" , minaToolchainBookworm = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a" , minaToolchain = - "gcr.io/o1labs-192920/mina-toolchain@sha256:fee11e64a54fd8f026c4632fed7b7b9835b8262a037cdb156deb61d3d0aac8b2" + "gcr.io/o1labs-192920/mina-toolchain@sha256:8dc3a721b2be119db98a45970c21249f7ed9a95a75f2b330d7c0f37cdafcc99a" , elixirToolchain = "elixir:1.10-alpine" , nodeToolchain = "node:14.13.1-stretch-slim" , ubuntu2004 = "ubuntu:20.04" diff --git a/dockerfiles/stages/1-build-deps b/dockerfiles/stages/1-build-deps index 3e5348cc6e5..2dc9f42f32f 100644 --- a/dockerfiles/stages/1-build-deps +++ b/dockerfiles/stages/1-build-deps @@ -23,10 +23,7 @@ ARG GO_CAPNP_VERSION=v3.0.0-alpha.5 # This should stay in line with: # - src/lib/crypto/kimchi_bindings/stubs/rust-toolchain.toml # - src/lib/crypto/proof-systems/rust-toolchain.toml -ARG RUST_VERSION=1.72 -# Nightly Rust Version used for WebAssembly builds -# - src/lib/crypto/kimchi_bindings/wasm/rust-toolchain.toml -ARG RUST_NIGHTLY=2023-09-01 +ARG RUST_VERSION=1.74 # wasm-pack version ARG WASM_PACK_VERSION=v0.12.1 @@ -97,7 +94,6 @@ RUN curl -sL \ RUN curl -s "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xz -C /usr/lib/ # --- Rust install via rustup-init to a given RUST_VERSION -# --- Additionally, install RUST_NIGHTLY via rustup # For more about rustup-init see: https://github.com/rust-lang/rustup/blob/master/README.md # As opposed to introducing another shell script here (that mostly just determines the platform) # we just download the binary for the only platform we care about in this docker environment @@ -106,7 +102,6 @@ RUN curl --proto '=https' --tlsv1.2 -sSf -o /tmp/rustup-init \ https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \ && chmod +x /tmp/rustup-init \ && /tmp/rustup-init -y --default-toolchain "${RUST_VERSION}" --profile minimal --component rust-src --target wasm32-unknown-unknown \ - && $HOME/.cargo/bin/rustup toolchain install "nightly-${RUST_NIGHTLY}" --profile minimal --component rust-src --target wasm32-unknown-unknown --no-self-update \ && rm /tmp/rustup-init USER root diff --git a/src/lib/crypto/kimchi_bindings/js/node_js/build.sh b/src/lib/crypto/kimchi_bindings/js/node_js/build.sh index 41a7c57bc5c..c06efddedd6 100755 --- a/src/lib/crypto/kimchi_bindings/js/node_js/build.sh +++ b/src/lib/crypto/kimchi_bindings/js/node_js/build.sh @@ -4,7 +4,7 @@ set -euo pipefail if [[ -z "${PLONK_WASM_NODEJS-}" ]]; then export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296" # The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml - rustup run nightly-2023-09-01 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- -Z build-std=panic_abort,std --features nodejs + rustup run 1.74 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- -Z build-std=panic_abort,std --features nodejs else cp "$PLONK_WASM_NODEJS"/* -R . fi diff --git a/src/lib/crypto/kimchi_bindings/js/web/build.sh b/src/lib/crypto/kimchi_bindings/js/web/build.sh index f99564ef81d..c9713673e03 100755 --- a/src/lib/crypto/kimchi_bindings/js/web/build.sh +++ b/src/lib/crypto/kimchi_bindings/js/web/build.sh @@ -4,7 +4,7 @@ set -euo pipefail if [[ -z "${PLONK_WASM_WEB-}" ]]; then export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296" # The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml - rustup run nightly-2023-09-01 wasm-pack build --target web --out-dir ../js/web ../../wasm -- -Z build-std=panic_abort,std + rustup run 1.74 wasm-pack build --target web --out-dir ../js/web ../../wasm -- -Z build-std=panic_abort,std else cp "$PLONK_WASM_WEB"/* -R . fi diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index bab4def927f..229d0424266 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit bab4def927f37e814418a196cf9f20a5cfa2ca14 +Subproject commit 229d0424266c4e82418d23f56f6e22ec6813608c From d0b632d0b2fe185648989fa25b566ca6155d3aa7 Mon Sep 17 00:00:00 2001 From: martyall Date: Thu, 12 Dec 2024 16:51:48 -0800 Subject: [PATCH 3/5] update proof systems --- src/lib/crypto/proof-systems | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index 229d0424266..2144a278651 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit 229d0424266c4e82418d23f56f6e22ec6813608c +Subproject commit 2144a27865129c413f1be1db8fa8ff2c169f7b58 From a4a992f1ec2db64c5c4def553bab57f91b5d8411 Mon Sep 17 00:00:00 2001 From: martyall Date: Fri, 13 Dec 2024 04:45:38 -0800 Subject: [PATCH 4/5] formatting --- .../crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs b/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs index 2a4e28260da..33c3e67aa62 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/caml/caml_pointer.rs @@ -40,9 +40,7 @@ macro_rules! impl_caml_pointer { type Target = $typ; fn deref(&self) -> &Self::Target { - unsafe { - &*self.0.get() - } + unsafe { &*self.0.get() } } } @@ -59,7 +57,7 @@ macro_rules! impl_caml_pointer { // mutable, since we can call [`get_mut_unchecked`] in // nightly, or can call [`get_mut`] and unwrap if this is // the only live reference. - &mut *self.0.get() + &mut *self.0.get() } } } From 308fa2744f0033795b5b83685c64b7383c226606 Mon Sep 17 00:00:00 2001 From: martyall Date: Fri, 13 Dec 2024 04:51:40 -0800 Subject: [PATCH 5/5] don't panic if you need to build wasm bindings --- src/lib/crypto/kimchi_bindings/js/node_js/build.sh | 2 +- src/lib/crypto/kimchi_bindings/js/web/build.sh | 2 +- src/lib/crypto/proof-systems | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/crypto/kimchi_bindings/js/node_js/build.sh b/src/lib/crypto/kimchi_bindings/js/node_js/build.sh index c06efddedd6..37fef114428 100755 --- a/src/lib/crypto/kimchi_bindings/js/node_js/build.sh +++ b/src/lib/crypto/kimchi_bindings/js/node_js/build.sh @@ -4,7 +4,7 @@ set -euo pipefail if [[ -z "${PLONK_WASM_NODEJS-}" ]]; then export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296" # The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml - rustup run 1.74 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- -Z build-std=panic_abort,std --features nodejs + rustup run 1.74 wasm-pack build --target nodejs --out-dir ../js/node_js ../../wasm -- --features nodejs else cp "$PLONK_WASM_NODEJS"/* -R . fi diff --git a/src/lib/crypto/kimchi_bindings/js/web/build.sh b/src/lib/crypto/kimchi_bindings/js/web/build.sh index c9713673e03..de4fa8bd4f7 100755 --- a/src/lib/crypto/kimchi_bindings/js/web/build.sh +++ b/src/lib/crypto/kimchi_bindings/js/web/build.sh @@ -4,7 +4,7 @@ set -euo pipefail if [[ -z "${PLONK_WASM_WEB-}" ]]; then export RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--no-check-features -C link-arg=--max-memory=4294967296" # The version should stay in line with the one in kimchi_bindings/wasm/rust-toolchain.toml - rustup run 1.74 wasm-pack build --target web --out-dir ../js/web ../../wasm -- -Z build-std=panic_abort,std + rustup run 1.74 wasm-pack build --target web --out-dir ../js/web ../../wasm else cp "$PLONK_WASM_WEB"/* -R . fi diff --git a/src/lib/crypto/proof-systems b/src/lib/crypto/proof-systems index 2144a278651..2fc23249a3d 160000 --- a/src/lib/crypto/proof-systems +++ b/src/lib/crypto/proof-systems @@ -1 +1 @@ -Subproject commit 2144a27865129c413f1be1db8fa8ff2c169f7b58 +Subproject commit 2fc23249a3d498da793780015ee44c0692ed45f0