Merge pull request privacy-scaling-explorations#147 from input-output…

…-hk/dev-feature/99-cache-fbsm-test-bases

Speed up testing of fixed-base scalar mul

# Project Context

This builds on top of privacy-scaling-explorations#146.

The corresponding Galois internal issue is [Galois#99](https://gitlab-ext.galois.com/iog-midnight/halo2/-/issues/99).

# Issue Description

For Pallas the fbsm tests already took 1+ hours, and blew up to 17+ hours for Pluto/Eris, when support for those curves was added in privacy-scaling-explorations#145. The goal of this issue is to greatly reduce the test times by caching to disk the expensive data that takes 17+ hours to compute.

The end result is that the test times are back down to a few minutes :)

halo2_gadgets/Cargo.toml

@@ -43,6 +43,13 @@ goldenfile = "1.5.2"
 criterion = { version = "0.3", features = ["html_reports"] }
 proptest = "1.0.0"
 seq-macro = "0.3.5"
+tempfile = "3.3.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+halo2curves = { git = "https://github.com/davidnevadoc/halo2curves", rev = "33b1d2fe0f4576fde180e171eeab24f3d8573f2e", features = ["bits", "derive_serde"] }
+# More recent versions of halo2curves enable ["pasta_curves/serde"], but our
+# version above is too old so we do it ourselves.
+pasta_curves = { version = "0.5.0", features = ["serde"] }
 
 [target.'cfg(unix)'.dev-dependencies]
 pprof = { version = "0.8", features = ["criterion", "flamegraph"] } # MSRV 1.56
@@ -55,6 +62,7 @@ dev-graph = ["halo2_proofs/dev-graph", "plotters"]
 circuit-params = ["halo2_proofs/circuit-params"]
 test-dependencies = ["proptest"]
 unstable = []
+cache-test-data = []
 
 [[bench]]
 name = "duplex"

halo2_gadgets/data/.gitattributes

@@ -0,0 +1 @@
+*.json binary

halo2_gadgets/data/README.md

@@ -0,0 +1,7 @@
+This directory contains cached data used to speed up testing.
+
+The various `*fixed_point*.json` files are used to test fixed-base scalar multiplication. They are generated by running
+
+    cargo test -p halo2_gadgets cache_to_disk --features cache-test-data
+
+Warning: the full generation process is very slow, and took 17.5 hours on a 4 core Intel(R) Core(TM) i7-3520M CPU @ 2.90GHz with 16GB memory.

halo2_gadgets/src/ecc.rs

@@ -637,12 +637,13 @@ pub(crate) mod tests {
 
     use super::{
         chip::{
-            find_zs_and_us, BaseFieldElem, EccChip, EccConfig, EccCurve, EccField, FixedPoint,
-            FullScalar, ShortScalar, H, NUM_WINDOWS_SHORT,
+            BaseFieldElem, EccChip, EccConfig, EccCurve, EccField, FixedPoint, FullScalar,
+            ShortScalar, H, NUM_WINDOWS_SHORT,
         },
         FixedPoints,
     };
     use crate::ecc::chip::mul_fixed::FbsmCurve;
+    use crate::ecc::chip::{compute_lagrange_coeffs, constants::serialize::CachedFixedPoint};
     use crate::utilities::{lookup_range_check::LookupRangeCheckConfig, pluto_eris::*};
 
     #[derive(Debug, Eq, PartialEq, Clone)]
@@ -653,45 +654,97 @@ pub(crate) mod tests {
     pub(crate) struct FullWidth<C: FbsmCurve> {
         base: C,
         zs_and_us: &'static [(u64, [C::Base; H])],
+        lagrange_coeffs: Vec<[C::Base; H]>,
     }
     #[derive(Debug, Eq, PartialEq, Clone)]
     pub(crate) struct BaseField<C: FbsmCurve> {
         base: C,
         zs_and_us: &'static [(u64, [C::Base; H])],
+        lagrange_coeffs: Vec<[C::Base; H]>,
     }
     #[derive(Debug, Eq, PartialEq, Clone)]
     pub(crate) struct Short<C: FbsmCurve> {
         base: C,
         zs_and_us: &'static [(u64, [C::Base; H])],
+        lagrange_coeffs: Vec<[C::Base; H]>,
     }
 
+    // Read very expensive test data from cache on disk.
     lazy_static! {
         static ref PALLAS_BASE: pallas::Affine = pallas::Point::generator().to_affine();
         static ref PALLAS_ZS_AND_US: Vec<(u64, [pallas::Base; H])> =
-            find_zs_and_us(*PALLAS_BASE, pallas::Affine::NUM_WINDOWS_FULL).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *PALLAS_BASE,
+                pallas::Affine::NUM_WINDOWS_FULL,
+                "pallas_fixed_point_full",
+            );
         static ref PALLAS_ZS_AND_US_SHORT: Vec<(u64, [pallas::Base; H])> =
-            find_zs_and_us(*PALLAS_BASE, NUM_WINDOWS_SHORT).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *PALLAS_BASE,
+                NUM_WINDOWS_SHORT,
+                "pallas_fixed_point_short",
+            );
+        static ref PALLAS_LAGRANGE_COEFFS_FULL: Vec<[pallas::Base; H]> =
+            compute_lagrange_coeffs(*PALLAS_BASE, pallas::Affine::NUM_WINDOWS_FULL);
+        static ref PALLAS_LAGRANGE_COEFFS_SHORT: Vec<[pallas::Base; H]> =
+            compute_lagrange_coeffs(*PALLAS_BASE, NUM_WINDOWS_SHORT);
     }
     lazy_static! {
         static ref VESTA_BASE: vesta::Affine = vesta::Point::generator().to_affine();
         static ref VESTA_ZS_AND_US: Vec<(u64, [vesta::Base; H])> =
-            find_zs_and_us(*VESTA_BASE, vesta::Affine::NUM_WINDOWS_FULL).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *VESTA_BASE,
+                vesta::Affine::NUM_WINDOWS_FULL,
+                "vesta_fixed_point_full",
+            );
         static ref VESTA_ZS_AND_US_SHORT: Vec<(u64, [vesta::Base; H])> =
-            find_zs_and_us(*VESTA_BASE, NUM_WINDOWS_SHORT).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *VESTA_BASE,
+                NUM_WINDOWS_SHORT,
+                "vesta_fixed_point_short",
+            );
+        static ref VESTA_LAGRANGE_COEFFS_FULL: Vec<[vesta::Base; H]> =
+            compute_lagrange_coeffs(*VESTA_BASE, vesta::Affine::NUM_WINDOWS_FULL);
+        static ref VESTA_LAGRANGE_COEFFS_SHORT: Vec<[vesta::Base; H]> =
+            compute_lagrange_coeffs(*VESTA_BASE, NUM_WINDOWS_SHORT);
     }
     lazy_static! {
         static ref PLUTO_BASE: pluto::Affine = pluto::Point::generator().to_affine();
         static ref PLUTO_ZS_AND_US: Vec<(u64, [pluto::Base; H])> =
-            find_zs_and_us(*PLUTO_BASE, pluto::Affine::NUM_WINDOWS_FULL).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *PLUTO_BASE,
+                pluto::Affine::NUM_WINDOWS_FULL,
+                "pluto_fixed_point_full",
+            );
         static ref PLUTO_ZS_AND_US_SHORT: Vec<(u64, [pluto::Base; H])> =
-            find_zs_and_us(*PLUTO_BASE, NUM_WINDOWS_SHORT).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *PLUTO_BASE,
+                NUM_WINDOWS_SHORT,
+                "pluto_fixed_point_short",
+            );
+        static ref PLUTO_LAGRANGE_COEFFS_FULL: Vec<[pluto::Base; H]> =
+            compute_lagrange_coeffs(*PLUTO_BASE, pluto::Affine::NUM_WINDOWS_FULL);
+        static ref PLUTO_LAGRANGE_COEFFS_SHORT: Vec<[pluto::Base; H]> =
+            compute_lagrange_coeffs(*PLUTO_BASE, NUM_WINDOWS_SHORT);
     }
     lazy_static! {
         static ref ERIS_BASE: eris::Affine = eris::Point::generator().to_affine();
         static ref ERIS_ZS_AND_US: Vec<(u64, [eris::Base; H])> =
-            find_zs_and_us(*ERIS_BASE, eris::Affine::NUM_WINDOWS_FULL).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *ERIS_BASE,
+                eris::Affine::NUM_WINDOWS_FULL,
+                "eris_fixed_point_full",
+            );
         static ref ERIS_ZS_AND_US_SHORT: Vec<(u64, [eris::Base; H])> =
-            find_zs_and_us(*ERIS_BASE, NUM_WINDOWS_SHORT).unwrap();
+            CachedFixedPoint::read_zs_and_us_from_disk_and_verify(
+                *ERIS_BASE,
+                NUM_WINDOWS_SHORT,
+                "eris_fixed_point_short",
+            );
+        static ref ERIS_LAGRANGE_COEFFS_FULL: Vec<[eris::Base; H]> =
+            compute_lagrange_coeffs(*ERIS_BASE, eris::Affine::NUM_WINDOWS_FULL);
+        static ref ERIS_LAGRANGE_COEFFS_SHORT: Vec<[eris::Base; H]> =
+            compute_lagrange_coeffs(*ERIS_BASE, NUM_WINDOWS_SHORT);
     }
 
     /// A fixed point that can be computed from a distinguished generator.
@@ -702,7 +755,12 @@ pub(crate) mod tests {
     impl<C: FbsmCurve> FullWidth<C> {
         pub(crate) fn from_parts(base: C, zs_and_us: &'static [(u64, [C::Base; H])]) -> Self {
             assert_eq!(zs_and_us.len(), Self::NUM_WINDOWS);
-            FullWidth { base, zs_and_us }
+            let lagrange_coeffs = compute_lagrange_coeffs(base, Self::NUM_WINDOWS);
+            FullWidth {
+                base,
+                zs_and_us,
+                lagrange_coeffs,
+            }
         }
     }
 
@@ -716,13 +774,18 @@ pub(crate) mod tests {
         fn zs_and_us(&self) -> &'static [(u64, [C::Base; H])] {
             self.zs_and_us
         }
+
+        fn lagrange_coeffs(&self) -> Vec<[C::Base; H]> {
+            self.lagrange_coeffs.clone()
+        }
     }
 
     impl FromGenerator<pallas::Affine> for FullWidth<pallas::Affine> {
         fn from_generator() -> Self {
             FullWidth {
                 base: *PALLAS_BASE,
                 zs_and_us: &PALLAS_ZS_AND_US,
+                lagrange_coeffs: PALLAS_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -732,6 +795,7 @@ pub(crate) mod tests {
             FullWidth {
                 base: *VESTA_BASE,
                 zs_and_us: &VESTA_ZS_AND_US,
+                lagrange_coeffs: VESTA_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -741,6 +805,7 @@ pub(crate) mod tests {
             FullWidth {
                 base: *PLUTO_BASE,
                 zs_and_us: &PLUTO_ZS_AND_US,
+                lagrange_coeffs: PLUTO_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -750,14 +815,20 @@ pub(crate) mod tests {
             FullWidth {
                 base: *ERIS_BASE,
                 zs_and_us: &ERIS_ZS_AND_US,
+                lagrange_coeffs: ERIS_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
 
     impl<C: FbsmCurve> BaseField<C> {
         pub(crate) fn from_parts(base: C, zs_and_us: &'static [(u64, [C::Base; H])]) -> Self {
             assert_eq!(zs_and_us.len(), Self::NUM_WINDOWS);
-            BaseField { base, zs_and_us }
+            let lagrange_coeffs = compute_lagrange_coeffs(base, Self::NUM_WINDOWS);
+            BaseField {
+                base,
+                zs_and_us,
+                lagrange_coeffs,
+            }
         }
     }
 
@@ -771,13 +842,18 @@ pub(crate) mod tests {
         fn zs_and_us(&self) -> &'static [(u64, [C::Base; H])] {
             self.zs_and_us
         }
+
+        fn lagrange_coeffs(&self) -> Vec<[C::Base; H]> {
+            self.lagrange_coeffs.clone()
+        }
     }
 
     impl FromGenerator<pallas::Affine> for BaseField<pallas::Affine> {
         fn from_generator() -> Self {
             BaseField {
                 base: *PALLAS_BASE,
                 zs_and_us: &PALLAS_ZS_AND_US,
+                lagrange_coeffs: PALLAS_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -787,6 +863,7 @@ pub(crate) mod tests {
             BaseField {
                 base: *VESTA_BASE,
                 zs_and_us: &VESTA_ZS_AND_US,
+                lagrange_coeffs: VESTA_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -796,6 +873,7 @@ pub(crate) mod tests {
             BaseField {
                 base: *PLUTO_BASE,
                 zs_and_us: &PLUTO_ZS_AND_US,
+                lagrange_coeffs: PLUTO_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
@@ -805,14 +883,20 @@ pub(crate) mod tests {
             BaseField {
                 base: *ERIS_BASE,
                 zs_and_us: &ERIS_ZS_AND_US,
+                lagrange_coeffs: ERIS_LAGRANGE_COEFFS_FULL.to_vec(),
             }
         }
     }
 
     impl<C: FbsmCurve> Short<C> {
         pub(crate) fn from_parts(base: C, zs_and_us: &'static [(u64, [C::Base; H])]) -> Self {
             assert_eq!(zs_and_us.len(), Self::NUM_WINDOWS);
-            Short { base, zs_and_us }
+            let lagrange_coeffs = compute_lagrange_coeffs(base, Self::NUM_WINDOWS);
+            Short {
+                base,
+                zs_and_us,
+                lagrange_coeffs,
+            }
         }
     }
 
@@ -826,13 +910,18 @@ pub(crate) mod tests {
         fn zs_and_us(&self) -> &'static [(u64, [C::Base; H])] {
             self.zs_and_us
         }
+
+        fn lagrange_coeffs(&self) -> Vec<[C::Base; H]> {
+            self.lagrange_coeffs.clone()
+        }
     }
 
     impl FromGenerator<pallas::Affine> for Short<pallas::Affine> {
         fn from_generator() -> Self {
             Short {
                 base: *PALLAS_BASE,
                 zs_and_us: &PALLAS_ZS_AND_US_SHORT,
+                lagrange_coeffs: PALLAS_LAGRANGE_COEFFS_SHORT.to_vec(),
             }
         }
     }
@@ -842,6 +931,7 @@ pub(crate) mod tests {
             Short {
                 base: *VESTA_BASE,
                 zs_and_us: &VESTA_ZS_AND_US_SHORT,
+                lagrange_coeffs: VESTA_LAGRANGE_COEFFS_SHORT.to_vec(),
             }
         }
     }
@@ -851,6 +941,7 @@ pub(crate) mod tests {
             Short {
                 base: *PLUTO_BASE,
                 zs_and_us: &PLUTO_ZS_AND_US_SHORT,
+                lagrange_coeffs: PLUTO_LAGRANGE_COEFFS_SHORT.to_vec(),
             }
         }
     }
@@ -860,6 +951,7 @@ pub(crate) mod tests {
             Short {
                 base: *ERIS_BASE,
                 zs_and_us: &ERIS_ZS_AND_US_SHORT,
+                lagrange_coeffs: ERIS_LAGRANGE_COEFFS_SHORT.to_vec(),
             }
         }
     }