From 8292216074d69f714e416a21c397b16207bedb74 Mon Sep 17 00:00:00 2001 From: Alexander Camuto <45801863+alexander-camuto@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:09:35 +0000 Subject: [PATCH 1/3] fix: lookup safety = 1 during calibration falls OOR --- .github/workflows/rust.yml | 7 ++---- tests/integration_tests.rs | 50 ++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1315efaff..48898812a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -392,6 +392,8 @@ jobs: - name: Replace memory definition in nodejs run: | sed -i "3s|.*|imports['env'] = {memory: new WebAssembly.Memory({initial:20,maximum:65536,shared:true})}|" tests/wasm/nodejs/ezkl.js + - name: KZG prove and verify tests (public outputs) + run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_tight_lookup_::t - name: IPA prove and verify tests run: cargo nextest run --release --verbose tests::ipa_prove_and_verify_::t --test-threads 1 - name: IPA prove and verify tests (ipa outputs) @@ -412,8 +414,6 @@ jobs: run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_with_overflow_fixed_params_ - name: KZG prove and verify tests (public outputs) run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t - - name: KZG prove and verify tests (public outputs + column overflow) - run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_::t - name: KZG prove and verify tests (public inputs) run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_public_input - name: KZG prove and verify tests (fixed params) @@ -456,7 +456,6 @@ jobs: - name: KZG prove and verify tests (hashed outputs) run: cargo nextest run --release --verbose tests::kzg_prove_and_verify_hashed --features icicle --test-threads 1 - prove-and-verify-mock-aggr-tests: runs-on: self-hosted needs: [build, library-tests] @@ -509,8 +508,6 @@ jobs: - name: KZG tests run: cargo nextest run --release --verbose tests_aggr::kzg_aggr_prove_and_verify_ --test-threads 4 -- --include-ignored - - prove-and-verify-aggr-evm-tests: runs-on: large-self-hosted needs: [build, library-tests] diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 5afd8a2d5..ce38f5040 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -484,7 +484,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(test_dir.path().to_str().unwrap(), test); - aggr_prove_and_verify(path, test.to_string(), "private", "private", "public", Commitments::KZG); + aggr_prove_and_verify(path, test.to_string(), "private", "private", "public", Commitments::KZG, 2); test_dir.close().unwrap(); } }); @@ -790,7 +790,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -799,7 +799,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 3, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 3, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -808,7 +808,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 4, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 4, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -817,7 +817,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 8, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 8, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -826,7 +826,16 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 2); + test_dir.close().unwrap(); + } + + #(#[test_case(TESTS[N])])* + fn kzg_prove_and_verify_tight_lookup_(test: &str) { + crate::native_tests::init_binary(); + let test_dir = TempDir::new(test).unwrap(); + let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::KZG, 1); test_dir.close().unwrap(); } @@ -835,7 +844,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::IPA); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, false, "single", Commitments::IPA, 2); test_dir.close().unwrap(); } @@ -844,7 +853,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "public", "private", "public", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "public", "private", "public", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -853,7 +862,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -862,7 +871,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "hashed", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "hashed", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -871,7 +880,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -880,7 +889,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::IPA); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "polycommit", 1, None, false, "single", Commitments::IPA, 2); test_dir.close().unwrap(); } @@ -895,7 +904,7 @@ mod native_tests { let test_dir = TempDir::new(test).unwrap(); env_logger::init(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, true, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "private", "public", 1, None, true, "single", Commitments::KZG, 2); #[cfg(not(feature = "icicle"))] run_js_tests(path, test.to_string(), "testWasm", false); // test_dir.close().unwrap(); @@ -908,7 +917,7 @@ mod native_tests { let test_dir = TempDir::new(test).unwrap(); env_logger::init(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, true, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "safe", "private", "fixed", "public", 1, None, true, "single", Commitments::KZG, 2); #[cfg(not(feature = "icicle"))] run_js_tests(path, test.to_string(), "testWasm", false); test_dir.close().unwrap(); @@ -924,7 +933,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(path, test); - prove_and_verify(path, test.to_string(), "unsafe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG); + prove_and_verify(path, test.to_string(), "unsafe", "private", "fixed", "public", 1, None, false, "single", Commitments::KZG, 2); test_dir.close().unwrap(); } @@ -1338,6 +1347,7 @@ mod native_tests { false, &mut tolerance, Commitments::KZG, + 2, ); if tolerance > 0.0 { @@ -1478,6 +1488,7 @@ mod native_tests { div_rebasing: bool, tolerance: &mut f32, commitment: Commitments, + lookup_safety_margin: usize, ) { let mut args = vec![ "gen-settings".to_string(), @@ -1518,6 +1529,7 @@ mod native_tests { test_dir, example_name ), format!("--target={}", cal_target), + format!("--lookup-safety-margin={}", lookup_safety_margin), ]; if let Some(scales) = scales_to_use { @@ -1613,6 +1625,7 @@ mod native_tests { div_rebasing, &mut 0.0, Commitments::KZG, + 2, ); println!( @@ -1666,6 +1679,7 @@ mod native_tests { false, "for-aggr", Commitments::KZG, + 2, ); let status = Command::new(format!("{}/release/ezkl", *CARGO_TARGET_DIR)) .args([ @@ -1700,6 +1714,7 @@ mod native_tests { false, "for-aggr", Commitments::KZG, + 2, ); download_srs(23, commitment); @@ -1861,6 +1876,7 @@ mod native_tests { overflow: bool, proof_type: &str, commitment: Commitments, + lookup_safety_margin: usize, ) { let target_str = if overflow { "resources/col-overflow" @@ -1881,6 +1897,7 @@ mod native_tests { false, &mut 0.0, commitment, + lookup_safety_margin, ); let settings_path = format!("{}/{}/settings.json", test_dir, example_name); @@ -1993,6 +2010,7 @@ mod native_tests { false, "single", Commitments::KZG, + 2, ); let settings_path = format!("{}/{}/settings.json", test_dir, example_name); @@ -2087,6 +2105,7 @@ mod native_tests { false, "single", Commitments::KZG, + 2, ); let settings_path = format!("{}/{}/settings.json", test_dir, example_name); @@ -2235,6 +2254,7 @@ mod native_tests { false, &mut 0.0, Commitments::KZG, + 2, ); let model_path = format!("{}/{}/network.compiled", test_dir, example_name); From 46d551bea971b78bdb799d3e3e6056ccb601fa18 Mon Sep 17 00:00:00 2001 From: dante <45801863+alexander-camuto@users.noreply.github.com> Date: Thu, 21 Mar 2024 00:23:08 +0000 Subject: [PATCH 2/3] rm dumb padding --- src/graph/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/graph/mod.rs b/src/graph/mod.rs index bc59d2c9e..b20f214a3 100644 --- a/src/graph/mod.rs +++ b/src/graph/mod.rs @@ -1049,14 +1049,10 @@ impl GraphCircuit { } fn calc_safe_lookup_range(min_max_lookup: Range, lookup_safety_margin: i128) -> Range { - let mut margin = ( + let margin = ( lookup_safety_margin * min_max_lookup.0, lookup_safety_margin * min_max_lookup.1, ); - if lookup_safety_margin == 1 { - margin.0 += 4; - margin.1 += 4; - } margin } From 747783886b8e4b9ba0282dad05e5ee3dbcc2dc04 Mon Sep 17 00:00:00 2001 From: dante <45801863+alexander-camuto@users.noreply.github.com> Date: Thu, 21 Mar 2024 01:50:51 +0000 Subject: [PATCH 3/3] patch gpu --- tests/integration_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ce38f5040..a1893eed9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -484,7 +484,7 @@ mod native_tests { crate::native_tests::init_binary(); let test_dir = TempDir::new(test).unwrap(); let path = test_dir.path().to_str().unwrap(); crate::native_tests::mv_test_(test_dir.path().to_str().unwrap(), test); - aggr_prove_and_verify(path, test.to_string(), "private", "private", "public", Commitments::KZG, 2); + aggr_prove_and_verify(path, test.to_string(), "private", "private", "public", Commitments::KZG); test_dir.close().unwrap(); } });