diff --git a/benchmarks/oss.py b/benchmarks/oss.py index cfa5ed71d..ff8a5e7c0 100755 --- a/benchmarks/oss.py +++ b/benchmarks/oss.py @@ -103,8 +103,8 @@ def validate_benchmark(measurements, final_loss, args, check_regression): logging.info(f"[{rank}] : Median speed: {median:.2f} +/- {mad:.2f}") if check_regression and rank == 0: - assert median + 3.0 * mad > golden_data["reference_speed"], ( - f"Speed regression detected: " f"{median + 3.0 * mad} vs. {golden_data['reference_speed']}" + assert median + 8.0 * mad > golden_data["reference_speed"], ( + f"Speed regression detected: " f"{median + 8.0 * mad} vs. {golden_data['reference_speed']}" ) assert max_memory < 1.05 * golden_data["reference_memory"], ( f"Memory use regression detected: " f"{max_memory} vs. {1.05* golden_data['reference_memory']}"