Multiply prefill time by prompt length

neuralmagic · Sep 18, 2023 · cceeddf · cceeddf
1 parent b1ed606
commit cceeddf
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/examples/chatbot_llm/chatbot.py b/examples/chatbot_llm/chatbot.py
@@ -161,7 +161,9 @@ def main(
         print("Bot: ", response.sequences[0])
         if show_tokens_per_sec:
             times = pipeline.timer_manager.times
-            prefill_speed = 1.0 / times["engine_prompt_prefill_single"]
+            prefill_speed = (
+                1.0 * prompt_sequence_length / times["engine_prompt_prefill_single"]
+            )
             generation_speed = 1.0 / times["engine_token_generation_single"]
             print(
                 f"[prefill: {prefill_speed:.2f} tokens/sec]",