fix merge stats

huggingface · Jul 10, 2024 · 0814338 · 0814338
1 parent bc19878
commit 0814338
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/datatrove/executor/slurm_nodes.py b/src/datatrove/executor/slurm_nodes.py
@@ -199,7 +199,7 @@ def launch_merge_stats(self):
                     "cpus-per-task": 1,
                     "dependency": f"afterok:{self.job_id}",
                 },
-                f'merge_stats {self.logging_dir.resolve_paths("stats")} '
+                f'srun --environment=datatrove merge_stats {self.logging_dir.resolve_paths("stats")} '
                 f'-o {self.logging_dir.resolve_paths("stats.json")}',
             )
         )