make train test traceable when error occurs

.ci-cd/build.sh

@@ -41,7 +41,7 @@ function check_style() {
 
 function test() {
     cd alf
-    python3 -m unittest discover -p "*_test.py" -v 
+    python3 -m unittest discover -p "*_test.py" -v
     cd ..
 }
 

alf/bin/train_test.py

@@ -18,7 +18,9 @@
 import subprocess
 from pathlib import Path
 import numpy as np
+import sys
 
+import logging as sys_logging
 from absl import logging
 import tensorflow as tf
 
@@ -34,17 +36,27 @@ def run_and_stream(cmd, cwd):
         cwd (str): working directory for the process
     """
     logging.info("Running %s", " ".join(cmd))
+
+    # create a logger for sub process outputs
+    # 1. logging all outputs of sub process to sys.stderr to make it traceable when an error
+    #   occurs (ci suppresses stdout output to prevent producing a big log file than 4 MB)
+    # 2. set a simple formatter without prefix for the logger, because a log_prefix
+    #   already exists for sub process log
+    logger = logging.ABSLLogger('')
+    handler = sys_logging.StreamHandler(sys.stderr)
+    handler.setFormatter(sys_logging.Formatter('%(message)s'))
+    logger.addHandler(handler)
+
     process = subprocess.Popen(
         cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
 
     while process.poll() is None:
         with io.TextIOWrapper(process.stdout, encoding="utf-8") as text_io:
             for line in text_io:
-                logging.info(line.strip())
+                logger.info(line.strip())
 
-    if process.returncode != 0:
-        logging.error("cmd: {0} exited with code {1}".format(
-            " ".join(cmd), process.returncode))
+    assert process.returncode == 0, ("cmd: {0} exit abnormally".format(
+        " ".join(cmd)))
 
 
 def get_metrics_from_eval_tfevents(eval_dir):