From 464c2c67138dba780deb35a12baf99700dfa5742 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 16 Aug 2022 16:49:32 +0200 Subject: [PATCH] Colab and Kaggle loggers removeHandler (#8985) Resolve duplicate outputs in colab. Also optimizes Colab and Kaggle environment checks. Signed-off-by: Glenn Jocher Signed-off-by: Glenn Jocher --- utils/general.py | 68 +++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 41 deletions(-) diff --git a/utils/general.py b/utils/general.py index 65130ca57762..2a3ce37cd853 100755 --- a/utils/general.py +++ b/utils/general.py @@ -56,13 +56,35 @@ os.environ['OMP_NUM_THREADS'] = '1' if platform.system() == 'darwin' else str(NUM_THREADS) # OpenMP (PyTorch and SciPy) +def is_ascii(s=''): + # Is string composed of all ASCII (no UTF) characters? (note str().isascii() introduced in python 3.7) + s = str(s) # convert list, tuple, None, etc. to str + return len(s.encode().decode('ascii', 'ignore')) == len(s) + + +def is_chinese(s='人工智能'): + # Is string composed of any Chinese characters? + return bool(re.search('[\u4e00-\u9fff]', str(s))) + + +def is_colab(): + # Is environment a Google Colab instance? + return 'COLAB_GPU' in os.environ + + def is_kaggle(): # Is environment a Kaggle Notebook? - try: - assert os.environ.get('PWD') == '/kaggle/working' - assert os.environ.get('KAGGLE_URL_BASE') == 'https://www.kaggle.com' + return os.environ.get('PWD') == '/kaggle/working' and os.environ.get('KAGGLE_URL_BASE') == 'https://www.kaggle.com' + + +def is_docker() -> bool: + """Check if the process runs inside a docker container.""" + if Path("/.dockerenv").exists(): return True - except AssertionError: + try: # check if docker is in control groups + with open("/proc/self/cgroup") as file: + return any("docker" in line for line in file) + except OSError: return False @@ -82,7 +104,7 @@ def is_writeable(dir, test=False): def set_logging(name=None, verbose=VERBOSE): # Sets level and returns logger - if is_kaggle(): + if is_kaggle() or is_colab(): for h in logging.root.handlers: logging.root.removeHandler(h) # remove all handlers associated with the root logger object rank = int(os.getenv('RANK', -1)) # rank in world for Multi-GPU trainings @@ -228,42 +250,6 @@ def get_latest_run(search_dir='.'): return max(last_list, key=os.path.getctime) if last_list else '' -def is_docker() -> bool: - """Check if the process runs inside a docker container.""" - if Path("/.dockerenv").exists(): - return True - try: # check if docker is in control groups - with open("/proc/self/cgroup") as file: - return any("docker" in line for line in file) - except OSError: - return False - - -def is_colab(): - # Is environment a Google Colab instance? - try: - import google.colab - return True - except ImportError: - return False - - -def is_pip(): - # Is file in a pip package? - return 'site-packages' in Path(__file__).resolve().parts - - -def is_ascii(s=''): - # Is string composed of all ASCII (no UTF) characters? (note str().isascii() introduced in python 3.7) - s = str(s) # convert list, tuple, None, etc. to str - return len(s.encode().decode('ascii', 'ignore')) == len(s) - - -def is_chinese(s='人工智能'): - # Is string composed of any Chinese characters? - return bool(re.search('[\u4e00-\u9fff]', str(s))) - - def emojis(str=''): # Return platform-dependent emoji-safe version of string return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str