From f5e3053787696a368c19e58b27de9fb520772d4c Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas00@users.noreply.github.com>
Date: Sat, 7 Nov 2020 18:55:17 -0800
Subject: [PATCH 01/11] [testing utils] get_auto_remove_tmp_dir default change

Now that I have been using `get_auto_remove_tmp_dir default change` for a while, I realized that the defaults aren't most optimal.

99% of the time we want the tmp dir to be empty at the beginning of the test - so changing the default to `before=True` - this shouldn't impact any tests since this feature is used only during debug.
---
 src/transformers/testing_utils.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index 02998bcfd656b6..e35751859ea683 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -518,16 +518,17 @@ class solves this problem by sorting out all the basic paths and provides easy a
 
     Feature 2: Flexible auto-removable temp dirs which are guaranteed to get removed at the end of test.
 
-    In all the following scenarios the temp dir will be auto-removed at the end of test, unless `after=False`.
+    In all the following scenarios the temp dir will be cleared out at the beginning of the test (unless `before=False`) 
+    and auto-removed at the end of test (unless `after=False`).
 
-    # 1. create a unique temp dir, `tmp_dir` will contain the path to the created temp dir
+    1. create a unique temp dir, `tmp_dir` will contain the path to the created temp dir
 
     ::
 
         def test_whatever(self):
             tmp_dir = self.get_auto_remove_tmp_dir()
 
-    # 2. create a temp dir of my choice and delete it at the end - useful for debug when you want to # monitor a
+    2. create a temp dir of my choice and delete it at the end - useful for debug when you want to monitor a
     specific directory
 
     ::
@@ -535,20 +536,19 @@ def test_whatever(self):
         def test_whatever(self):
             tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
 
-    # 3. create a temp dir of my choice and do not delete it at the end - useful for when you want # to look at the
+    3. create a temp dir of my choice and do not delete it at the end - useful for when you want to look at the
     temp results
 
     ::
         def test_whatever(self):
             tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
 
-    # 4. create a temp dir of my choice and ensure to delete it right away - useful for when you # disabled deletion in
-    the previous test run and want to make sure the that tmp dir is empty # before the new test is run
+    4. create a temp dir of my choice, but if it already exists leave its contents as they are
 
     ::
 
         def test_whatever(self):
-            tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)
+            tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=False)
 
     Note 1: In order to run the equivalent of `rm -r` safely, only subdirs of the project repository checkout are
     allowed if an explicit `tmp_dir` is used, so that by mistake no `/tmp` or similar important part of the filesystem
@@ -654,15 +654,15 @@ def get_env(self):
         env["PYTHONPATH"] = ":".join(paths)
         return env
 
-    def get_auto_remove_tmp_dir(self, tmp_dir=None, after=True, before=False):
+    def get_auto_remove_tmp_dir(self, tmp_dir=None, after=True, before=True):
         """
         Args:
             tmp_dir (:obj:`string`, `optional`):
                 use this path, if None a unique path will be assigned
-            before (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            before (:obj:`bool`, `optional`, defaults to :obj:`True`):
                 if `True` and tmp dir already exists make sure to empty it right away
             after (:obj:`bool`, `optional`, defaults to :obj:`True`):
-                delete the tmp dir at the end of the test
+                if `True` delete the tmp dir at the end of the test
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp

From 32a6019916f24ac7c29bfeb83e1fadb13afd30d8 Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sat, 7 Nov 2020 19:47:32 -0800
Subject: [PATCH 02/11] simplify things

---
 src/transformers/testing_utils.py | 73 ++++++++++++++++++++-----------
 1 file changed, 47 insertions(+), 26 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index e35751859ea683..c0bc3a21433ffd 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -518,41 +518,41 @@ class solves this problem by sorting out all the basic paths and provides easy a
 
     Feature 2: Flexible auto-removable temp dirs which are guaranteed to get removed at the end of test.
 
-    In all the following scenarios the temp dir will be cleared out at the beginning of the test (unless `before=False`) 
-    and auto-removed at the end of test (unless `after=False`).
-
-    1. create a unique temp dir, `tmp_dir` will contain the path to the created temp dir
+    1. Create a unique temporary dir:
 
     ::
 
         def test_whatever(self):
             tmp_dir = self.get_auto_remove_tmp_dir()
 
-    2. create a temp dir of my choice and delete it at the end - useful for debug when you want to monitor a
-    specific directory
+    ``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
+
+
+    2. Create a temporary dir of my choice, ensure it's empty before the test starts and don't
+    empty it after the test.
 
     ::
 
         def test_whatever(self):
-            tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
+            tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
 
-    3. create a temp dir of my choice and do not delete it at the end - useful for when you want to look at the
-    temp results
+    This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests
+    didn't leave any data in there.
 
-    ::
-        def test_whatever(self):
-            tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
+    3. You can override the first two options by directly overriding the ``before`` and ``after`` args.
 
-    4. create a temp dir of my choice, but if it already exists leave its contents as they are
+    When ``before=True`` is passed the temporary dir will always be cleared at the beginning of the test.
 
-    ::
+    With ``before=False``if the temporary dir already existed and contained some files they will remain there.
 
-        def test_whatever(self):
-            tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=False)
+    When ``after=True`` is passed the temporary dir will always be deleted at the end of the test.
 
-    Note 1: In order to run the equivalent of `rm -r` safely, only subdirs of the project repository checkout are
-    allowed if an explicit `tmp_dir` is used, so that by mistake no `/tmp` or similar important part of the filesystem
-    will get nuked. i.e. please always pass paths that start with `./`
+    When ``after=False`` is passed the temporary dir will always be left intact at the end of the test.
+
+
+    Note 1: In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are
+    allowed if an explicit ``tmp_dir`` is used, so that by mistake no ``/tmp`` or similar important part of the
+    filesystem will get nuked. i.e. please always pass paths that start with ``./``
 
     Note 2: Each test can register multiple temp dirs and they all will get auto-removed, unless requested otherwise.
 
@@ -567,6 +567,7 @@ def test_whatever(self):
     """
 
     def setUp(self):
+        # get_auto_remove_tmp_dir feature:
         self.teardown_tmp_dirs = []
 
         # figure out the resolved paths for repo_root, tests, examples, etc.
@@ -654,21 +655,31 @@ def get_env(self):
         env["PYTHONPATH"] = ":".join(paths)
         return env
 
-    def get_auto_remove_tmp_dir(self, tmp_dir=None, after=True, before=True):
+    def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
         """
         Args:
             tmp_dir (:obj:`string`, `optional`):
-                use this path, if None a unique path will be assigned
-            before (:obj:`bool`, `optional`, defaults to :obj:`True`):
-                if `True` and tmp dir already exists make sure to empty it right away
-            after (:obj:`bool`, `optional`, defaults to :obj:`True`):
-                if `True` delete the tmp dir at the end of the test
+                if not :obj:`None` use this path, otherwise, use a unique tmp path will be created
+            before (:obj:`bool`, `optional`, defaults to :obj:`None`):
+                if :obj:`True` and the tmp dir already exists make sure to empty it right away
+            after (:obj:`bool`, `optional`, defaults to :obj:`None`):
+                if :obj:`True` delete the tmp dir at the end of the test
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp
             dir
         """
         if tmp_dir is not None:
+
+            # defining the most likely desired behavior for when a custom path is provided.
+            # this most likely indicates the debug mode where we want an easily locatable dir that:
+            # 1. gets cleared out before the test (if it already exists)
+            # 2. is left intact after the test
+            if before is None:
+                before = True
+            if after is None:
+                after = False
+
             # using provided path
             path = Path(tmp_dir).resolve()
 
@@ -685,6 +696,15 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, after=True, before=True):
             path.mkdir(parents=True, exist_ok=True)
 
         else:
+            # defining the most likely desired behavior for when a unique tmp path is auto generated
+            # (not a debug mode), here we require a unique tmp dir that:
+            # 1. is empty before the test (it will be empty in this situation anyway)
+            # 2. gets fully removed after the test
+            if before is None:
+                before = True
+            if after is None:
+                after = True
+
             # using unique tmp dir (always empty, regardless of `before`)
             tmp_dir = tempfile.mkdtemp()
 
@@ -695,7 +715,8 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, after=True, before=True):
         return tmp_dir
 
     def tearDown(self):
-        # remove registered temp dirs
+
+        # get_auto_remove_tmp_dir feature: remove registered temp dirs
         for path in self.teardown_tmp_dirs:
             shutil.rmtree(path, ignore_errors=True)
         self.teardown_tmp_dirs = []

From e0c5d20e3bf994d64e50627446eb83ac3226c6e8 Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sat, 7 Nov 2020 20:13:31 -0800
Subject: [PATCH 03/11] update docs

---
 docs/source/testing.rst           | 28 ++++++++++++++--------------
 src/transformers/testing_utils.py | 12 ++++++------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/source/testing.rst b/docs/source/testing.rst
index 0a9d3d525bfa9e..9db21e6e4c8b84 100644
--- a/docs/source/testing.rst
+++ b/docs/source/testing.rst
@@ -720,32 +720,32 @@ Here is an example of its usage:
 
 This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its location.
 
-In this and all the following scenarios the temporary directory will be auto-removed at the end of test, unless
-``after=False`` is passed to the helper function.
-
-* Create a temporary directory of my choice and delete it at the end - useful for debugging when you want to monitor a
-  specific directory:
+* Create a unique temporary dir:
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test")
+        tmp_dir = self.get_auto_remove_tmp_dir()
+
+    ``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
 
-* Create a temporary directory of my choice and do not delete it at the end---useful for when you want to look at the
-  temp results:
+* Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
 
 .. code-block:: python
 
     def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", after=False)
+        tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
 
-* Create a temporary directory of my choice and ensure to delete it right away---useful for when you disabled deletion
-  in the previous test run and want to make sure the that temporary directory is empty before the new test is run:
 
-.. code-block:: python
+    This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't leave any data in there.
+
+* You can override the first two options by directly overriding the ``before`` and ``after`` args, leading to the
+  following behavior:
 
-   def test_whatever(self):
-        tmp_dir = self.get_auto_remove_tmp_dir(tmp_dir="./tmp/run/test", before=True)
+    - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
+    - ``before=False``: if the temporary dir already existed, any existing files will remain there.
+    - ``after=True``: the temporary dir will always be deleted at the end of the test.
+    - ``after=False``: the temporary dir will always be left intact at the end of the test.
 
 .. note::
    In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are allowed if
diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index c0bc3a21433ffd..48babc546b4bfa 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -539,16 +539,16 @@ def test_whatever(self):
     This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests
     didn't leave any data in there.
 
-    3. You can override the first two options by directly overriding the ``before`` and ``after`` args.
+    3. You can override the first two options by directly overriding the ``before`` and ``after`` args, leading to the
+       following behavior:
 
-    When ``before=True`` is passed the temporary dir will always be cleared at the beginning of the test.
+    ``before=True``: the temporary dir will always be cleared at the beginning of the test.
 
-    With ``before=False``if the temporary dir already existed and contained some files they will remain there.
+    ``before=False``: if the temporary dir already existed, any existing files will remain there.
 
-    When ``after=True`` is passed the temporary dir will always be deleted at the end of the test.
-
-    When ``after=False`` is passed the temporary dir will always be left intact at the end of the test.
+    ``after=True``: the temporary dir will always be deleted at the end of the test.
 
+    ``after=False``: the temporary dir will always be left intact at the end of the test.
 
     Note 1: In order to run the equivalent of ``rm -r`` safely, only subdirs of the project repository checkout are
     allowed if an explicit ``tmp_dir`` is used, so that by mistake no ``/tmp`` or similar important part of the

From a4fa5ef694f5e82db26530d16e78b56122225f9d Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sat, 7 Nov 2020 20:25:38 -0800
Subject: [PATCH 04/11] fix doc layout

---
 docs/source/testing.rst | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/docs/source/testing.rst b/docs/source/testing.rst
index 9db21e6e4c8b84..82eefb0bb2acdb 100644
--- a/docs/source/testing.rst
+++ b/docs/source/testing.rst
@@ -727,7 +727,7 @@ This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its l
     def test_whatever(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
 
-    ``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
+``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
 
 * Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
 
@@ -736,8 +736,7 @@ This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its l
     def test_whatever(self):
         tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
 
-
-    This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't leave any data in there.
+This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't leave any data in there.
 
 * You can override the first two options by directly overriding the ``before`` and ``after`` args, leading to the
   following behavior:
@@ -799,7 +798,7 @@ or the ``xfail`` way:
     @pytest.mark.xfail
     def test_feature_x():
 
-Here is how to skip a test based on some internal check inside the test:
+- Here is how to skip a test based on some internal check inside the test:
 
 .. code-block:: python
 
@@ -822,7 +821,7 @@ or the ``xfail`` way:
     def test_feature_x():
         pytest.xfail("expected to fail until bug XYZ is fixed")
 
-Here is how to skip all tests in a module if some import is missing:
+- Here is how to skip all tests in a module if some import is missing:
 
 .. code-block:: python
 

From b18eea5e8887c7181d682e2721599e6b741be01f Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sat, 7 Nov 2020 20:28:07 -0800
Subject: [PATCH 05/11] style

---
 docs/source/testing.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/source/testing.rst b/docs/source/testing.rst
index 82eefb0bb2acdb..3b44a6623b2473 100644
--- a/docs/source/testing.rst
+++ b/docs/source/testing.rst
@@ -736,10 +736,11 @@ This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its l
     def test_whatever(self):
         tmp_dir = self.get_auto_remove_tmp_dir("./xxx")
 
-This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't leave any data in there.
+This is useful for debug when you want to monitor a specific directory and want to make sure the previous tests didn't
+leave any data in there.
 
-* You can override the first two options by directly overriding the ``before`` and ``after`` args, leading to the
-  following behavior:
+* You can override the default behavior by directly overriding the ``before`` and ``after`` args, leading to one of the
+  following behaviors:
 
     - ``before=True``: the temporary dir will always be cleared at the beginning of the test.
     - ``before=False``: if the temporary dir already existed, any existing files will remain there.

From 2cfc9beaaad62cfc2141ae4dfe0b4103b4f72b7e Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas00@users.noreply.github.com>
Date: Sun, 8 Nov 2020 07:55:28 -0800
Subject: [PATCH 06/11] Update src/transformers/testing_utils.py

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 src/transformers/testing_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index 48babc546b4bfa..1f19734fee98fb 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -660,7 +660,7 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
         Args:
             tmp_dir (:obj:`string`, `optional`):
                 if not :obj:`None` use this path, otherwise, use a unique tmp path will be created
-            before (:obj:`bool`, `optional`, defaults to :obj:`None`):
+            before (:obj:`bool`, `optional`):
                 if :obj:`True` and the tmp dir already exists make sure to empty it right away
             after (:obj:`bool`, `optional`, defaults to :obj:`None`):
                 if :obj:`True` delete the tmp dir at the end of the test

From df2f3aeadef9990edea64888485642e8c54a14c9 Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sun, 8 Nov 2020 08:05:13 -0800
Subject: [PATCH 07/11] better 3-state doc

---
 src/transformers/testing_utils.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index 1f19734fee98fb..dab05514f76a4b 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -659,11 +659,20 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
         """
         Args:
             tmp_dir (:obj:`string`, `optional`):
-                if not :obj:`None` use this path, otherwise, use a unique tmp path will be created
+                if :obj:`None`:
+                   - a unique tmp path will be created
+                   - sets ``before=True`` if ``before`` is :obj:`None`
+                   - sets ``after=True`` if ``after`` is :obj:`None`
+                else:
+                   - a unique tmp path will be chosen and created
+                   - sets ``before=True`` if ``before`` is :obj:`None`
+                   - sets ``after=False`` if ``after`` is :obj:`None`
             before (:obj:`bool`, `optional`):
-                if :obj:`True` and the tmp dir already exists make sure to empty it right away
-            after (:obj:`bool`, `optional`, defaults to :obj:`None`):
-                if :obj:`True` delete the tmp dir at the end of the test
+                if :obj:`True` and the tmp dir already exists, make sure to empty it right away
+                if :obj:`False` and the tmp dir already exists, any existing files will remain there.
+            after (:obj:`bool`, `optional`):
+                if :obj:`True`, delete the tmp dir at the end of the test
+                if :obj:`False`, leave the tmp dir and its contents intact at the end of the test
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp

From d58257264483b5a3471604cb78ee3f49602de9c5 Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Sun, 8 Nov 2020 08:14:28 -0800
Subject: [PATCH 08/11] style

---
 src/transformers/testing_utils.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index dab05514f76a4b..c7872ebbf8b286 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -660,19 +660,21 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
         Args:
             tmp_dir (:obj:`string`, `optional`):
                 if :obj:`None`:
+
                    - a unique tmp path will be created
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=True`` if ``after`` is :obj:`None`
                 else:
+
                    - a unique tmp path will be chosen and created
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=False`` if ``after`` is :obj:`None`
             before (:obj:`bool`, `optional`):
-                if :obj:`True` and the tmp dir already exists, make sure to empty it right away
-                if :obj:`False` and the tmp dir already exists, any existing files will remain there.
+                if :obj:`True` and the tmp dir already exists, make sure to empty it right away if :obj:`False` and the
+                tmp dir already exists, any existing files will remain there.
             after (:obj:`bool`, `optional`):
-                if :obj:`True`, delete the tmp dir at the end of the test
-                if :obj:`False`, leave the tmp dir and its contents intact at the end of the test
+                if :obj:`True`, delete the tmp dir at the end of the test if :obj:`False`, leave the tmp dir and its
+                contents intact at the end of the test
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp

From 893d6df6e4a603fc9e201d0b8a2f7e0f5a01bcea Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas00@users.noreply.github.com>
Date: Mon, 9 Nov 2020 09:00:14 -0800
Subject: [PATCH 09/11] Apply suggestions from code review

Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
---
 src/transformers/testing_utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index c7872ebbf8b286..d8b1ec1dd9b9b5 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -670,11 +670,11 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=False`` if ``after`` is :obj:`None`
             before (:obj:`bool`, `optional`):
-                if :obj:`True` and the tmp dir already exists, make sure to empty it right away if :obj:`False` and the
-                tmp dir already exists, any existing files will remain there.
+                If :obj:`True` and the :obj:`tmp_dir` already exists, make sure to empty it right away if :obj:`False` and the
+                :obj:`tmp_dir` already exists, any existing files will remain there.
             after (:obj:`bool`, `optional`):
-                if :obj:`True`, delete the tmp dir at the end of the test if :obj:`False`, leave the tmp dir and its
-                contents intact at the end of the test
+                If :obj:`True`, delete the :obj:`tmp_dir` at the end of the test if :obj:`False`, leave the :obj:`tmp_dir` and its
+                contents intact at the end of the test.
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp

From ede20cd309b5848df3d59db4685d4227ba54eb7f Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Mon, 9 Nov 2020 09:17:15 -0800
Subject: [PATCH 10/11] s/tmp/temporary/ + style

---
 docs/source/testing.rst           |  9 +++++----
 src/transformers/testing_utils.py | 20 +++++++++++---------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/docs/source/testing.rst b/docs/source/testing.rst
index 3b44a6623b2473..49f9765d1bcf12 100644
--- a/docs/source/testing.rst
+++ b/docs/source/testing.rst
@@ -700,11 +700,11 @@ Temporary files and directories
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Using unique temporary files and directories are essential for parallel test running, so that the tests won't overwrite
-each other's data. Also we want to get the temp files and directories removed at the end of each test that created
+each other's data. Also we want to get the temporary files and directories removed at the end of each test that created
 them. Therefore, using packages like ``tempfile``, which address these needs is essential.
 
-However, when debugging tests, you need to be able to see what goes into the temp file or directory and you want to
-know it's exact path and not having it randomized on every test re-run.
+However, when debugging tests, you need to be able to see what goes into the temporary file or directory and you want
+to know it's exact path and not having it randomized on every test re-run.
 
 A helper class :obj:`transformers.test_utils.TestCasePlus` is best used for such purposes. It's a sub-class of
 :obj:`unittest.TestCase`, so we can easily inherit from it in the test modules.
@@ -727,7 +727,8 @@ This code creates a unique temporary directory, and sets :obj:`tmp_dir` to its l
     def test_whatever(self):
         tmp_dir = self.get_auto_remove_tmp_dir()
 
-``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
+``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
+test.
 
 * Create a temporary dir of my choice, ensure it's empty before the test starts and don't empty it after the test.
 
diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index d8b1ec1dd9b9b5..f31a07c95f0d3a 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -516,7 +516,7 @@ class solves this problem by sorting out all the basic paths and provides easy a
        - ``repo_root_dir_str``
        - ``src_dir_str``
 
-    Feature 2: Flexible auto-removable temp dirs which are guaranteed to get removed at the end of test.
+    Feature 2: Flexible auto-removable temporary dirs which are guaranteed to get removed at the end of test.
 
     1. Create a unique temporary dir:
 
@@ -525,7 +525,8 @@ class solves this problem by sorting out all the basic paths and provides easy a
         def test_whatever(self):
             tmp_dir = self.get_auto_remove_tmp_dir()
 
-    ``tmp_dir`` will contain the path to the created temp dir. It will be automatically removed at the end of the test.
+    ``tmp_dir`` will contain the path to the created temporary dir. It will be automatically removed at the end of the
+    test.
 
 
     2. Create a temporary dir of my choice, ensure it's empty before the test starts and don't
@@ -554,7 +555,8 @@ def test_whatever(self):
     allowed if an explicit ``tmp_dir`` is used, so that by mistake no ``/tmp`` or similar important part of the
     filesystem will get nuked. i.e. please always pass paths that start with ``./``
 
-    Note 2: Each test can register multiple temp dirs and they all will get auto-removed, unless requested otherwise.
+    Note 2: Each test can register multiple temporary dirs and they all will get auto-removed, unless requested
+    otherwise.
 
     Feature 3: Get a copy of the ``os.environ`` object that sets up ``PYTHONPATH`` specific to the current test suite.
     This is useful for invoking external programs from the test suite - e.g. distributed training.
@@ -661,20 +663,20 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
             tmp_dir (:obj:`string`, `optional`):
                 if :obj:`None`:
 
-                   - a unique tmp path will be created
+                   - a unique temporary path will be created
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=True`` if ``after`` is :obj:`None`
                 else:
 
-                   - a unique tmp path will be chosen and created
+                   - a unique temporary path will be chosen and created
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=False`` if ``after`` is :obj:`None`
             before (:obj:`bool`, `optional`):
-                If :obj:`True` and the :obj:`tmp_dir` already exists, make sure to empty it right away if :obj:`False` and the
-                :obj:`tmp_dir` already exists, any existing files will remain there.
+                If :obj:`True` and the :obj:`tmp_dir` already exists, make sure to empty it right away if :obj:`False`
+                and the :obj:`tmp_dir` already exists, any existing files will remain there.
             after (:obj:`bool`, `optional`):
-                If :obj:`True`, delete the :obj:`tmp_dir` at the end of the test if :obj:`False`, leave the :obj:`tmp_dir` and its
-                contents intact at the end of the test.
+                If :obj:`True`, delete the :obj:`tmp_dir` at the end of the test if :obj:`False`, leave the
+                :obj:`tmp_dir` and its contents intact at the end of the test.
 
         Returns:
             tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp

From d13bbbb63ba43e465ef048c231b1e456e2583c6f Mon Sep 17 00:00:00 2001
From: Stas Bekman <stas@stason.org>
Date: Tue, 10 Nov 2020 08:40:56 -0800
Subject: [PATCH 11/11] correct the statement

---
 src/transformers/testing_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index f31a07c95f0d3a..657ac8a3ce504f 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -668,7 +668,7 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
                    - sets ``after=True`` if ``after`` is :obj:`None`
                 else:
 
-                   - a unique temporary path will be chosen and created
+                   - :obj:`tmp_dir` will be created
                    - sets ``before=True`` if ``before`` is :obj:`None`
                    - sets ``after=False`` if ``after`` is :obj:`None`
             before (:obj:`bool`, `optional`):
@@ -679,7 +679,7 @@ def get_auto_remove_tmp_dir(self, tmp_dir=None, before=None, after=None):
                 :obj:`tmp_dir` and its contents intact at the end of the test.
 
         Returns:
-            tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-created tmp
+            tmp_dir(:obj:`string`): either the same value as passed via `tmp_dir` or the path to the auto-selected tmp
             dir
         """
         if tmp_dir is not None: