chore: warn about det deploy det-version mistmach (#8994)

(cherry picked from commit f52f43b)

harness/determined/deploy/aws/cli.py

@@ -13,7 +13,7 @@
 from determined.cli.errors import CliError
 from determined.common.declarative_argparse import Arg, ArgGroup, BoolOptArg, Cmd
 from determined.deploy.aws import aws, constants
-from determined.deploy.errors import MasterTimeoutExpired
+from determined.deploy.errors import MasterTimeoutExpired, warn_version_mismatch
 
 from .deployment_types import base, govcloud, secure, simple, vpc
 from .preflight import check_quotas, get_default_cf_parameter
@@ -155,6 +155,8 @@ def deploy_aws(command: str, args: argparse.Namespace) -> None:
                 f"deployment-type={args.deployment_type}."
             )
 
+    warn_version_mismatch(args.det_version)
+
     if args.deployment_type == constants.deployment_types.GOVCLOUD:
         if args.region not in ["us-gov-east-1", "us-gov-west-1"]:
             raise ValueError(

harness/determined/deploy/errors.py

@@ -1,6 +1,31 @@
+from typing import Optional
+
+import termcolor
+
+import determined as det
+
+
 class PreflightFailure(Exception):
     pass
 
 
 class MasterTimeoutExpired(Exception):
     pass
+
+
+def warn_version_mismatch(requestd_version: Optional[str]) -> None:
+    """
+    check for and warn about version compatibility between the `det` CLI and
+    the requested det version.
+    """
+    if not requestd_version or requestd_version == det.__version__:
+        return
+    print(
+        termcolor.colored(
+            f"Warning: The specified --det-version ({requestd_version}) does not match the "
+            f"the current `det` CLI version ({det.__version__}), proceed with caution. "
+            "You should probably use a matching version of det CLI instead. "
+            "Please refer to the CLI documentation for more information",
+            "yellow",
+        )
+    )

harness/determined/deploy/gcp/cli.py

@@ -11,7 +11,7 @@
 import determined.deploy
 from determined.cli.errors import CliError
 from determined.common.declarative_argparse import Arg, ArgGroup, Cmd, Group, string_to_bool
-from determined.deploy.errors import MasterTimeoutExpired
+from determined.deploy.errors import MasterTimeoutExpired, warn_version_mismatch
 from determined.deploy.gcp import constants, gcp
 
 
@@ -38,6 +38,11 @@ def deploy_gcp(command: str, args: argparse.Namespace) -> None:
         os.makedirs(args.local_state_path)
     os.chdir(args.local_state_path)
 
+    warn_version_mismatch(args.det_version)
+    if args.det_version is None:
+        # keep the existing default value behavior of the cli.
+        args.det_version = determined.__version__
+
     # Set default tf state gcs bucket as '$PROJECT_NAME-determined-deploy` if local tf
     # state doesn't exist and user has not provided a gcs bucket.
     if (
@@ -404,7 +409,6 @@ def parse(s: str) -> Tuple[str, str]:
                         Arg(
                             "--det-version",
                             type=str,
-                            default=determined.__version__,
                             help=argparse.SUPPRESS,
                         ),
                         Arg(

harness/determined/deploy/gke/cli.py

@@ -11,6 +11,7 @@
 from determined.common import yaml
 from determined.common.declarative_argparse import Arg, ArgGroup, Cmd
 from determined.common.util import safe_load_yaml_with_exceptions
+from determined.deploy.errors import warn_version_mismatch
 from determined.deploy.gke.constants import defaults
 
 
@@ -95,6 +96,7 @@ def validate_accelerator_zone(args: argparse.Namespace, zone: str) -> None:
 
 
 def validate_args(args: argparse.Namespace) -> None:
+    warn_version_mismatch(args.det_version)
     validate_location(args.zone, isZone=True)
     validate_accelerator_type(args.gpu_type)
 

harness/determined/deploy/local/cli.py

@@ -5,6 +5,7 @@
 
 import determined
 from determined.common.declarative_argparse import Arg, BoolOptArg, Cmd, Group
+from determined.deploy.errors import warn_version_mismatch
 from determined.deploy.local import cluster_utils
 
 from .preflight import check_docker_install
@@ -14,6 +15,10 @@ def handle_cluster_up(args: argparse.Namespace) -> None:
     if not args.no_preflight_checks:
         check_docker_install()
 
+    warn_version_mismatch(args.det_version)
+    if args.det_version is None:
+        args.det_version = determined.__version__
+
     cluster_utils.cluster_up(
         num_agents=args.agents,
         port=args.master_port,
@@ -39,6 +44,10 @@ def handle_logs(args: argparse.Namespace) -> None:
 
 
 def handle_master_up(args: argparse.Namespace) -> None:
+    warn_version_mismatch(args.det_version)
+    if args.det_version is None:
+        args.det_version = determined.__version__
+
     cluster_utils.master_up(
         port=args.master_port,
         master_config_path=args.master_config_path,
@@ -61,6 +70,9 @@ def handle_master_down(args: argparse.Namespace) -> None:
 
 
 def handle_agent_up(args: argparse.Namespace) -> None:
+    warn_version_mismatch(args.det_version)
+    if args.det_version is None:
+        args.det_version = determined.__version__
     cluster_utils.agent_up(
         master_host=args.master_host,
         master_port=args.master_port,
@@ -141,7 +153,6 @@ def deploy_local(args: argparse.Namespace) -> None:
                 Arg(
                     "--det-version",
                     type=str,
-                    default=determined.__version__,
                     help="version or commit to use",
                 ),
                 Arg(
@@ -228,7 +239,6 @@ def deploy_local(args: argparse.Namespace) -> None:
                 Arg(
                     "--det-version",
                     type=str,
-                    default=determined.__version__,
                     help="version or commit to use",
                 ),
                 Arg(
@@ -326,7 +336,6 @@ def deploy_local(args: argparse.Namespace) -> None:
                 Arg(
                     "--det-version",
                     type=str,
-                    default=determined.__version__,
                     help="version or commit to use",
                 ),
                 Arg(