Merge remote-tracking branch 'origin/master' into sort-commands

CHANGELOG.md

@@ -5,6 +5,15 @@
 
 [comment]: # (towncrier release notes start)
 
+Neuro SDK/CLI 21.12.1 (2021-12-22)
+==================================
+
+Bugfixes
+--------
+
+- Fix importing of asynccontextmanager from wrong place
+
+
 Neuro SDK/CLI 21.12.0 (2021-12-21)
 ==================================
 

Makefile

@@ -32,7 +32,8 @@ update-deps: ### Update dependencies
 .PHONY: .e2e
 .e2e:
 	COLUMNS=160 LINES=75 pytest \
-	    -n ${PYTEST_XDIST_NUM_THREADS} \
+		-n ${PYTEST_XDIST_NUM_THREADS} \
+		--dist loadgroup \
 		-m "e2e" \
 		--cov=neuro-cli --cov=neuro-sdk \
 		--cov-report term-missing:skip-covered \

VERSION.txt

@@ -1,4 +1,4 @@
 # Global version number,
 # Update the file and run 'make fmt' to apply it everywhere
 
-21.12.0
+21.12.1

neuro-cli/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = neuro-cli
-version = 21.12.0
+version = 21.12.1
 description = Neuro Platform client
 url = https://github.com/neuro-inc/platform-client-python
 long_description = file: README.md
@@ -35,7 +35,7 @@ include_package_data = True
 install_requires =
     python-jose>=3.0.0
     python-dateutil>=2.7.0
-    neuro-sdk>=21.12.0
+    neuro-sdk>=21.12.1
     click>=8.0
     humanize>=3.3
     typing_extensions>=3.7.4
@@ -87,6 +87,7 @@ markers =
   e2e
   e2e_job
   require_admin
+  xdist_group
 filterwarnings=error
   ignore:.*PROTOCOL_TLS is deprecated:DeprecationWarning:neuro_sdk
   ignore:.*PROTOCOL_TLS is deprecated:DeprecationWarning:tests

neuro-cli/src/neuro_cli/__init__.py

@@ -1 +1 @@
-__version__ = "21.12.0"
+__version__ = "21.12.1"

neuro-cli/tests/e2e/conftest.py

@@ -43,6 +43,7 @@
 
 from neuro_sdk import (
     AuthorizationError,
+    Bucket,
     Client,
     Config,
     Container,
@@ -639,10 +640,10 @@ async def akill_job(self, id_or_name: str, *, wait: bool = True) -> None:
 
     kill_job = run_async(akill_job)
 
-    async def acreate_bucket(self, name: str, *, wait: bool = False) -> None:
+    async def acreate_bucket(self, name: str, *, wait: bool = False) -> Bucket:
         __tracebackhide__ = True
         async with api_get(timeout=CLIENT_TIMEOUT, path=self._nmrc_path) as client:
-            await client.buckets.create(name)
+            bucket = await client.buckets.create(name)
             if wait:
                 t0 = time()
                 delay = 1
@@ -652,39 +653,43 @@ async def acreate_bucket(self, name: str, *, wait: bool = False) -> None:
                         async with client.buckets.list_blobs(url, limit=1) as it:
                             async for _ in it:
                                 pass
-                        return
+                        return bucket
                     except Exception as e:
                         print(e)
                         delay = min(delay * 2, 10)
                         await asyncio.sleep(delay)
                 raise RuntimeError(
                     f"Bucket {name} doesn't available after the creation"
                 )
+            return bucket
 
     create_bucket = run_async(acreate_bucket)
 
-    async def adelete_bucket(self, bucket_name_or_id: str) -> None:
+    async def adelete_bucket(self, bucket: Bucket) -> None:
         __tracebackhide__ = True
         async with api_get(timeout=CLIENT_TIMEOUT, path=self._nmrc_path) as client:
-            await client.buckets.rm(bucket_name_or_id)
+            await client.buckets.rm(bucket.id, bucket_owner=bucket.owner)
 
     delete_bucket = run_async(adelete_bucket)
 
-    async def acleanup_bucket(self, bucket_name_or_id: str) -> None:
+    async def acleanup_bucket(self, bucket: Bucket) -> None:
         __tracebackhide__ = True
         # Each test needs a clean bucket state and we can't delete bucket until it's
         # cleaned
         async with api_get(timeout=CLIENT_TIMEOUT, path=self._nmrc_path) as client:
             async with client.buckets.list_blobs(
-                URL(f"blob:{bucket_name_or_id}"), recursive=True
+                bucket.uri, recursive=True
             ) as blobs_it:
-                # XXX: We do assume we will not have tests that run 10000 of objects.
+                # XXX: We do assume we will not have tests that run
+                # 10000 of objects.
                 # If we do, please add a semaphore here.
                 tasks = []
                 async for blob in blobs_it:
-                    log.info("Removing %s %s", bucket_name_or_id, blob.key)
+                    log.info("Removing %s", blob.uri)
                     tasks.append(
-                        client.buckets.delete_blob(bucket_name_or_id, key=blob.key)
+                        client.buckets.delete_blob(
+                            bucket.id, key=blob.key, bucket_owner=bucket.owner
+                        )
                     )
             await asyncio.gather(*tasks)
 
@@ -702,11 +707,14 @@ async def drop_stale_buckets(self, bucket_prefix: str) -> None:
                         and datetime.now(timezone.utc) - bucket.created_at
                         > timedelta(hours=4)
                     ):
-                        with suppress(ResourceNotFound):
+                        try:
+                            await self.acleanup_bucket(bucket)
+                            await self.adelete_bucket(bucket)
+                        except Exception as e:
                             # bucket can be deleted by another parallel test run,
-                            # ignore ResourceNotFound errors
-                            await self.acleanup_bucket(bucket.id)
-                            await self.adelete_bucket(bucket.id)
+                            # this can lead for botocore/sdk exceptions, so
+                            # we have to ignore everything here
+                            print(e)
 
     @run_async
     async def upload_blob(
@@ -854,7 +862,7 @@ def nested_data(static_path: Path) -> Tuple[str, str, str]:
 @pytest.fixture(scope="session")
 def _tmp_bucket_create(
     tmp_path_factory: Any, request: Any
-) -> Iterator[Tuple[str, Helper]]:
+) -> Iterator[Tuple[Bucket, Helper]]:
     tmp_path = tmp_path_factory.mktemp("tmp_bucket" + str(uuid()))
     tmpbucketname = f"neuro-e2e-{secrets.token_hex(10)}"
     nmrc_path = _get_nmrc_path(tmp_path_factory.mktemp("config"), require_admin=False)
@@ -863,20 +871,20 @@ def _tmp_bucket_create(
 
     try:
         helper.drop_stale_buckets("neuro-e2e-")
-        helper.create_bucket(tmpbucketname, wait=True)
+        bucket = helper.create_bucket(tmpbucketname, wait=True)
     except AuthorizationError:
         pytest.skip("No permission to create bucket for user E2E_TOKEN")
-    yield tmpbucketname, helper
-    helper.delete_bucket(tmpbucketname)
+    yield bucket, helper
+    helper.delete_bucket(bucket)
     helper.close()
 
 
 @pytest.fixture
-def tmp_bucket(_tmp_bucket_create: Tuple[str, Helper]) -> Iterator[str]:
-    tmpbucketname, helper = _tmp_bucket_create
-    yield tmpbucketname
+def tmp_bucket(_tmp_bucket_create: Tuple[Bucket, Helper]) -> Iterator[str]:
+    bucket, helper = _tmp_bucket_create
+    yield bucket.name or bucket.id
     try:
-        helper.cleanup_bucket(tmpbucketname)
+        helper.cleanup_bucket(bucket)
     except aiohttp.ClientOSError as exc:
         if exc.errno == errno.ETIMEDOUT:
             # Try next time

neuro-cli/tests/e2e/test_e2e_admin.py

@@ -1,18 +1,57 @@
 import secrets
 import subprocess
+import tempfile
+from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Iterator, List, Tuple
 
 import pytest
 
 from tests.e2e import Helper
+from tests.e2e.conftest import SysCap, _get_nmrc_path
 
-pytestmark = pytest.mark.require_admin
+pytestmark = [pytest.mark.xdist_group(name="admin_group")]
+
+
+CLUSTER_DATETIME_FORMAT = "%Y%m%d%H%M"
+CLUSTER_DATETIME_SEP = "-date"
+
+
+def make_cluster_name() -> str:
+    time_str = datetime.now().strftime(CLUSTER_DATETIME_FORMAT)
+    return (
+        f"e2e-testing-{secrets.token_hex(4)}{CLUSTER_DATETIME_SEP}{time_str}"
+        f"{CLUSTER_DATETIME_SEP}"
+    )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def drop_old_clusters() -> None:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir_path = Path(tmpdir)
+        nmrc_path = _get_nmrc_path(tmpdir_path, False)
+        subdir = tmpdir_path / "tmp"
+        subdir.mkdir()
+        helper = Helper(nmrc_path=nmrc_path, tmp_path=subdir)
+
+        res: SysCap = helper.run_cli(["admin", "get-clusters"])
+        for out_line in res.out.splitlines():
+            if not out_line.startswith("e2e-testing-"):
+                continue
+            cluster_name = out_line.strip()
+            try:
+                _, time_str, _ = out_line.split(CLUSTER_DATETIME_SEP)
+                cluster_time = datetime.strptime(time_str, CLUSTER_DATETIME_FORMAT)
+                if datetime.now() - cluster_time < timedelta(days=1):
+                    continue
+                helper.run_cli(["admin", "remove-cluster", "--force", cluster_name])
+            except Exception:
+                pass
 
 
 @pytest.fixture
 def tmp_test_cluster(helper: Helper, tmp_path: Path) -> Iterator[str]:
-    cluster_name = "e2e-testing-" + secrets.token_hex(10)
+    cluster_name = make_cluster_name()
     fake_conf = tmp_path / "fake_cluster_config"
     fake_conf.write_text("")
     helper.run_cli(

neuro-cli/tests/e2e/test_e2e_blob_storage.py

@@ -320,6 +320,7 @@ def test_e2e_blob_storage_rm_dir(
 @pytest.mark.skipif(
     sys.platform == "win32", reason="Autocompletion is not supported on Windows"
 )
+@pytest.mark.xfail(reason="autocomplete is not working")
 @pytest.mark.e2e
 def test_blob_autocomplete(helper: Helper, tmp_path: Path, tmp_bucket: str) -> None:
     folder = tmp_path / "folder"

neuro-cli/tests/e2e/test_e2e_storage.py

@@ -694,6 +694,7 @@ def test_tree(helper: Helper, data: _Data, tmp_path: Path) -> None:
 @pytest.mark.skipif(
     sys.platform == "win32", reason="Autocompletion is not supported on Windows"
 )
+@pytest.mark.xfail(reason="autocomplete is not working")
 @pytest.mark.e2e
 def test_storage_autocomplete_remote(helper: Helper, tmp_path: Path) -> None:
     folder = tmp_path / "folder"
@@ -734,6 +735,7 @@ def test_storage_autocomplete_remote(helper: Helper, tmp_path: Path) -> None:
 @pytest.mark.skipif(
     sys.platform == "win32", reason="Autocompletion is not supported on Windows"
 )
+@pytest.mark.xfail(reason="autocomplete is not working")
 @pytest.mark.e2e
 def test_storage_autocomplete_local(helper: Helper, tmp_path: Path) -> None:
     folder = tmp_path / "folder"

neuro-sdk/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = neuro-sdk
-version = 21.12.0
+version = 21.12.1
 description = Neu.ro SDK
 url = https://github.com/neuro-inc/platform-client-python
 long_description = file: README.md
@@ -83,6 +83,7 @@ markers =
   e2e
   e2e_job
   require_admin
+  xdist_group
 filterwarnings=error
   ignore:.*PROTOCOL_TLS is deprecated:DeprecationWarning:neuro_sdk
   ignore:.*PROTOCOL_TLS is deprecated:DeprecationWarning:tests

neuro-sdk/src/neuro_sdk/__init__.py

@@ -109,7 +109,7 @@
 from ._users import Action, Permission, Quota, Share, Users
 from ._utils import _ContextManager, find_project_root
 
-__version__ = "21.12.0"
+__version__ = "21.12.1"
 
 
 __all__ = (

neuro-sdk/src/neuro_sdk/_admin.py

@@ -1,6 +1,6 @@
 # Admin API is experimental,
 # remove underscore prefix after stabilizing and making public
-
+from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from typing import Any, AsyncIterator, Dict, List, Mapping, Optional
 
@@ -18,7 +18,6 @@
 from neuro_admin_client import OrgUserWithInfo as _OrgUserWithInfo
 from neuro_admin_client import Quota as _Quota
 from neuro_admin_client import UserInfo as _UserInfo
-from prompt_toolkit.eventloop.async_context_manager import asynccontextmanager
 from yarl import URL
 
 from ._config import Config