diff --git a/CHANGELOG.D/2492.feature b/CHANGELOG.D/2492.feature new file mode 100644 index 000000000..e18b82273 --- /dev/null +++ b/CHANGELOG.D/2492.feature @@ -0,0 +1 @@ +Add support for Open-Sourced user-less services deployment. diff --git a/neuro-sdk/src/neuro_sdk/_config.py b/neuro-sdk/src/neuro_sdk/_config.py index ff4fa6cd8..fb529c985 100644 --- a/neuro-sdk/src/neuro_sdk/_config.py +++ b/neuro-sdk/src/neuro_sdk/_config.py @@ -57,7 +57,7 @@ class _ConfigData: auth_config: _AuthConfig auth_token: _AuthToken url: URL - admin_url: URL + admin_url: Optional[URL] version: str cluster_name: str org_name: Optional[str] @@ -231,7 +231,7 @@ def api_url(self) -> URL: return self._config_data.url @property - def admin_url(self) -> URL: + def admin_url(self) -> Optional[URL]: return self._config_data.admin_url @property @@ -410,7 +410,10 @@ def _load(path: Path) -> _ConfigData: payload = cur.fetchone() api_url = URL(payload["url"]) - admin_url = URL(payload["admin_url"]) + if not payload["admin_url"]: + admin_url = None + else: + admin_url = URL(payload["admin_url"]) auth_config = _deserialize_auth_config(payload) clusters = _deserialize_clusters(payload) version = payload["version"] @@ -546,7 +549,10 @@ def _save(config: _ConfigData, path: Path, suppress_errors: bool = True) -> None # Factory._save() try: url = str(config.url) - admin_url = str(config.admin_url) + if not config.admin_url: + admin_url = None + else: + admin_url = str(config.admin_url) auth_config = _serialize_auth_config(config.auth_config) clusters = _serialize_clusters(config.clusters) version = config.version diff --git a/neuro-sdk/src/neuro_sdk/_config_factory.py b/neuro-sdk/src/neuro_sdk/_config_factory.py index 89be0f379..97dbc031c 100644 --- a/neuro-sdk/src/neuro_sdk/_config_factory.py +++ b/neuro-sdk/src/neuro_sdk/_config_factory.py @@ -20,6 +20,7 @@ AuthTokenClient, HeadlessNegotiator, _AuthToken, + create_standalone_token, logout_from_browser, ) from ._plugins import PluginManager @@ -160,15 +161,20 @@ async def login( raise ConfigError(f"Config at {self._path} already exists. Please logout") async with _make_session(timeout, self._trace_configs) as session: config_unauthorized = await get_server_config(session, url) - negotiator = AuthNegotiator( - session, config_unauthorized.auth_config, show_browser_cb - ) - auth_token = await negotiator.get_token() - config_authorized = await get_server_config( - session, url, token=auth_token.token - ) - config = self._gen_config(config_authorized, auth_token, url) + if config_unauthorized.clusters: + config_authorized = config_unauthorized + auth_token = create_standalone_token() + else: + negotiator = AuthNegotiator( + session, config_unauthorized.auth_config, show_browser_cb + ) + auth_token = await negotiator.get_token() + + config_authorized = await get_server_config( + session, url, token=auth_token.token + ) + config = self._gen_config(config_authorized, auth_token, url) self._save(config) async def login_headless( @@ -242,8 +248,6 @@ def _gen_config( ) -> _ConfigData: from . import __version__ - assert server_config.admin_url, "Authorized config should include admin_url" - cluster_name = next(iter(server_config.clusters)) org_name = next(iter(server_config.clusters[cluster_name].orgs)) config = _ConfigData( diff --git a/neuro-sdk/src/neuro_sdk/_core.py b/neuro-sdk/src/neuro_sdk/_core.py index f20b74449..add7d97d8 100644 --- a/neuro-sdk/src/neuro_sdk/_core.py +++ b/neuro-sdk/src/neuro_sdk/_core.py @@ -120,7 +120,9 @@ async def request( real_headers: CIMultiDict[str] = CIMultiDict(headers) else: real_headers = CIMultiDict() - real_headers["Authorization"] = auth + if len(auth.split()) > 1: + # auth contains scheme and parameter + real_headers["Authorization"] = auth if "Content-Type" not in real_headers: if json is not None: real_headers["Content-Type"] = "application/json" diff --git a/neuro-sdk/src/neuro_sdk/_login.py b/neuro-sdk/src/neuro_sdk/_login.py index e9c9bd7ac..a5bba3091 100644 --- a/neuro-sdk/src/neuro_sdk/_login.py +++ b/neuro-sdk/src/neuro_sdk/_login.py @@ -47,6 +47,9 @@ def urlsafe_unpadded_b64encode(payload: bytes) -> str: return base64.urlsafe_b64encode(payload).decode().rstrip("=") +# Used only for standalone platform deploymens +JWT_STANDALONE_SECRET = "neuro" + JWT_IDENTITY_CLAIM = "https://platform.neuromation.io/user" JWT_IDENTITY_CLAIM_OPTIONS = ("identity", JWT_IDENTITY_CLAIM) @@ -486,3 +489,8 @@ async def logout_from_browser( ) -> None: logout_url = config.logout_url.update_query(client_id=config.client_id) await show_browser_cb(logout_url) + + +def create_standalone_token() -> _AuthToken: + token = jwt.encode({JWT_IDENTITY_CLAIM: "user"}, JWT_STANDALONE_SECRET) + return _AuthToken.create_non_expiring(token) diff --git a/neuro-sdk/src/neuro_sdk/_s3_bucket_provider.py b/neuro-sdk/src/neuro_sdk/_s3_bucket_provider.py index fb17916a2..818b76c2f 100644 --- a/neuro-sdk/src/neuro_sdk/_s3_bucket_provider.py +++ b/neuro-sdk/src/neuro_sdk/_s3_bucket_provider.py @@ -1,3 +1,4 @@ +import ssl from contextlib import asynccontextmanager from datetime import datetime from email.utils import parsedate_to_datetime @@ -5,6 +6,7 @@ import aiobotocore.session import botocore.exceptions +import certifi from aiobotocore.client import AioBaseClient from aiobotocore.credentials import AioCredentials, AioRefreshableCredentials @@ -76,11 +78,30 @@ async def _refresher() -> Mapping[str, str]: secret_key=initial_credentials.credentials["secret_access_key"], ) + # Use system root CA certificates + # Currently you cannot override ssl context. + # + # Aiobotocore always sets it's own context if verify is None. + # + # If verify is not None aiohttp raises error `verify_ssl, ssl_context, + # fingerprint and ssl parameters are mutually exclusive`. + # + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS) + ssl_context.load_verify_locations(capath=certifi.where()) + # config = AioConfig(connector_args={"ssl_context": ssl_context}) + async with session.create_client( "s3", endpoint_url=initial_credentials.credentials.get("endpoint_url"), region_name=initial_credentials.credentials.get("region_name"), ) as client: + # Dirty hack to override ssl context in aiobotocore + # The check exists to make sure that the patch is compatible + # with used aiobotocore version + assert isinstance( + client._endpoint.http_session._session._connector._ssl, ssl.SSLContext + ) + client._endpoint.http_session._session._connector._ssl = ssl_context yield cls(client, bucket, initial_credentials.credentials["bucket_name"]) @asyncgeneratorcontextmanager diff --git a/neuro-sdk/src/neuro_sdk/_server_cfg.py b/neuro-sdk/src/neuro_sdk/_server_cfg.py index 645184f90..9a708e131 100644 --- a/neuro-sdk/src/neuro_sdk/_server_cfg.py +++ b/neuro-sdk/src/neuro_sdk/_server_cfg.py @@ -123,7 +123,7 @@ async def get_server_config( admin_url: Optional[URL] = None if "admin_url" in payload: admin_url = URL(payload["admin_url"]) - if headers and (not clusters or not admin_url): + if headers and not clusters: raise AuthError("Cannot authorize user") return _ServerConfig( admin_url=admin_url, diff --git a/neuro-sdk/tests/conftest.py b/neuro-sdk/tests/conftest.py index 29806b054..4e60a485d 100644 --- a/neuro-sdk/tests/conftest.py +++ b/neuro-sdk/tests/conftest.py @@ -2,7 +2,7 @@ from dataclasses import replace from decimal import Decimal from pathlib import Path -from typing import Callable, Dict, Optional +from typing import Any, Callable, Dict, Optional import aiohttp import aiohttp.pytest_plugin @@ -102,8 +102,8 @@ def go( trace_id: str = "bd7a977555f6b982", clusters: Optional[Dict[str, Cluster]] = None, token_url: Optional[URL] = None, - admin_url: Optional[URL] = None, plugin_manager: Optional[PluginManager] = None, + **kwargs: Any, ) -> Client: url = URL(url_str) if clusters is None: @@ -167,8 +167,10 @@ def go( real_auth_config = replace(auth_config, token_url=token_url) else: real_auth_config = auth_config - if admin_url is None: + if "admin_url" not in kwargs: admin_url = URL(url) / ".." / ".." / "apis" / "admin" / "v1" + else: + admin_url = kwargs["admin_url"] if plugin_manager is None: plugin_manager = PluginManager() cluster_name = next(iter(clusters)) diff --git a/neuro-sdk/tests/test_config.py b/neuro-sdk/tests/test_config.py index cd4e75176..4f0c90ef5 100644 --- a/neuro-sdk/tests/test_config.py +++ b/neuro-sdk/tests/test_config.py @@ -466,6 +466,66 @@ async def handler(request: web.Request) -> web.Response: } +async def test_fetch_without_admin_url( + aiohttp_server: _TestServerFactory, make_client: _MakeClient +) -> None: + registry_url = "https://registry2-dev.neu.ro" + storage_url = "https://storage2-dev.neu.ro" + users_url = "https://users2-dev.neu.ro" + monitoring_url = "https://jobs2-dev.neu.ro" + secrets_url = "https://secrets2-dev.neu.ro" + disks_url = "https://disks2-dev.neu.ro" + buckets_url = "https://buckets2-dev.neu.ro" + auth_url = "https://dev-neuro.auth0.com/authorize" + token_url = "https://dev-neuro.auth0.com/oauth/token" + logout_url = "https://dev-neuro.auth0.com/v2/logout" + client_id = "this_is_client_id" + audience = "https://platform.dev.neu.ro." + headless_callback_url = "https://dev.neu.ro/oauth/show-code" + success_redirect_url = "https://platform.neu.ro" + JSON = { + "auth_url": auth_url, + "token_url": token_url, + "logout_url": logout_url, + "client_id": client_id, + "audience": audience, + "headless_callback_url": headless_callback_url, + "success_redirect_url": success_redirect_url, + "clusters": [ + { + "name": "default", + "orgs": [None, "some-org"], + "registry_url": registry_url, + "storage_url": storage_url, + "users_url": users_url, + "monitoring_url": monitoring_url, + "secrets_url": secrets_url, + "disks_url": disks_url, + "buckets_url": buckets_url, + "resource_presets": [ + { + "name": "cpu-small", + "credits_per_hour": "10", + "cpu": 2, + "memory_mb": 2 * 1024, + } + ], + } + ], + } + + async def handler(request: web.Request) -> web.Response: + return web.json_response(JSON) + + app = web.Application() + app.add_routes([web.get("/config", handler)]) + srv = await aiohttp_server(app) + + async with make_client(srv.make_url("/"), admin_url=None) as client: + await client.config.fetch() + assert client.config.admin_url is None + + async def test_fetch_dropped_selected_cluster( aiohttp_server: _TestServerFactory, make_client: _MakeClient ) -> None: diff --git a/neuro-sdk/tests/test_config_factory.py b/neuro-sdk/tests/test_config_factory.py index bb74b65b3..4b60dc0ee 100644 --- a/neuro-sdk/tests/test_config_factory.py +++ b/neuro-sdk/tests/test_config_factory.py @@ -12,6 +12,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestServer as _TestServer +from jose import jws from yarl import URL from neuro_sdk import ( @@ -23,6 +24,7 @@ __version__, ) from neuro_sdk._config import _AuthConfig, _AuthToken, _ConfigData +from neuro_sdk._login import JWT_STANDALONE_SECRET from tests import _TestServerFactory @@ -76,7 +78,9 @@ async def mock_for_login_factory( token: str, aiohttp_unused_port: Callable[[], int], ) -> Callable[[MockForLoginControl], Awaitable[_TestServer]]: - async def _factory(control: MockForLoginControl) -> _TestServer: + async def _factory( + control: MockForLoginControl, auth_enabled: bool = True + ) -> _TestServer: callback_urls = [ f"http://127.0.0.1:{aiohttp_unused_port()}", f"http://127.0.0.1:{aiohttp_unused_port()}", @@ -96,7 +100,7 @@ async def config_handler(request: web.Request) -> web.Response: "success_redirect_url": "http://example.com", } - if ( + if not auth_enabled or ( "Authorization" in request.headers and "incorrect" not in request.headers["Authorization"] ): @@ -278,6 +282,27 @@ async def test_normal_login( nmrc_path = tmp_home / ".neuro" assert Path(nmrc_path).exists(), "Config file not written after login " + async def test_login_to_server_without_auth( + self, + tmp_home: Path, + mock_for_login_factory: Callable[..., Awaitable[_TestServer]], + ) -> None: + mock_for_login = await mock_for_login_factory( + MockForLoginControl(), auth_enabled=False + ) + await Factory().login(self.show_dummy_browser, url=mock_for_login.make_url("/")) + nmrc_path = tmp_home / ".neuro" + assert Path(nmrc_path).exists(), "Config file not written after login " + + client = await Factory(Path(nmrc_path)).get() + await client.close() + token = await client.config.token() + + assert client.config.username == "user" + jws.verify( + token, JWT_STANDALONE_SECRET, algorithms="HS256" + ) # verify it is standalone token + class TestLoginWithToken: async def test_login_with_token_already_logged(self, config_dir: Path) -> None: