Merge pull request #98 from ecmwf-projects/multiple_connections

Support for read-only and r&w access to the database

cads_catalogue/config.py

@@ -32,14 +32,16 @@ class SqlalchemySettings:
 
     - ``catalogue_db_user``: postgres username.
     - ``catalogue_db_password``: postgres password.
-    - ``catalogue_db_host``: hostname for the connection.
+    - ``catalogue_db_host``: hostname for r/w connection.
+    - ``catalogue_db_host``: hostname for read-only connection.
     - ``catalogue_db_name``: database name.
     """
 
-    catalogue_db_password: str = dataclasses.field(repr=False)
-    catalogue_db_user: str = "catalogue"
-    catalogue_db_host: str = "catalogue-db"
-    catalogue_db_name: str = "catalogue"
+    catalogue_db_user: str | None = None
+    catalogue_db_password: str | None = None
+    catalogue_db_host: str | None = None
+    catalogue_db_host_read: str | None = None
+    catalogue_db_name: str | None = None
     pool_recycle: int = 60
 
     def __init__(self, **kwargs):
@@ -76,23 +78,23 @@ def __post_init__(self):
                     )
 
         # validations
-        # defined fields without a default must have a value
+        # defined fields must have a not None value
         for field in dataclasses.fields(self):
             value = getattr(self, field.name)
-            if field.default == dataclasses.MISSING and value == dataclasses.MISSING:
+            if value in (dataclasses.MISSING, None):
                 raise ValueError(f"{field.name} must be set")
-        # catalogue_db_password must be set
-        if self.catalogue_db_password is None:
-            raise ValueError("catalogue_db_password must be set")
 
     @property
     def connection_string(self) -> str:
         """Create reader psql connection string."""
-        return (
-            f"postgresql://{self.catalogue_db_user}"
-            f":{self.catalogue_db_password}@{self.catalogue_db_host}"
-            f"/{self.catalogue_db_name}"
-        )
+        url = f"postgresql://{self.catalogue_db_user}:{self.catalogue_db_password}@{self.catalogue_db_host}/{self.catalogue_db_name}"
+        return url
+
+    @property
+    def connection_string_read(self) -> str:
+        """Create reader psql connection string in read-only mode."""
+        url = f"postgresql://{self.catalogue_db_user}:{self.catalogue_db_password}@{self.catalogue_db_host_read}/{self.catalogue_db_name}"
+        return url
 
 
 @dataclasses.dataclass(kw_only=True)
@@ -146,14 +148,11 @@ def __post_init__(self):
                     )
 
         # validations
-        # defined fields without a default must have a value
+        # defined fields must have a not None value
         for field in dataclasses.fields(self):
             value = getattr(self, field.name)
-            if field.default == dataclasses.MISSING and value == dataclasses.MISSING:
+            if value in (dataclasses.MISSING, None):
                 raise ValueError(f"{field.name} must be set")
-        # storage_password must be set
-        if self.storage_password is None:
-            raise ValueError("storage_password must be set")
 
     @property
     def storage_kws(self) -> dict[str, str | bool | None]:

cads_catalogue/database.py

@@ -279,24 +279,28 @@ class Licence(BaseModel):
     )
 
 
-def ensure_session_obj(session_obj: sa.orm.sessionmaker | None) -> sa.orm.sessionmaker:
-    """If `session_obj` is None, create a new session object.
+def ensure_session_obj(read_only: bool = False) -> sa.orm.sessionmaker:
+    """Create a new session object bound to the catalogue database.
 
     Parameters
     ----------
-    session_obj: sqlalchemy Session object
+    read_only: if True, return the sessionmaker object for read-only sessions (default False).
 
     Returns
     -------
     session_obj:
         a SQLAlchemy Session object
     """
-    if session_obj:
-        return session_obj
     settings = config.ensure_settings(config.dbsettings)
-    session_obj = sa.orm.sessionmaker(
-        sa.create_engine(settings.connection_string, pool_recycle=settings.pool_recycle)
-    )
+    if read_only:
+        engine = sa.create_engine(
+            settings.connection_string_read, pool_recycle=settings.pool_recycle
+        )
+    else:
+        engine = sa.create_engine(
+            settings.connection_string, pool_recycle=settings.pool_recycle
+        )
+    session_obj = sa.orm.sessionmaker(engine)
     return session_obj
 
 

cads_catalogue/faceted_search.py

@@ -27,7 +27,7 @@
 from cads_catalogue import database
 from cads_catalogue.faceted_search import get_datasets_by_keywords, get_faceted_stats
 
-session_obj = database.ensure_session_obj()
+session_obj = database.ensure_session_obj(read_only=True)
 session = session_obj()
 
 # consider all the datasets (but you can start with a filtered set of resources,

tests/test_01_config.py

@@ -8,48 +8,66 @@
 
 def test_sqlalchemysettings(temp_environ: Any) -> None:
     # check settings must have a password set (no default)
+    temp_environ.update(
+        dict(
+            catalogue_db_host="host1",
+            catalogue_db_host_read="host2",
+            catalogue_db_name="dbname",
+            catalogue_db_user="dbuser",
+        )
+    )
     temp_environ.pop("catalogue_db_password", default=None)
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ValueError):
         config.SqlalchemySettings()
-    assert "catalogue_db_password" in str(excinfo.value)
-    config.dbsettings = None
 
     # also an empty password can be set
-    settings = config.SqlalchemySettings(catalogue_db_password="")
+    settings = config.SqlalchemySettings(
+        catalogue_db_password="",
+        catalogue_db_host="host1",
+        catalogue_db_host_read="host2",
+        catalogue_db_name="dbname1",
+        catalogue_db_user="user1",
+    )
     assert settings.catalogue_db_password == ""
     config.dbsettings = None
 
     # also a not empty password can be set
-    temp_environ["catalogue_db_password"] = "a password"
+    temp_environ.update(
+        dict(
+            catalogue_db_password="apassword",
+            catalogue_db_host="host1",
+            catalogue_db_host_read="host2",
+            catalogue_db_name="dbname",
+            catalogue_db_user="dbuser",
+        )
+    )
     settings = config.SqlalchemySettings()
-    assert settings.catalogue_db_password == "a password"
+    assert settings.catalogue_db_password == "apassword"
     config.dbsettings = None
-
-    # take also other values from the environment
-    temp_environ["catalogue_db_password"] = "1"
-    temp_environ["catalogue_db_user"] = "2"
-    temp_environ["catalogue_db_host"] = "3"
-    temp_environ["catalogue_db_name"] = "4"
-    temp_environ["pool_recycle"] = "5"
-    settings = config.SqlalchemySettings()
-    assert settings.catalogue_db_password == "1"
-    assert settings.catalogue_db_user == "2"
-    assert settings.catalogue_db_host == "3"
-    assert settings.catalogue_db_name == "4"
-    assert settings.pool_recycle == 5
+    assert settings.connection_string == "postgresql://dbuser:apassword@host1/dbname"
+    assert (
+        settings.connection_string_read == "postgresql://dbuser:apassword@host2/dbname"
+    )
 
 
 def test_ensure_settings(session_obj: sa.orm.sessionmaker, temp_environ: Any) -> None:
+    temp_environ["catalogue_db_user"] = "auser"
     temp_environ["catalogue_db_password"] = "apassword"
-
+    temp_environ["catalogue_db_host"] = "ahost"
+    temp_environ["catalogue_db_host_read"] = "ahost2"
+    temp_environ["catalogue_db_name"] = "aname"
     # initially global settings is importable, but it is None
     assert config.dbsettings is None
 
     # at first run returns right connection and set global setting
     effective_settings = config.ensure_settings()
     assert (
         effective_settings.connection_string
-        == "postgresql://catalogue:apassword@catalogue-db/catalogue"
+        == "postgresql://auser:apassword@ahost/aname"
+    )
+    assert (
+        effective_settings.connection_string_read
+        == "postgresql://auser:apassword@ahost2/aname"
     )
     assert config.dbsettings == effective_settings
     config.dbsettings = None
@@ -59,18 +77,24 @@ def test_ensure_settings(session_obj: sa.orm.sessionmaker, temp_environ: Any) ->
         "catalogue_db_user": "monica",
         "catalogue_db_password": "secret1",
         "catalogue_db_host": "myhost",
-        "catalogue_db_name": "mycatalogue",
+        "catalogue_db_host_read": "myhost2",
+        "catalogue_db_name": "mybroker",
     }
     my_settings_connection_string = (
         "postgresql://%(catalogue_db_user)s:%(catalogue_db_password)s"
         "@%(catalogue_db_host)s/%(catalogue_db_name)s" % my_settings_dict
     )
-    mysettings = config.SqlalchemySettings(**my_settings_dict)  # type: ignore
+    my_settings_connection_string_ro = (
+        "postgresql://%(catalogue_db_user)s:%(catalogue_db_password)s"
+        "@%(catalogue_db_host_read)s/%(catalogue_db_name)s" % my_settings_dict
+    )
+    mysettings = config.SqlalchemySettings(**my_settings_dict)
     effective_settings = config.ensure_settings(mysettings)
 
     assert config.dbsettings == effective_settings
     assert effective_settings == mysettings
     assert effective_settings.connection_string == my_settings_connection_string
+    assert effective_settings.connection_string_read == my_settings_connection_string_ro
     config.dbsettings = None
 
 

tests/test_02_database.py

@@ -23,18 +23,26 @@ def test_init_database(postgresql: Connection[str]) -> None:
 
     database.init_database(connection_string, force=True)
     assert set(conn.execute(query).scalars()) == expected_tables_complete  # type: ignore
+    conn.close()
 
 
-def test_ensure_session_obj(
-    postgresql: Connection[str], session_obj: sessionmaker, temp_environ: Any
-) -> None:
-    # case of session is already set
-    ret_value = database.ensure_session_obj(session_obj)
-    assert ret_value is session_obj
-    config.dbsettings = None
-
-    # case of session not set
+def test_ensure_session_obj(postgresql: Connection[str], temp_environ: Any) -> None:
+    temp_environ["catalogue_db_host"] = "cataloguehost"
     temp_environ["catalogue_db_password"] = postgresql.info.password
-    ret_value = database.ensure_session_obj(None)
+    temp_environ["catalogue_db_user"] = "user"
+    temp_environ["catalogue_db_host_read"] = "read_only_host"
+    temp_environ["catalogue_db_name"] = "catalogue"
+    ret_value = database.ensure_session_obj()
     assert isinstance(ret_value, sessionmaker)
+    assert (
+        str(ret_value.kw["bind"].url) == "postgresql://user:***@cataloguehost/catalogue"
+    )
+
+    config.dbsettings = None
+    ret_value = database.ensure_session_obj(read_only=True)
+    assert isinstance(ret_value, sessionmaker)
+    assert (
+        str(ret_value.kw["bind"].url)
+        == "postgresql://user:***@read_only_host/catalogue"
+    )
     config.dbsettings = None