Skip to content

Commit

Permalink
Merge pull request #82 from ecmwf-projects/COPDS-1145-layout
Browse files Browse the repository at this point in the history
COPDS-1145: layout using cads-forms-cim-json
  • Loading branch information
alex75 authored Jul 28, 2023
2 parents ad40059 + b7ce886 commit e5658fe
Show file tree
Hide file tree
Showing 12 changed files with 866 additions and 23 deletions.
24 changes: 24 additions & 0 deletions alembic/versions/9d4affddbd2b_support_cim_repo_hash.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
"""support cim repo hash.
Revision ID: 9d4affddbd2b
Revises: e5e9a8a41828
Create Date: 2023-07-26 11:20:16.888415
"""
import sqlalchemy as sa

from alembic import op

# revision identifiers, used by Alembic.
revision = "9d4affddbd2b"
down_revision = "e5e9a8a41828"
branch_labels = None
depends_on = None


def upgrade() -> None:
op.add_column("catalogue_updates", sa.Column("cim_repo_commit", sa.String))


def downgrade() -> None:
op.drop_column("catalogue_updates", "cim_repo_commit")
1 change: 1 addition & 0 deletions cads_catalogue/database.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class CatalogueUpdate(BaseModel):
catalogue_repo_commit = sa.Column(sa.String)
licence_repo_commit = sa.Column(sa.String)
message_repo_commit = sa.Column(sa.String)
cim_repo_commit = sa.Column(sa.String)


class ResourceLicence(BaseModel):
Expand Down
16 changes: 14 additions & 2 deletions cads_catalogue/entry_points.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ def update_catalogue(
resources_folder_path: str = os.path.join(PACKAGE_DIR, "cads-forms-json"),
messages_folder_path: str = os.path.join(PACKAGE_DIR, "cads-messages"),
licences_folder_path: str = os.path.join(PACKAGE_DIR, "cads-licences"),
cim_folder_path: str = os.path.join(PACKAGE_DIR, "cads-forms-cim-json"),
connection_string: Optional[str] = None,
force: bool = False,
delete_orphans: bool = True,
Expand All @@ -176,6 +177,7 @@ def update_catalogue(
resources_folder_path: path to the root folder containing metadata files for resources
messages_folder_path: path to the root folder containing metadata files for system messages
licences_folder_path: path to the root folder containing metadata files for licences
cim_folder_path: str = path to the root folder containing CIM generated Quality Assessment layouts
connection_string: something like 'postgresql://user:password@netloc:port/dbname'
force: if True, run update regardless input folders has no changes from last update (default False)
delete_orphans: if True, delete resources not involved in the update process (default True)
Expand Down Expand Up @@ -208,8 +210,13 @@ def update_catalogue(
resource_hash,
licence_hash,
message_hash,
cimlayout_hash,
) = manager.is_db_to_update(
session, resources_folder_path, licences_folder_path, messages_folder_path
session,
resources_folder_path,
licences_folder_path,
messages_folder_path,
cim_folder_path,
)
if not force and not is_db_to_update:
logger.info(
Expand All @@ -223,7 +230,11 @@ def update_catalogue(
)

involved_resource_uids = manager.update_catalogue_resources(
session, resources_folder_path, storage_settings, force=force
session,
resources_folder_path,
cim_folder_path,
storage_settings,
force=force,
)
messages.update_catalogue_messages(session, messages_folder_path)

Expand All @@ -239,6 +250,7 @@ def update_catalogue(
catalogue_repo_commit=resource_hash,
licence_repo_commit=licence_hash,
message_repo_commit=message_hash,
cim_repo_commit=cimlayout_hash,
)
session.add(new_update_info)
logger.info(
Expand Down
58 changes: 58 additions & 0 deletions cads_catalogue/layout_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,58 @@ def transform_licences_blocks(
return new_data


def transform_cim_blocks(layout_data: dict[str, Any], cim_layout_path: str):
"""Transform layout.json data according to CIM Quality Assessment layout.
Parameters
----------
layout_data: data of the layout.json to transform
cim_layout_path: path to the file containing CIM Quality Assessment
Returns
-------
dict: dictionary of layout_data modified
"""
remove_tab = True
remove_aside = True
qa_tab_blocks = None
qa_aside_blocks = None
if os.path.exists(cim_layout_path):
with open(cim_layout_path) as fp:
cim_layout_data = json.load(fp)
qa_tab = cim_layout_data.get("quality_assurance_tab", {})
qa_tab_blocks = qa_tab.get("blocks")
if qa_tab_blocks is not None:
remove_tab = False
qa_aside = cim_layout_data.get("quality_assurance_aside", {})
qa_aside_blocks = qa_aside.get("blocks")
if qa_aside_blocks is not None:
remove_aside = False
new_data = copy.deepcopy(layout_data)
body = new_data.get("body", {})
body_main = body.get("main", {})
sections = body_main.get("sections", [])
aside = body.get("aside", {})
aside_blocks = aside.get("blocks", [])

for i, section in enumerate(copy.deepcopy(sections)):
if section.get("id") == "quality_assurance_tab":
if remove_tab:
del sections[i]
else:
sections[i]["blocks"] = qa_tab_blocks
break

for i, aside_block in enumerate(copy.deepcopy(aside_blocks)):
if aside_block.get("id") == "quality_assurance_aside":
if remove_aside:
del aside_blocks[i]
else:
aside_blocks[i]["blocks"] = qa_aside_blocks
break
return new_data


def store_layout_by_data(
layout_data: dict[str, Any],
resource: dict[str, Any],
Expand Down Expand Up @@ -305,6 +357,7 @@ def store_layout_by_data(
def transform_layout(
session: sa.orm.session.Session,
resource_folder_path: str | pathlib.Path,
cim_folder_path: str | pathlib.Path,
resource: dict[str, Any],
storage_settings: config.ObjectStorageSettings,
):
Expand All @@ -316,6 +369,7 @@ def transform_layout(
session: opened SQLAlchemy session
resource_folder_path: folder path where to find layout.json
resource: metadata of a loaded resource from files
cim_folder_path: the folder path containing CIM generated Quality Assessment layouts
storage_settings: object with settings to access the object storage
Returns
Expand All @@ -329,6 +383,10 @@ def transform_layout(
with open(layout_file_path) as fp:
layout_data = json.load(fp)
logger.debug(f"input layout_data: {layout_data}")
cim_layout_path = os.path.join(
cim_folder_path, resource["resource_uid"], "quality_assurance.layout.json"
)
layout_data = transform_cim_blocks(layout_data, cim_layout_path)
layout_data = transform_image_blocks(
layout_data, resource_folder_path, resource, storage_settings
)
Expand Down
33 changes: 27 additions & 6 deletions cads_catalogue/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ def is_db_to_update(
resources_folder_path: str | pathlib.Path,
licences_folder_path: str | pathlib.Path,
messages_folder_path: str | pathlib.Path,
) -> Tuple[bool, Optional[str], Optional[str], Optional[str]]:
cim_folder_path: str | pathlib.Path,
) -> Tuple[bool, Optional[str], Optional[str], Optional[str], Optional[str]]:
"""
Compare current and last run's status of repo folders and return if the database is to update.
Expand All @@ -82,6 +83,7 @@ def is_db_to_update(
resources_folder_path: the folder path where to look for metadata files of all the resources
licences_folder_path: the folder path where to look for metadata files of all the licences
messages_folder_path: the folder path where to look for metadata files of all the messages
cim_folder_path: the folder path containing CIM generated Quality Assessment layouts
Returns
-------
Expand All @@ -90,6 +92,7 @@ def is_db_to_update(
resource_hash = None
licence_hash = None
message_hash = None
cim_repo_hash = None
last_update_record = session.scalars(
sa.select(database.CatalogueUpdate)
.order_by(database.CatalogueUpdate.update_time.desc())
Expand Down Expand Up @@ -118,16 +121,23 @@ def is_db_to_update(
% messages_folder_path
)

try:
cim_repo_hash = utils.get_last_commit_hash(cim_folder_path)
except Exception: # noqa
logger.exception(
"no check on commit hash for folder %r, error follows" % cim_folder_path
)

if not last_update_record:
logger.warning("table catalogue_updates is currently empty")
is_to_update = True
return is_to_update, resource_hash, licence_hash, message_hash
return is_to_update, resource_hash, licence_hash, message_hash, cim_repo_hash

# last_update_record exists
last_resource_hash = getattr(last_update_record, "catalogue_repo_commit")
last_licence_hash = getattr(last_update_record, "licence_repo_commit")
last_message_hash = getattr(last_update_record, "message_repo_commit")

last_cim_hash = getattr(last_update_record, "cim_repo_commit")
if not last_resource_hash:
logger.warning("no information of last resource repository commit")
elif last_resource_hash != resource_hash:
Expand All @@ -140,16 +150,21 @@ def is_db_to_update(
logger.warning("no information of last message repository commit")
elif last_message_hash != message_hash:
logger.info("detected update of message repository")
if not last_cim_hash:
logger.warning("no information of last CIM repository commit")
elif last_cim_hash != cim_repo_hash:
logger.info("detected update of CIM repository")

if last_resource_hash and (
last_resource_hash,
last_licence_hash,
last_message_hash,
) == (resource_hash, licence_hash, message_hash):
last_cim_hash,
) == (resource_hash, licence_hash, message_hash, cim_repo_hash):
is_to_update = False
else:
is_to_update = True
return is_to_update, resource_hash, licence_hash, message_hash
return is_to_update, resource_hash, licence_hash, message_hash, cim_repo_hash


def is_resource_to_update(session, resource_folder_path):
Expand Down Expand Up @@ -575,6 +590,7 @@ def find_related_resources(
def update_catalogue_resources(
session: sa.orm.session.Session,
resources_folder_path: str | pathlib.Path,
cim_folder_path: str | pathlib.Path,
storage_settings: config.ObjectStorageSettings,
force: bool = False,
) -> List[str]:
Expand All @@ -586,6 +602,7 @@ def update_catalogue_resources(
session: opened SQLAlchemy session
resources_folder_path: path to the root folder containing metadata files for resources
storage_settings: object with settings to access the object storage
cim_folder_path: the folder path containing CIM generated Quality Assessment layouts
force: if True, no skipping of dataset update based on detected changes of sources is made
Returns
Expand Down Expand Up @@ -614,7 +631,11 @@ def update_catalogue_resources(
resource["sources_hash"] = sources_hash
logger.info("resource %s loaded successful" % resource_uid)
resource = layout_manager.transform_layout(
session, resource_folder_path, resource, storage_settings
session,
resource_folder_path,
cim_folder_path,
resource,
storage_settings,
)
resource = form_manager.transform_form(
session, resource_folder_path, resource, storage_settings
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
{
"quality_assurance_tab": {
"blocks": [ {
"id": "fitness-for-purpose",
"type": "thumb-markdown",
"content": "# Title for insitu-gridded-observations-nordic\n\nThis is really a sample text used for testing the CIM - GitHub integration.\n\n## This is a subsection\n\nThis is really a sample text used for testing the CIM - GitHub integration. This is really a sample text used for testing the CIM - GitHub integration. This is really a sample text used for testing the CIM - GitHub integration. This is really a sample text used for testing the CIM - GitHub integration. A large text is written here. A large text is written here. A large text is written here. A large text is written here.\n A large text is written here.\n A large text is written here. A large text is written here.\n A large text is written here. A large text is written here. A large text is written here.\n\n ![Captura de pantalla 2023-06-23 a las 13.31.33.png](https://s3.cds.ecmwf.int/swift/v1/AUTH_3e237111c3a144df8e0e0980577062b4/cds2-backoffice-dev-images/cim-localdocument_28b6c87c-fb51-4b28-8318-99ec2ad1e3bf)"
}, {
"collapsed": false,
"blocks": [ {
"blocks": [ ],
"id": "trends-and-variability",
"type": "section",
"title": "Trends and variability"
} ],
"id": "quality-assessments",
"type": "accordion",
"title": "Quality Assessments"
} ]
},
"quality_assurance_aside": {
"blocks": [ {
"collapsed": true,
"blocks": [ {
"id": "general_qc",
"type": "markdown",
"content": "**General**: Fully meet requirements"
}, {
"id": "accuracy_and_consistency_qc",
"type": "markdown",
"content": "**Accuracy and Consistency**: Fully meet requirements"
}, {
"id": "reliable_access_qc",
"type": "markdown",
"content": "**Reliable Access**: Not fully meet"
}, {
"id": "versioning_and_archiving_qc",
"type": "markdown",
"content": "**Versioning and Archiving**: Not fully meet versioning and archiving."
} ],
"id": "general",
"type": "accordion",
"title": "1. General"
}, {
"collapsed": true,
"blocks": [ {
"id": "data_records_qc",
"type": "markdown",
"content": "**Data record**: Fully meet requirements"
}, {
"id": "consistency_qc",
"type": "markdown",
"content": "**Consistency**: Fully meet requirements"
}, {
"id": "uncertainty_qc",
"type": "markdown",
"content": "**Uncertainty**: Fully meet requirements"
}, {
"id": "updates_qc",
"type": "markdown",
"content": "**Updates**: Fully meet requirements"
} ],
"id": "data_records",
"type": "accordion",
"title": "2. Data record"
}, {
"collapsed": true,
"blocks": [ {
"id": "metadata_qc",
"type": "markdown",
"content": "**Metadata**: Fully meet requirements"
}, {
"id": "discovery_and_use_qc",
"type": "markdown",
"content": "**Discovery and Use**: Fully meet requirements"
}, {
"id": "interoperability_qc",
"type": "markdown",
"content": "**Interoperability**: Fully meet requirements"
} ],
"id": "metadata",
"type": "accordion",
"title": "3. Metadata"
}, {
"collapsed": true,
"blocks": [ {
"id": "documentation_qc",
"type": "markdown",
"content": "**Documentation**: Fully meet requirements"
}, {
"id": "content_qc",
"type": "markdown",
"content": "**Content**: Fully meet requirements"
}, {
"id": "scientific_basis_qc",
"type": "markdown",
"content": "**Scientific Basis**: Fully meet requirements"
}, {
"id": "quality_control_qc",
"type": "markdown",
"content": "**Quality Control**: N/A"
}, {
"id": "user_guidance_qc",
"type": "markdown",
"content": "**User Guidance**: Fully meet requirements"
} ],
"id": "documentation",
"type": "accordion",
"title": "4. Documentation"
} ]
}
}
Loading

0 comments on commit e5658fe

Please sign in to comment.