Support GitLab CI pipelines as source for the source-up-to-dateness metric

components/collector/src/source_collectors/gitlab/base.py

@@ -73,7 +73,7 @@ async def _next_urls(self, responses: SourceResponses) -> list[URL]:
         lookback_date = date.today() - timedelta(days=int(cast(str, self._parameter("lookback_days"))))
         for response in responses:
             for job in await response.json():
-                if self.__build_date(job) > lookback_date:
+                if self._build_date(job) > lookback_date:
                     return await super()._next_urls(responses)
         return []
 
@@ -88,7 +88,7 @@ async def _parse_entities(self, responses: SourceResponses) -> Entities:
                     build_status=job["status"],
                     branch=job["ref"],
                     stage=job["stage"],
-                    build_date=str(self.__build_date(job)),
+                    build_date=str(self._build_date(job)),
                 )
                 for job in await self._jobs(responses)
             ]
@@ -115,6 +115,6 @@ def _count_job(self, job: Job) -> bool:
         ) and not match_string_or_regular_expression(job["ref"], self._parameter("refs_to_ignore"))
 
     @staticmethod
-    def __build_date(job: Job) -> date:
+    def _build_date(job: Job) -> date:
         """Return the build date of the job."""
         return parse(job.get("finished_at") or job["created_at"]).date()

components/collector/src/source_collectors/gitlab/source_up_to_dateness.py

@@ -2,18 +2,22 @@
 
 import asyncio
 import itertools
+from abc import ABC
+from datetime import datetime
 from urllib.parse import quote
 
+import aiohttp
 from dateutil.parser import parse
 
+from base_collectors import SourceCollector, TimePassedCollector
 from collector_utilities.functions import days_ago
-from collector_utilities.type import URL, Value
-from model import SourceResponses
+from collector_utilities.type import Response, URL, Value
+from model import SourceMeasurement, SourceResponses
 
-from .base import GitLabProjectBase
+from .base import GitLabJobsBase, GitLabProjectBase
 
 
-class GitLabSourceUpToDateness(GitLabProjectBase):
+class GitLabFileUpToDateness(GitLabProjectBase):
     """Collector class to measure the up-to-dateness of a repo or folder/file in a repo."""
 
     async def _api_url(self) -> URL:
@@ -68,3 +72,35 @@ async def _parse_value(self, responses: SourceResponses) -> Value:
         commit_responses = responses[1:]
         commit_dates = [parse((await response.json())["committed_date"]) for response in commit_responses]
         return str(days_ago(max(commit_dates)))
+
+
+class GitLabPipelineUpToDateness(TimePassedCollector, GitLabJobsBase):
+    """Collector class to measure the up-to-dateness of a job/pipeline."""
+
+    async def _landing_url(self, responses: SourceResponses) -> URL:
+        """Override to return a landing URL for the pipeline."""
+        if responses and (json := await responses[0].json()):
+            return URL(json[0]["pipeline"]["web_url"])
+        return await super()._landing_url(responses)
+
+    async def _parse_source_response_date_time(self, response: Response) -> datetime:
+        """Override to get the date and time of the pipeline."""
+        jobs = await self._jobs(SourceResponses(responses=[response]))
+        build_dates = [self._build_date(job) for job in jobs]
+        return datetime.combine(max(build_dates, default=datetime.min.date()), datetime.min.time())
+
+
+class GitLabSourceUpToDateness(SourceCollector, ABC):
+    """Factory class to create a collector to get the up-to-dateness of either pipelines or files."""
+
+    def __new__(cls, session: aiohttp.ClientSession, source):
+        """Create an instance of either the file up-to-dateness collector or the jobs up-to-dateness collector."""
+        file_path = source.get("parameters", {}).get("file_path")
+        collector_class = GitLabFileUpToDateness if file_path else GitLabPipelineUpToDateness
+        instance = collector_class(session, source)
+        instance.source_type = cls.source_type
+        return instance
+
+    async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement:
+        """Override to document that this class does not parse responses itself."""
+        raise NotImplementedError

components/collector/tests/source_collectors/gitlab/base.py

@@ -21,6 +21,7 @@ def setUp(self):
                 name="job1",
                 stage="stage",
                 created_at="2019-03-31T19:40:39.927Z",
+                pipeline=dict(web_url="https://gitlab/project/-/pipelines/1"),
                 web_url="https://gitlab/job1",
                 ref="master",
             ),

components/collector/tests/source_collectors/gitlab/test_source_up_to_dateness.py

@@ -1,6 +1,6 @@
 """Unit tests for the GitLab source up-to-dateness collector."""
 
-from datetime import datetime, timezone
+from datetime import datetime, date, timezone
 from unittest.mock import AsyncMock, Mock, patch
 
 from .base import GitLabTestCase
@@ -17,12 +17,16 @@ def setUp(self):
         super().setUp()
         self.commit_json = dict(committed_date="2019-01-01T09:06:12+00:00")
         self.expected_age = (datetime.now(timezone.utc) - datetime(2019, 1, 1, 9, 6, 9, tzinfo=timezone.utc)).days
-        self.head_response = Mock()
-        self.head_response.headers = {"X-Gitlab-Last-Commit-Id": "commit-sha"}
+
+    @staticmethod
+    def patched_client_session_head():
+        """Return a patched version of the client session head method."""
+        head_response = Mock(headers={"X-Gitlab-Last-Commit-Id": "commit-sha"})
+        return patch("aiohttp.ClientSession.head", AsyncMock(side_effect=[head_response, head_response]))
 
     async def test_source_up_to_dateness_file(self):
         """Test that the age of a file in a repo can be measured."""
-        with patch("aiohttp.ClientSession.head", AsyncMock(return_value=self.head_response)):
+        with self.patched_client_session_head():
             response = await self.collect(
                 get_request_json_side_effect=[[], self.commit_json, dict(web_url="https://gitlab.com/project")]
             )
@@ -32,7 +36,7 @@ async def test_source_up_to_dateness_file(self):
 
     async def test_source_up_to_dateness_folder(self):
         """Test that the age of a folder in a repo can be measured."""
-        with patch("aiohttp.ClientSession.head", AsyncMock(side_effect=[self.head_response, self.head_response])):
+        with self.patched_client_session_head():
             response = await self.collect(
                 get_request_json_side_effect=[
                     [dict(type="blob", path="file.txt"), dict(type="tree", path="folder")],
@@ -46,7 +50,22 @@ async def test_source_up_to_dateness_folder(self):
             response, value=str(self.expected_age), landing_url="https://gitlab.com/project/blob/branch/file"
         )
 
-    async def test_landing_url_on_failure(self):
+    async def test_source_up_to_dateness_pipeline(self):
+        """Test that the age of a pipeline can be measured."""
+        self.set_source_parameter("file_path", "")
+        build_date = datetime.fromisoformat(self.gitlab_jobs_json[0]["created_at"].strip("Z")).date()
+        expected_age = (date.today() - build_date).days
+        with self.patched_client_session_head():
+            response = await self.collect(get_request_json_return_value=self.gitlab_jobs_json)
+        self.assert_measurement(response, value=str(expected_age), landing_url="https://gitlab/project/-/pipelines/1")
+
+    async def test_file_landing_url_on_failure(self):
+        """Test that the landing url is the API url when GitLab cannot be reached."""
+        response = await self.collect(get_request_json_side_effect=[ConnectionError])
+        self.assert_measurement(response, landing_url="https://gitlab", connection_error="Traceback")
+
+    async def test_pipeline_landing_url_on_failure(self):
         """Test that the landing url is the API url when GitLab cannot be reached."""
+        self.set_source_parameter("file_path", "")
         response = await self.collect(get_request_json_side_effect=[ConnectionError])
         self.assert_measurement(response, landing_url="https://gitlab", connection_error="Traceback")

components/shared_data_model/src/shared_data_model/sources/azure_devops.py

@@ -76,7 +76,7 @@
             mandatory=False,
             help="This should only contain the WHERE clause of a WIQL query, as the selected fields are static. "
             "For example, use the following clause to hide issues marked as done: \"[System.State] <> 'Done'\". "
-            "See: https://docs.microsoft.com/en-us/azure/devops/boards/queries/wiql-syntax?view=azure-devops",
+            "See https://docs.microsoft.com/en-us/azure/devops/boards/queries/wiql-syntax?view=azure-devops",
             metrics=["issues", "lead_time_for_changes", "user_story_points"],
         ),
         file_path=StringParameter(

components/shared_data_model/src/shared_data_model/sources/gitlab.py

@@ -114,8 +114,8 @@
         file_path=StringParameter(
             name="File or folder path",
             short_name="path",
-            mandatory=True,
-            help_url="https://docs.gitlab.com/ee/api/repository_files.html",
+            help="Use the date and time the path was last changed to determine the up-to-dateness. If no path "
+            "is specified, the pipeline is used to determine the up-to-dateness.",
             metrics=["source_up_to_dateness"],
         ),
         branch=Branch(help_url=GITLAB_BRANCH_HELP_URL),
@@ -124,7 +124,7 @@
             name="Branches and tags to ignore (regular expressions, branch names or tag names)",
             short_name="branches and tags to ignore",
             help_url=GITLAB_BRANCH_HELP_URL,
-            metrics=["failed_jobs", "job_runs_within_time_period", "unused_jobs"],
+            metrics=["failed_jobs", "job_runs_within_time_period", "source_up_to_dateness", "unused_jobs"],
         ),
         inactive_days=Days(
             name="Number of days since last commit after which to consider branches inactive",
@@ -143,13 +143,13 @@
             name="Jobs to ignore (regular expressions or job names)",
             short_name="jobs to ignore",
             help="Jobs to ignore can be specified by job name or by regular expression.",
-            metrics=["failed_jobs", "job_runs_within_time_period", "unused_jobs"],
+            metrics=["failed_jobs", "job_runs_within_time_period", "source_up_to_dateness", "unused_jobs"],
         ),
         lookback_days=Days(
             name="Number of days to look back in selecting pipeline jobs to consider",
             short_name="number of days to look back",
             default_value="90",
-            metrics=["failed_jobs", "job_runs_within_time_period", "unused_jobs"],
+            metrics=["failed_jobs", "job_runs_within_time_period", "source_up_to_dateness", "unused_jobs"],
         ),
         merge_request_state=MergeRequestState(values=["opened", "locked", "merged", "closed"]),
         approval_state=MultipleChoiceParameter(

docs/src/changelog.md

@@ -8,6 +8,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 <!-- The line "## <square-bracket>Unreleased</square-bracket>" is replaced by the release/release.py script with the new release version and release date. -->
 
+## [Unreleased]
+
+### Deployment notes
+
+If your currently installed *Quality-time* version is v4.0.0 or older, please read the v4.0.0 deployment notes.
+
+### Added
+
+- Support GitLab CI pipelines as source for the source-up-to-dateness metric. Closes [#3927](https://github.com/ICTU/quality-time/issues/3927).
+
 ## v4.6.1 - 2022-11-07
 
 ### Deployment notes