From eab3a0677dc9338f94c332e4dc831df86816f0c6 Mon Sep 17 00:00:00 2001 From: Frank Niessink Date: Tue, 15 Dec 2020 23:06:57 +0100 Subject: [PATCH] Allow for filtering axe-selenium-python accessibility violations by tag. Closes [#1752. --- .../axe_selenium_python.py | 61 ++++++++++++------- .../test_axe_selenium_python.py | 12 ++++ components/server/src/data/datamodel.json | 38 +++++++++--- docs/CHANGELOG.md | 1 + docs/METRICS_AND_SOURCES.md | 2 + 5 files changed, 86 insertions(+), 28 deletions(-) diff --git a/components/collector/src/source_collectors/file_source_collectors/axe_selenium_python.py b/components/collector/src/source_collectors/file_source_collectors/axe_selenium_python.py index 8fd4df37a9..ebf9eae807 100644 --- a/components/collector/src/source_collectors/file_source_collectors/axe_selenium_python.py +++ b/components/collector/src/source_collectors/file_source_collectors/axe_selenium_python.py @@ -1,11 +1,12 @@ """axe-selenium-python accessibility analysis metric source.""" from datetime import datetime +from typing import Collection from dateutil.parser import parse from base_collectors import JSONFileSourceCollector, SourceUpToDatenessCollector -from collector_utilities.functions import md5_hash +from collector_utilities.functions import md5_hash, match_string_or_regular_expression from collector_utilities.type import Response from source_model import Entity, SourceMeasurement, SourceResponses @@ -14,35 +15,53 @@ class AxeSeleniumPythonAccessibility(JSONFileSourceCollector): """Collector class to get accessibility violations.""" async def _parse_source_responses(self, responses: SourceResponses) -> SourceMeasurement: - impact_levels = self._parameter("impact") entity_attributes = [] for response in responses: json = await response.json(content_type=None) url = json["url"] for violation in json.get("violations", []): for node in violation.get("nodes", []): - if node.get("impact") not in impact_levels: - continue - entity_attributes.append( - dict( - description=violation.get("description"), - element=node.get("html"), - help=violation.get("helpUrl"), - impact=node.get("impact"), - page=url, - url=url, - tags=", ".join(sorted(violation.get("tags", []))), - violation_type=violation.get("id"), + tags = violation.get("tags", []) + impact = node.get("impact") + if self.__include_violation(impact, tags): + entity_attributes.append( + dict( + description=violation.get("description"), + element=node.get("html"), + help=violation.get("helpUrl"), + impact=impact, + page=url, + url=url, + tags=", ".join(sorted(tags)), + violation_type=violation.get("id"), + ) ) - ) - entities = [ - Entity( - key=md5_hash(",".join(str(value) for key, value in attributes.items() if key != "tags")), **attributes - ) - for attributes in entity_attributes - ] + entities = [Entity(key=self.__create_key(attributes), **attributes) for attributes in entity_attributes] return SourceMeasurement(entities=entities) + def __include_violation(self, impact: str, tags: Collection[str]) -> bool: + """Return whether to include the violation.""" + if impact not in self._parameter("impact"): + return False + if tags_to_include := self._parameter("tags_to_include"): + for tag in tags: + if match_string_or_regular_expression(tag, tags_to_include): + break + else: + return False + if tags_to_ignore := self._parameter("tags_to_ignore"): + for tag in tags: + if match_string_or_regular_expression(tag, tags_to_ignore): + return False + return True + + @staticmethod + def __create_key(attributes) -> str: + """Create a key for the entity based on the attributes.""" + # We ignore tags for two reasons: 1) If the violation is the same, so should the tags be. 2) Tags were added to + # the entities later and including them in the key would change the key for existing entities. + return md5_hash(",".join(str(value) for key, value in attributes.items() if key != "tags")) + class AxeSeleniumPythonSourceUpToDateness(JSONFileSourceCollector, SourceUpToDatenessCollector): """Collector to get the source up-to-dateness of axe-selenium-python JSON reports.""" diff --git a/components/collector/tests/source_collectors/file_source_collectors/test_axe_selenium_python.py b/components/collector/tests/source_collectors/file_source_collectors/test_axe_selenium_python.py index d6b6726ac8..65556b4162 100644 --- a/components/collector/tests/source_collectors/file_source_collectors/test_axe_selenium_python.py +++ b/components/collector/tests/source_collectors/file_source_collectors/test_axe_selenium_python.py @@ -86,6 +86,18 @@ async def test_filter_by_impact(self): response = await self.collect(self.metric, get_request_json_return_value=self.json) self.assert_measurement(response, value="1") + async def test_filter_by_tag_include(self): + """Test that violations can be filtered by tag.""" + self.sources["source_id"]["parameters"]["tags_to_include"] = ["wcag2aa"] + response = await self.collect(self.metric, get_request_json_return_value=self.json) + self.assert_measurement(response, value="1", entities=[self.expected_entities[0]]) + + async def test_filter_by_tag_ignore(self): + """Test that violations can be filtered by tag.""" + self.sources["source_id"]["parameters"]["tags_to_ignore"] = ["wcag2aa"] + response = await self.collect(self.metric, get_request_json_return_value=self.json) + self.assert_measurement(response, value="1", entities=[self.expected_entities[1]]) + async def test_zipped_json(self): """Test that a zip archive with JSON files is processed correctly.""" self.sources["source_id"]["parameters"]["url"] = f"{self.axe_json_url}.zip" diff --git a/components/server/src/data/datamodel.json b/components/server/src/data/datamodel.json index 8aa0b63d78..7befe509cc 100644 --- a/components/server/src/data/datamodel.json +++ b/components/server/src/data/datamodel.json @@ -921,6 +921,30 @@ "source_up_to_dateness" ] }, + "tags_to_include": { + "name": "Tags to include (regular expressions or tags)", + "short_name": "tags to include", + "help": "Tags to include can be specified by tag or by regular expression.", + "type": "multiple_choice_with_addition", + "placeholder": "all", + "mandatory": false, + "default_value": [], + "metrics": [ + "accessibility" + ] + }, + "tags_to_ignore": { + "name": "Tags to ignore (regular expressions or tags)", + "short_name": "tags to ignore", + "help": "Tags to ignore can be specified by tag or by regular expression.", + "type": "multiple_choice_with_addition", + "placeholder": "none", + "mandatory": false, + "default_value": [], + "metrics": [ + "accessibility" + ] + }, "impact": { "name": "Impact levels", "short_name": "impact levels", @@ -1373,7 +1397,7 @@ "help": "Pipelines to ignore can be specified by pipeline name or by regular expression. Use {folder name}/{pipeline name} for the names of pipelines in folders.", "type": "multiple_choice_with_addition", "placeholder": "none", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "failed_jobs", @@ -2314,7 +2338,7 @@ "help": "Jobs to ignore can be specified by job name or by regular expression.", "type": "multiple_choice_with_addition", "placeholder": "none", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "failed_jobs", @@ -2546,7 +2570,7 @@ "help": "Jobs to include can be specified by job name or by regular expression. Use {parent job name}/{child job name} for the names of nested jobs.", "type": "multiple_choice_with_addition", "placeholder": "all", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "failed_jobs", @@ -2560,7 +2584,7 @@ "help": "Jobs to ignore can be specified by job name or by regular expression. Use {parent job name}/{child job name} for the names of nested jobs.", "type": "multiple_choice_with_addition", "placeholder": "none", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "failed_jobs", @@ -4061,7 +4085,7 @@ "help": "Transactions to ignore can be specified by transaction name or by regular expression.", "type": "multiple_choice_with_addition", "placeholder": "none", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "slow_transactions", @@ -4074,7 +4098,7 @@ "help": "Transactions to include can be specified by transaction name or by regular expression.", "type": "multiple_choice_with_addition", "placeholder": "all", - "mandatory": false, + "mandatory": false, "default_value": [], "metrics": [ "slow_transactions", @@ -5143,7 +5167,7 @@ "component": { "name": "Project key", "short_name": "project key", - "help": "The project key can be found by opening the project in SonarQube and looking at the bottom of the grey column on the right.", + "help": "The project key can be found by opening the project in SonarQube and looking at the bottom of the grey column on the right.", "type": "string", "mandatory": true, "default_value": "", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b993cb178b..1242439e83 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Add the tags of accessibility rules to the detail information of axe-selenium-python sources. Closes [#1751](https://github.com/ICTU/quality-time/issues/1751). +- Allow for filtering axe-selenium-python accessibility violations by tag. Closes [#1752](https://github.com/ICTU/quality-time/issues/1752). ## [3.16.0] - [2020-12-12] diff --git a/docs/METRICS_AND_SOURCES.md b/docs/METRICS_AND_SOURCES.md index e9bfb066b4..c6924c103d 100644 --- a/docs/METRICS_AND_SOURCES.md +++ b/docs/METRICS_AND_SOURCES.md @@ -101,6 +101,8 @@ This document lists all [metrics](#metrics) that *Quality-time* can measure and | Impact levels | Multiple choice | No | If provided, only count accessibility violations with the impact levels. | | Password for basic authentication | Password | No | | | Private token | Password | No | | +| Tags to ignore (regular expressions or tags) | Multiple choice with addition | No | Tags to ignore can be specified by tag or by regular expression. | +| Tags to include (regular expressions or tags) | Multiple choice with addition | No | Tags to include can be specified by tag or by regular expression. | | URL to an axe-selenium-python report in JSON format or to a zip with axe-selenium-python reports in JSON format | URL | Yes | | | URL to an axe-selenium-python report in a human readable format | String | No | If provided, users clicking the source URL will visit this URL instead of the axe-selenium-report report in JSON format. | | Username for basic authentication | String | No | |