diff --git a/teuthology/task/ansible.py b/teuthology/task/ansible.py index 6ccdc7664..ea0f9c594 100644 --- a/teuthology/task/ansible.py +++ b/teuthology/task/ansible.py @@ -43,7 +43,9 @@ def analyze(self, failure_log): lines = set() if failure_obj is None: return lines - for host_obj in failure_obj.items(): + for host_obj in failure_obj.values(): + if not isinstance(host_obj, dict): + continue lines = lines.union(self.analyze_host_record(host_obj)) return lines @@ -57,24 +59,41 @@ def analyze_host_record(self, record): if "cpan" in cmd: lines.add(f"CPAN command failed: {cmd}") continue - lines_to_analyze = result.get("stderr_lines", result["msg"].split("\n")) + lines_to_analyze = [] + if "stderr_lines" in result: + lines_to_analyze = result["stderr_lines"] + elif "msg" in result: + lines_to_analyze = result["msg"].split("\n") + lines_to_analyze.extend(result.get("err", "").split("\n")) for line in lines_to_analyze: - line = self.analyze_line(line) + line = self.analyze_line(line.strip()) if line: lines.add(line) return list(lines) def analyze_line(self, line): - # apt output sometimes contains warnings or suggestions. Those won't be - # helpful, so throw them out. if line.startswith("W: ") or line.endswith("?"): return "" + drop_phrases = [ + # apt output sometimes contains warnings or suggestions. Those won't be + # helpful, so throw them out. + r"^W: ", + r"\?$", + # some output from SSH is not useful + r"Warning: Permanently added .+ to the list of known hosts.", + r"^@+$", + ] + for phrase in drop_phrases: + match = re.search(rf"({phrase})", line, flags=re.IGNORECASE) + if match: + return "" # Next, we can normalize some common phrases. phrases = [ "connection timed out", r"(unable to|could not) connect to [^ ]+", r"temporary failure resolving [^ ]+", + r"Permissions \d+ for '.+' are too open.", ] for phrase in phrases: match = re.search(rf"({phrase})", line, flags=re.IGNORECASE) @@ -363,12 +382,12 @@ def _handle_failure(self, command, status): try: analyzer = FailureAnalyzer() failures = analyzer.analyze(fail_log) - except Exception as e: + except yaml.YAMLError as e: log.error( - "Failed to parse ansible failure log: {0} ({1})".format( - self.failure_log.name, e - ) + f"Failed to parse ansible failure log: {self.failure_log.name} ({e})" ) + except Exception: + log.exception(f"Failed to analyze ansible failure log: {self.failure_log.name}") # If we hit an exception, or if analyze() returned nothing, use the log as-is if not failures: failures = fail_log.replace('\n', '') diff --git a/teuthology/test/task/test_ansible.py b/teuthology/test/task/test_ansible.py index 939ec3f93..9f378b480 100644 --- a/teuthology/test/task/test_ansible.py +++ b/teuthology/test/task/test_ansible.py @@ -26,6 +26,14 @@ class TestFailureAnalyzer: @mark.parametrize( 'line,result', [ + [ + "W: --force-yes is deprecated, use one of the options starting with --allow instead.", + "", + ], + [ + "E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?", + "", + ], [ "E: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.41-4ubuntu3.14_amd64.deb Unable to connect to archive.ubuntu.com:http:", "Unable to connect to archive.ubuntu.com:http:" @@ -34,6 +42,14 @@ class TestFailureAnalyzer: "E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/libb/libb-hooks-op-check-perl/libb-hooks-op-check-perl_0.22-1build2_amd64.deb Temporary failure resolving 'archive.ubuntu.com'", "Temporary failure resolving 'archive.ubuntu.com'" ], + [ + "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\".", + "Data could not be sent to remote host \"smithi068.front.sepia.ceph.com\"." + ], + [ + "Permissions 0644 for '/root/.ssh/id_rsa' are too open.", + "Permissions 0644 for '/root/.ssh/id_rsa' are too open." + ], ] ) def test_lines(self, line, result):