Skip to content

Commit

Permalink
Simplify scanning compute logs in tests (#7997)
Browse files Browse the repository at this point in the history
Implement LogUtils in the Endpoint fixture class, so that the
"log_contains" function can be used on compute logs too.

Per discussion at:
#7288 (comment)
  • Loading branch information
hlinnaka committed Jun 10, 2024
1 parent ae5badd commit 5a7e285
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 25 deletions.
3 changes: 2 additions & 1 deletion test_runner/fixtures/neon_fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -3386,7 +3386,7 @@ def static_proxy(
yield proxy


class Endpoint(PgProtocol):
class Endpoint(PgProtocol, LogUtils):
"""An object representing a Postgres compute endpoint managed by the control plane."""

def __init__(
Expand Down Expand Up @@ -3452,6 +3452,7 @@ def create(
)
path = Path("endpoints") / self.endpoint_id / "pgdata"
self.pgdata_dir = os.path.join(self.env.repo_dir, path)
self.logfile = self.endpoint_path() / "compute.log"

config_lines = config_lines or []

Expand Down
21 changes: 5 additions & 16 deletions test_runner/regress/test_hot_standby.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import asyncio
import os
import re
import threading
import time
from functools import partial
Expand All @@ -18,20 +17,6 @@
from fixtures.utils import wait_until


# Check for corrupted WAL messages which might otherwise go unnoticed if
# reconnection fixes this.
def scan_standby_log_for_errors(secondary):
log_path = secondary.endpoint_path() / "compute.log"
with log_path.open("r") as f:
markers = re.compile(
r"incorrect resource manager data|record with incorrect|invalid magic number|unexpected pageaddr"
)
for line in f:
if markers.search(line):
log.info(f"bad error in standby log: {line}")
raise AssertionError()


def test_hot_standby(neon_simple_env: NeonEnv):
env = neon_simple_env

Expand Down Expand Up @@ -91,7 +76,11 @@ def test_hot_standby(neon_simple_env: NeonEnv):
assert response is not None
assert response == responses[query]

scan_standby_log_for_errors(secondary)
# Check for corrupted WAL messages which might otherwise go unnoticed if
# reconnection fixes this.
assert not secondary.log_contains(
"incorrect resource manager data|record with incorrect|invalid magic number|unexpected pageaddr"
)

# clean up
if slow_down_send:
Expand Down
10 changes: 2 additions & 8 deletions test_runner/regress/test_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ def test_migrations(neon_simple_env: NeonEnv):
env.neon_cli.create_branch("test_migrations", "empty")

endpoint = env.endpoints.create("test_migrations")
log_path = endpoint.endpoint_path() / "compute.log"

endpoint.respec(skip_pg_catalog_updates=False)
endpoint.start()

Expand All @@ -22,9 +20,7 @@ def test_migrations(neon_simple_env: NeonEnv):
migration_id = cur.fetchall()
assert migration_id[0][0] == num_migrations

with open(log_path, "r") as log_file:
logs = log_file.read()
assert f"INFO handle_migrations: Ran {num_migrations} migrations" in logs
endpoint.assert_log_contains(f"INFO handle_migrations: Ran {num_migrations} migrations")

endpoint.stop()
endpoint.start()
Expand All @@ -36,6 +32,4 @@ def test_migrations(neon_simple_env: NeonEnv):
migration_id = cur.fetchall()
assert migration_id[0][0] == num_migrations

with open(log_path, "r") as log_file:
logs = log_file.read()
assert "INFO handle_migrations: Ran 0 migrations" in logs
endpoint.assert_log_contains("INFO handle_migrations: Ran 0 migrations")

1 comment on commit 5a7e285

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

3280 tests run: 3128 passed, 0 failed, 152 skipped (full report)


Flaky tests (3)

Postgres 16

  • test_vm_bit_clear_on_heap_lock: debug

Postgres 15

  • test_vm_bit_clear_on_heap_lock: debug

Postgres 14

Code coverage* (full report)

  • functions: 31.5% (6598 of 20963 functions)
  • lines: 48.4% (51061 of 105401 lines)

* collected from Rust tests only


The comment gets automatically updated with the latest test results
5a7e285 at 2024-06-10T14:20:08.988Z :recycle:

Please sign in to comment.