Skip to content

Commit

Permalink
pg_waldump segment on safekeeper in test_pg_waldump.
Browse files Browse the repository at this point in the history
To test it as well.
  • Loading branch information
arssher committed May 6, 2024
1 parent ce4d3da commit 0353a72
Showing 1 changed file with 25 additions and 10 deletions.
35 changes: 25 additions & 10 deletions test_runner/regress/test_pg_waldump.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,28 @@
import os
import shutil

from fixtures.neon_fixtures import NeonEnv, PgBin
from fixtures.utils import subprocess_capture


def check_wal_segment(pg_waldump_path: str, segment_path: str, test_output_dir):
# use special --ignore option to ignore the validation checks in pg_waldump
# this is necessary, because neon WAL files contain gap at the beginning
output_path, _, _ = subprocess_capture(
test_output_dir, [pg_waldump_path, "--ignore", segment_path]
)

with open(f"{output_path}.stdout", "r") as f:
stdout = f.read()
assert "ABORT" in stdout
assert "COMMIT" in stdout


# Simple test to check that pg_waldump works with neon WAL files
def test_pg_waldump(neon_simple_env: NeonEnv, test_output_dir, pg_bin: PgBin):
env = neon_simple_env
env.neon_cli.create_branch("test_pg_waldump", "empty")
tenant_id = env.initial_tenant
timeline_id = env.neon_cli.create_branch("test_pg_waldump", "empty")
endpoint = env.endpoints.create_start("test_pg_waldump")

cur = endpoint.connect().cursor()
Expand Down Expand Up @@ -35,12 +50,12 @@ def test_pg_waldump(neon_simple_env: NeonEnv, test_output_dir, pg_bin: PgBin):
assert endpoint.pgdata_dir
wal_path = os.path.join(endpoint.pgdata_dir, "pg_wal/000000010000000000000001")
pg_waldump_path = os.path.join(pg_bin.pg_bin_path, "pg_waldump")

# use special --ignore option to ignore the validation checks in pg_waldump
# this is necessary, because neon WAL files contain gap at the beginning
output_path, _, _ = subprocess_capture(test_output_dir, [pg_waldump_path, "--ignore", wal_path])

with open(f"{output_path}.stdout", "r") as f:
stdout = f.read()
assert "ABORT" in stdout
assert "COMMIT" in stdout
# check segment on compute
check_wal_segment(pg_waldump_path, wal_path, test_output_dir)

# Check file on safekeepers as well. pg_waldump is strict about file naming, so remove .partial suffix.
sk = env.safekeepers[0]
sk_tli_dir = sk.timeline_dir(tenant_id, timeline_id)
non_partial_path = os.path.join(sk_tli_dir, "000000010000000000000001")
shutil.copyfile(os.path.join(sk_tli_dir, "000000010000000000000001.partial"), non_partial_path)
check_wal_segment(pg_waldump_path, non_partial_path, test_output_dir)

1 comment on commit 0353a72

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2961 tests run: 2822 passed, 5 failed, 134 skipped (full report)


Failures on Postgres 14

  • test_compare_pg_stats_rw_with_pgbench_default[neon-github-actions-selfhosted-5-10-100]: release
  • test_compare_pg_stats_wo_with_pgbench_simple_update[neon-github-actions-selfhosted-5-10-100]: release
  • test_compare_pg_stats_wal_with_pgbench_default[neon-github-actions-selfhosted-5-10-100]: release
  • test_storage_controller_many_tenants[github-actions-selfhosted]: release
  • test_bulk_tenant_create[github-actions-selfhosted-5]: release
# Run all failed tests locally:
scripts/pytest -vv -n $(nproc) -k "test_compare_pg_stats_rw_with_pgbench_default[neon-release-pg14-github-actions-selfhosted-5-10-100] or test_compare_pg_stats_wo_with_pgbench_simple_update[neon-release-pg14-github-actions-selfhosted-5-10-100] or test_compare_pg_stats_wal_with_pgbench_default[neon-release-pg14-github-actions-selfhosted-5-10-100] or test_storage_controller_many_tenants[release-pg14-github-actions-selfhosted] or test_bulk_tenant_create[release-pg14-github-actions-selfhosted-5]"

Code coverage* (full report)

  • functions: 31.4% (6236 of 19852 functions)
  • lines: 47.1% (46704 of 99117 lines)

* collected from Rust tests only


The comment gets automatically updated with the latest test results
0353a72 at 2024-05-06T05:58:19.269Z :recycle:

Please sign in to comment.