Skip to content

Commit

Permalink
WIP - poll with python
Browse files Browse the repository at this point in the history
This reduces the roundtrips and allows us to maintain "temporary"
mount points for btrfs filesystems that are not otherwise mounted.

The process will be shutdown when the cockpit session is closed by
cockpit-ws, so we get reliable cleanup.
  • Loading branch information
mvollmer committed Aug 16, 2024
1 parent 01357bd commit f4fd732
Show file tree
Hide file tree
Showing 4 changed files with 282 additions and 115 deletions.
199 changes: 199 additions & 0 deletions pkg/storaged/btrfs/btrfs-tool.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,199 @@
#! /usr/bin/python3

# btrfs-tool -- Query and monitor btrfs filesystems
#
# This program monitors all btrfs filesystems and reports their
# subvolumes and other things.
#
# It can do that continously, or as a one shot operation. The tool
# mounts btrfs filesystems as necessary to retrieve the requested
# information, but does it in a polite way: they are mounted once and
# then left mounted until that is no longer needed. Typically, you
# might see some mounts when a Cockpit session starts, and the
# corresponding unmounts when it ends.
#
# This tool can be run multiple times concurrently, and it wont get
# confused.

import contextlib
import subprocess
import json
import re
import sys
import time
import os
import fcntl
import select

Check notice

Code scanning / CodeQL

Unused import Note

Import of 'select' is not used.
import signal

TMP_MP_DIR = "/var/lib/cockpit/btrfs"

def ensure_tmp_mp_dir():
os.makedirs(TMP_MP_DIR, mode=0o700, exist_ok=True)

@contextlib.contextmanager
def atomic_file(path):
fd = os.open(path, os.O_RDWR | os.O_CREAT)
fcntl.flock(fd, fcntl.LOCK_EX)
data = os.read(fd, 100000)
blob = json.loads(data) if len(data) > 0 else { }
try:
yield blob
data = json.dumps(blob).encode() + b"\n"
os.lseek(fd, 0, os.SEEK_SET)
os.truncate(fd, 0)
os.write(fd, data)
finally:
os.close(fd)

def list_filesystems():
output = json.loads(subprocess.check_output(["lsblk", "-Jplno", "NAME,FSTYPE,UUID,MOUNTPOINTS"]))
filesystems = {}
for b in output['blockdevices']:
if b['fstype'] == "btrfs":
uuid = b['uuid']
mps = list(filter(lambda x: x is not None and not x.startswith(TMP_MP_DIR), b['mountpoints']))
if uuid not in filesystems:
filesystems[uuid] = { 'uuid': uuid, 'devices': [ b['name'] ], 'mountpoints': mps }
else:
filesystems[uuid]['devices'] += [ b['name'] ]
filesystems[uuid]['mountpoints'] += mps
return filesystems

tmp_mountpoints = set()

def add_tmp_mountpoint(uuid, dev):
global tmp_mountpoints
if uuid not in tmp_mountpoints:
sys.stderr.write(f"ADDING {uuid}\n")
tmp_mountpoints.add(uuid)
ensure_tmp_mp_dir()
with atomic_file(TMP_MP_DIR + "/db") as db:
if uuid in db and db[uuid] > 0:
db[uuid] += 1
else:
db[uuid] = 1
# XXX - mount as read-only? That interferes with other mounts.
dir = TMP_MP_DIR + "/" + uuid
sys.stderr.write(f"MOUNTING {dir}\n")
os.makedirs(dir, exist_ok=True)
subprocess.check_call(["mount", dev, dir])

def remove_tmp_mountpoint(uuid):
global tmp_mountpoints
if uuid in tmp_mountpoints:
sys.stderr.write(f"REMOVING {uuid}\n")
tmp_mountpoints.remove(uuid)
ensure_tmp_mp_dir()
with atomic_file(TMP_MP_DIR + "/db") as db:
if db[uuid] == 1:
dir = TMP_MP_DIR + "/" + uuid
try:
sys.stderr.write(f"UNMOUNTING {dir}\n")
subprocess.check_call(["umount", dir])
# subprocess.check_call(["rmdir", dir])
except:

Check notice

Code scanning / CodeQL

Except block handles 'BaseException' Note

Except block directly handles BaseException.
# XXX - log error, try harder?
pass
del db[uuid]
else:
db[uuid] -= 1

def remove_all_tmp_mountpoints():
for mp in set(tmp_mountpoints):
remove_tmp_mountpoint(mp)

def ensure_mount_point(fs):
if len(fs['mountpoints']) > 0:
remove_tmp_mountpoint(fs['uuid'])
return fs['mountpoints'][0]
else:
add_tmp_mountpoint(fs['uuid'], fs['devices'][0])
return TMP_MP_DIR + "/" + fs['uuid']

def get_subvolume_info(mp):
try:
lines = subprocess.check_output(["btrfs", "subvolume", "list", "-apuq", mp]).splitlines()
subvols = []
for line in lines:
match = re.match(b"ID (\\d+).*parent (\\d+).*parent_uuid (.*)uuid (.*) path (<FS_TREE>/)?(.*)", line);
if match:
subvols += [
{
'pathname': match[6].decode(errors='replace'),
'id': int(match[1]),
'parent': int(match[2]),
'uuid': match[4].decode(),
'parent_uuid': None if match[3][0] == ord("-") else match[3].decode()
}
]
return subvols
except:

Check notice

Code scanning / CodeQL

Except block handles 'BaseException' Note

Except block directly handles BaseException.
# XXX - export error message?
return None

def get_default_subvolume(mp):
output = subprocess.check_output(["btrfs", "subvolume", "get-default", mp])
match = re.match(b"ID (\\d+).*", output);
if match:
return int(match[1]);
else:
return None

def get_usages(uuid):
output = subprocess.check_output(["btrfs", "filesystem", "show", "--raw", uuid])
usages = {}
for line in output.splitlines():
match = re.match(b".*used\\s+(\\d+)\\s+path\\s+([\\w/]+).*", line)
if match:
usages[match[2].decode()] = int(match[1]);
return usages;

def poll():
sys.stderr.write("POLL\n")
filesystems = list_filesystems()
info = { }
for fs in filesystems.values():
mp = ensure_mount_point(fs)
if mp:
info[fs['uuid']] = {
'subvolumes': get_subvolume_info(mp),
'default_subvolume': get_default_subvolume(mp),
'usages': get_usages(fs['uuid']),
}
return info

def cmd_monitor():
old_infos = poll()
sys.stdout.write(json.dumps(old_infos) + "\n")
sys.stdout.flush()
while True:
time.sleep(5.0)
new_infos = poll()
if new_infos != old_infos:
sys.stdout.write(json.dumps(new_infos) + "\n")
sys.stdout.flush()
old_infos = new_infos

def cmd_poll():
infos = poll()
sys.stdout.write(json.dumps(infos) + "\n")
sys.stdout.flush()

def cmd(args):
if len(args) > 1:
if args[1] == "poll":
cmd_poll()
elif args[1] == "monitor":
cmd_monitor()

def main(args):
signal.signal(signal.SIGTERM, lambda _signo, _stack: sys.exit(0))
try:
cmd(args)
except Exception as err:
sys.stderr.write(str(err) + "\n")
finally:
remove_all_tmp_mountpoints()

main(sys.argv)
Loading

0 comments on commit f4fd732

Please sign in to comment.