Skip to content

Commit

Permalink
Fifth pass, restore oc.py from f-string leakage and manual conversions
Browse files Browse the repository at this point in the history 
Signed-off-by: Trevor Benson <trevor.benson@gmail.com>
  • Loading branch information
@TrevorBenson
TrevorBenson committed Feb 25, 2024
1 parent a32b414 commit 118087f
Show file tree
Hide file tree
Showing 136 changed files with 1,157 additions and 1,437 deletions.
2 changes: 1 addition & 1 deletion bin/sos-collector
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ except KeyboardInterrupt:
if __name__ == '__main__':
msg = ("Please note the 'sos-collector' command has been deprecated in "
" favor of the new 'sos' command, E.G. 'sos collect'.\n"
"Redirecting to 'sos collect %s'" % (' '.join(sys.argv[1:]) or ''))
f"Redirecting to 'sos collect {' '.join(sys.argv[1:]) or ''}'")
print(msg)
time.sleep(0.5)
args = sys.argv[1:]
Expand Down
2 changes: 1 addition & 1 deletion bin/sosreport
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ except KeyboardInterrupt:
if __name__ == '__main__':
msg = ("Please note the 'sosreport' command has been deprecated in favor "
"of the new 'sos' command, E.G. 'sos report'.\n"
"Redirecting to 'sos report %s'" % (' '.join(sys.argv[1:]) or ''))
f"Redirecting to 'sos report {' '.join(sys.argv[1:]) or ''}'")
print(msg)
time.sleep(0.5)
args = sys.argv[1:]
Expand Down
37 changes: 24 additions & 13 deletions plugins_overview.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
# - list of paths it forbits to collect (add_forbidden_path)
# - list of commands it calls (add_cmd_output)
#
# Output of the script:
# Output of the script:
# - a JSON object with plugins in keys
# - or CSV format in case "csv" cmdline is provided
#
Expand All @@ -29,23 +29,26 @@
plugs_data = {} # the map of all plugins data to collect
plugcontent = '' # content of plugin file just being processed


# method to parse an item of a_s_c/a_c_o/.. methods
# we work on an assumption the item is a string quoted by \" or optionally
# by \'. If we detect at least 2 such chars in the item, take what is between those.
# by \'. If we detect at least 2 such chars in the item, take what is between
# those.
def add_valid_item(dest, item):
for qoutemark in "\"\'":
split = item.split(qoutemark)
if len(split) > 2:
dest.append(split[1])
return

# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..) split
# by comma; add each valid item to the `dest` list

# method to find in `plugcontent` all items of given method (a_c_s/a_c_o/..)
# split by comma; add each valid item to the `dest` list
def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
regexp = "%s%s(.*?)%s" % (method, wrapopen, wrapclose)
regexp = f"{method}{wrapopen}(.*?){wrapclose}"
for match in re.findall(regexp, plugcontent, flags=re.MULTILINE|re.DOTALL):
# tuple of distros ended by either (class|from|import)
if isinstance(match,tuple):
if isinstance(match, tuple):
for item in list(match):
if item not in ['class', 'from', 'import']:
for it in item.split(','):
Expand All @@ -63,7 +66,8 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
else:
add_valid_item(dest, match)

# main body: traverse report's plugins directory and for each plugin, grep for

# main body: traverse report's plugins directory and for each plugin, grep for
# add_copy_spec / add_forbidden_path / add_cmd_output there
for plugfile in sorted(os.listdir(PLUGDIR)):
# ignore non-py files and __init__.py
Expand All @@ -73,7 +77,8 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
# if plugname != 'bcache':
# continue
plugs_data[plugname] = {
'sourcecode': 'https://github.com/sosreport/sos/blob/main/sos/report/plugins/%s.py' % plugname,
'sourcecode': 'https://github.com/sosreport/sos/blob/'
f'main/sos/report/plugins/{plugname}.py',
'distros': [],
'profiles': [],
'packages': [],
Expand All @@ -84,8 +89,13 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
'journals': [],
'env': [],
}
plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n','')
add_all_items("from sos.report.plugins import ", plugs_data[plugname]['distros'], wrapopen='', wrapclose='(class|from|import)')
plugcontent = open(os.path.join(PLUGDIR, plugfile)).read().replace('\n', '')
add_all_items(
"from sos.report.plugins import ",
plugs_data[plugname]['distros'],
wrapopen='',
wrapclose='(class|from|import)'
)
add_all_items("profiles = ", plugs_data[plugname]['profiles'], wrapopen='')
add_all_items("packages = ", plugs_data[plugname]['packages'], wrapopen='')
add_all_items("add_copy_spec", plugs_data[plugname]['copyspecs'])
Expand All @@ -98,7 +108,8 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):

# print output; if "csv" is cmdline argument, print in CSV format, else JSON
if (len(sys.argv) > 1) and (sys.argv[1] == "csv"):
print("plugin;url;distros;profiles;packages;copyspecs;forbidden;commands;service_status;journals;env_vars")
print("plugin;url;distros;profiles;packages;copyspecs;forbidden;commands;"
"service_status;journals;env_vars")
for plugname in plugs_data.keys():
plugin = plugs_data[plugname]
# determine max number of lines - usually "max(len(copyspec),len(commands))"
Expand All @@ -109,10 +120,10 @@ def add_all_items(method, dest, wrapopen='\(', wrapclose='\)'):
for key in plugkeys:
maxline = max(maxline, len(plugin[key]))
for line in range(maxline):
out = ";" if line>0 else ("%s;%s" % (plugname, plugin['sourcecode']))
out = ";" if line > 0 else f"{plugname};{plugin['sourcecode']}"
for key in plugkeys:
out += ";"
if line<len(plugin[key]):
if line < len(plugin[key]):
out += plugin[key][line]
print(out)
else:
Expand Down
95 changes: 44 additions & 51 deletions sos/archive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,9 +148,8 @@ def __init__(self, name, tmpdir, policy, threads, enc_opts, sysroot,
self._archive_root = os.path.join(tmpdir, name)
with self._path_lock:
os.makedirs(self._archive_root, 0o700)
self.log_info(
f"initialised empty FileCacheArchive at '{self._archive_root}'"
)
self.log_info("initialised empty FileCacheArchive at "
f"'{self._archive_root}'")

def dest_path(self, name):
if os.path.isabs(name):
Expand Down Expand Up @@ -246,9 +245,8 @@ def in_archive(path):
if os.path.isabs(target):
target = os.path.relpath(target, target_dir)

self.log_debug(
f"Making symlink '{abs_path}' -> '{target}'"
)
self.log_debug("Making symlink "
f"'{abs_path}' -> '{target}'")
os.symlink(target, abs_path)
else:
self.log_debug(f"Making directory {abs_path}")
Expand Down Expand Up @@ -298,9 +296,8 @@ def check_path(self, src, path_type, dest=None, force=False):

# Check containing directory presence and path type
if os.path.exists(dest_dir) and not os.path.isdir(dest_dir):
raise ValueError(
f"path '{dest_dir}' exists and is not a directory"
)
raise ValueError(f"path '{dest_dir}' exists and is not a "
"directory")
if not os.path.exists(dest_dir):
src_dir = src if path_type == P_DIR else os.path.split(src)[0]
self._make_leading_paths(src_dir)
Expand All @@ -320,15 +317,23 @@ def is_special(mode):
if os.path.exists(dest):
# Use lstat: we care about the current object, not the referent.
st = os.lstat(dest)
ve_msg = "path '%s' exists and is not a %s"
ve_msg = "path '{path}' exists and is not a {path_type}"
if path_type == P_FILE and not stat.S_ISREG(st.st_mode):
raise ValueError(ve_msg % (dest, "regular file"))
raise ValueError(
f"{ve_msg.format(path=dest, path_type='regular file')}"
)
if path_type == P_LINK and not stat.S_ISLNK(st.st_mode):
raise ValueError(ve_msg % (dest, "symbolic link"))
raise ValueError(
f"{ve_msg.format(path=dest, path_type='symbolic link')}"
)
if path_type == P_NODE and not is_special(st.st_mode):
raise ValueError(ve_msg % (dest, "special file"))
raise ValueError(
f"{ve_msg.format(path=dest, path_type='special file')}"
)
if path_type == P_DIR and not stat.S_ISDIR(st.st_mode):
raise ValueError(ve_msg % (dest, "directory"))
raise ValueError(
f"{ve_msg.format(path=dest, path_type='directory')}"
)
# Path has already been copied: skip
return None
return dest
Expand Down Expand Up @@ -379,9 +384,8 @@ def add_file(self, src, dest=None, force=False):
f.write(line)
file_name = "open file"

self.log_debug(
f"added {file_name} to FileCacheArchive '{self._archive_root}'"
)
self.log_debug(f"added {file_name} to FileCacheArchive "
f"'{self._archive_root}'")

def add_string(self, content, dest, mode='w'):
with self._path_lock:
Expand All @@ -399,10 +403,8 @@ def add_string(self, content, dest, mode='w'):
f.write(content)
if os.path.exists(src):
self._copy_attributes(src, dest)
self.log_debug(
f"added string at '{src}' to FileCacheArchive"
f" '{self._archive_root}'"
)
self.log_debug(f"added string at '{src}' to FileCacheArchive"
f" '{self._archive_root}'")

def add_binary(self, content, dest):
with self._path_lock:
Expand All @@ -412,10 +414,8 @@ def add_binary(self, content, dest):

with codecs.open(dest, 'wb', encoding=None) as f:
f.write(content)
self.log_debug(
f"added binary content at '{dest}' to archive"
f" '{self._archive_root}'"
)
self.log_debug(f"added binary content at '{dest}' to archive"
f" '{self._archive_root}'")

def add_link(self, source, link_name):
self.log_debug(f"adding symlink at '{link_name}' -> '{source}'")
Expand All @@ -426,10 +426,8 @@ def add_link(self, source, link_name):

if not os.path.lexists(dest):
os.symlink(source, dest)
self.log_debug(
f"added symlink at '{dest}' to '{source}' in archive"
f" '{self._archive_root}'"
)
self.log_debug(f"added symlink at '{dest}' to '{source}' in "
f"archive '{self._archive_root}'")

# Follow-up must be outside the path lock: we recurse into
# other monitor methods that will attempt to reacquire it.
Expand Down Expand Up @@ -476,13 +474,11 @@ def is_loop(link_name, source):
source = os.path.join(dest_dir, os.readlink(host_path_name))
source = os.path.relpath(source, dest_dir)
if is_loop(link_name, source):
self.log_debug(
f"Link '{link_name}' - '{source}' loops: skipping..."
)
self.log_debug(f"Link '{link_name}' - '{source}' loops: "
"skipping...")
return
self.log_debug(
f"Adding link {link_name} -> {source} for link follow up"
)
self.log_debug(f"Adding link {link_name} -> {source} for "
"link follow up")
self.add_link(source, link_name)
elif os.path.isdir(host_path_name):
self.log_debug(f"Adding dir {source} for link follow up")
Expand All @@ -491,9 +487,8 @@ def is_loop(link_name, source):
self.log_debug(f"Adding file {source} for link follow up")
self.add_file(host_path_name)
else:
self.log_debug(
f"No link follow up: source={source} link_name={link_name}"
)
self.log_debug(f"No link follow up: source={source} "
f"link_name={link_name}")

def add_dir(self, path):
"""Create a directory in the archive.
Expand Down Expand Up @@ -539,10 +534,8 @@ def makedirs(self, path, mode=0o700):
Used by sos.sosreport to set up sos_* directories.
"""
os.makedirs(os.path.join(self._archive_root, path), mode=mode)
self.log_debug(
f"created directory at '{path}' in FileCacheArchive"
f" '{self._archive_root}'"
)
self.log_debug(f"created directory at '{path}' in FileCacheArchive"
f" '{self._archive_root}'")

def open_file(self, path):
path = self.dest_path(path)
Expand Down Expand Up @@ -624,8 +617,10 @@ def finalize(self, method):
return self.name()

self.cleanup()
self.log_info("built archive at '%s' (size=%d)" % (self._archive_name,
os.stat(self._archive_name).st_size))
self.log_info(
f"built archive at '{self._archive_name}'"
f" (size={os.stat(self._archive_name).st_size})"
)

if self.enc_opts['encrypt']:
try:
Expand Down Expand Up @@ -661,10 +656,11 @@ def _encrypt(self, archive):
if self.enc_opts["password"]:
# prevent change of gpg options using a long password, but also
# prevent the addition of quote characters to the passphrase
passwd = "%s" % self.enc_opts["password"].replace('\'"', '')
replaced_password = self.enc_opts['password'].replace('\'\"', '')
passwd = f"{replaced_password}"
env = {"sos_gpg": passwd}
enc_cmd += "-c --passphrase-fd 0 "
enc_cmd = "/bin/bash -c \"echo $sos_gpg | %s\"" % enc_cmd
enc_cmd = f"/bin/bash -c \"echo $sos_gpg | {enc_cmd}\""
enc_cmd += archive
r = sos_get_command_output(enc_cmd, timeout=0, env=env)
if r["status"] == 0:
Expand Down Expand Up @@ -753,11 +749,8 @@ def _build_archive(self, method):
kwargs = {'compresslevel': 6}
else:
kwargs = {'preset': 3}
tar = tarfile.open(
self._archive_name,
mode=f"w:{_comp_mode}",
**kwargs
)
tar = tarfile.open(self._archive_name,
mode=f"w:{_comp_mode}", **kwargs)
# add commonly reviewed files first, so that they can be more easily
# read from memory without needing to extract the whole archive
for _content in ['version.txt', 'sos_reports', 'sos_logs']:
Expand Down
Loading

0 comments on commit 118087f

Please sign in to comment.