Skip to content

Commit

Permalink
[generate_dump] remove secrets from dump files (sonic-net#1886)
Browse files Browse the repository at this point in the history 
Remove secrets from dump files.

What I did
    Add bash functions to remove secrets from dump files.
How I did it
    For tacacs key, radius key, snmp community srring, use sed command with regex to remove user secrets from dump files.
    For certs, update tar command exclude list to remove those certs from dump file.
How to verify it
    Run 'show techsupport' command and check secrets removed from dump files.
Previous command output (if the output of a command-line utility has changed)
New command output (if the output of a command-line utility has changed)
  • Loading branch information
@liuh-80
liuh-80 authored Oct 28, 2021
1 parent 3a8ab73 commit 9f123c0
Showing 1 changed file with 91 additions and 8 deletions.
99 changes: 91 additions & 8 deletions scripts/generate_dump
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ SAVE_STDERR=true
RETURN_CODE=0
DEBUG_DUMP=false


handle_signal()
{
echo "Generate Dump received interrupt" >&2
Expand Down Expand Up @@ -155,6 +154,7 @@ save_bcmcmd_all_ns() {
# cmd: The command to run. Make sure that arguments with spaces have quotes
# filename: the filename to save the output as in $BASE/dump
# do_gzip: (OPTIONAL) true or false. Should the output be gzipped
# cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
# Returns:
# None
###############################################################################
Expand All @@ -168,6 +168,7 @@ save_cmd() {
local do_gzip=${3:-false}
local tarpath="${BASE}/dump/$filename"
local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m"
local cleanup_method=${4:-dummy_cleanup_method}
local redirect='&>'
local redirect_eval='2>&1'
if [ ! -d $LOGDIR ]; then
Expand All @@ -187,7 +188,9 @@ save_cmd() {
if $do_gzip; then
tarpath="${tarpath}.gz"
filepath="${filepath}.gz"
local cmds="$cmd $redirect_eval | gzip -c > '${filepath}'"
# cleanup_method will run in a sub-shell, need declare it first
local cleanup_method_declration=$(declare -f $cleanup_method)
local cmds="$cleanup_method_declration; $cmd $redirect_eval | $cleanup_method | gzip -c > '${filepath}'"
if $NOOP; then
echo "${timeout_cmd} bash -c \"${cmds}\""
else
Expand All @@ -199,22 +202,36 @@ save_cmd() {
fi
else
if $NOOP; then
echo "${timeout_cmd} $cmd $redirect '$filepath'"
echo "${timeout_cmd} $cmd | $cleanup_method $redirect '$filepath'"
else
RC=0
eval "${timeout_cmd} $cmd" "$redirect" "$filepath" || RC=$?
eval "${timeout_cmd} $cmd | $cleanup_method" "$redirect" "$filepath" || RC=$?
if [ $RC -ne 0 ]; then
echo "Command: $cmd timedout after ${TIMEOUT_MIN} minutes."
fi
fi
fi

($TAR $V -rhf $TARFILE -C $DUMPDIR "$tarpath" \
|| abort "${ERROR_TAR_FAILED}" "tar append operation failed. Aborting to prevent data loss.") \
&& $RM $V -rf "$filepath"
end_t=$(date +%s%3N)
echo "[ save_cmd:$cmd ] : $(($end_t-$start_t)) msec" >> $TECHSUPPORT_TIME_INFO
}

###############################################################################
# Dummy cleanup method.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
###############################################################################
dummy_cleanup_method() {
cat
}

###############################################################################
# Runs a given command in all namesapces in case of multi ASIC platform, in
# default (host) namespace in single ASIC platform
Expand All @@ -224,22 +241,24 @@ save_cmd() {
# cmd: The command to run. Make sure that arguments with spaces have quotes
# filename: the filename to save the output as in $BASE/dump
# do_gzip: (OPTIONAL) true or false. Should the output be gzipped
# cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
# Returns:
# None
###############################################################################
save_cmd_all_ns() {
trap 'handle_error $? $LINENO' ERR
local do_zip=${3:-false}
local cleanup_method=${4:-dummy_cleanup_method}

# host or default namespace
save_cmd "$1" "$2" "$do_zip"
save_cmd "$1" "$2" "$do_zip" $cleanup_method

if [[ ( "$NUM_ASICS" > 1 ) ]] ; then
for (( i=0; i<$NUM_ASICS; i++ ))
do
local cmd="sonic-netns-exec asic$i $1"
local file="$2.$i"
save_cmd "$cmd" "$file" "$do_zip"
save_cmd "$cmd" "$file" "$do_zip" $cleanup_method
done
fi
}
Expand Down Expand Up @@ -591,7 +610,8 @@ save_redis_info() {
save_redis "APPL_DB"
save_redis "ASIC_DB"
save_redis "COUNTERS_DB"
save_redis "CONFIG_DB"
# There are secrets in CONFIG_DB need to be cleanup.
save_redis "CONFIG_DB" "CONFIG_DB" remove_secret_from_config_db_dump
save_redis "FLEX_COUNTER_DB"
save_redis "STATE_DB"
}
Expand Down Expand Up @@ -637,18 +657,20 @@ save_proc() {
# Arguments:
# DB name: DB name
# Filename: Destination filename, if not given then filename would be DB name
# cleanup_method: (OPTIONAL) the cleanup method to procress dump file after it generated.
# Returns:
# None
###############################################################################
save_redis() {
local cleanup_method=${3:-dummy_cleanup_method}
trap 'handle_error $? $LINENO' ERR
local db_name=$1
if [ $# -ge 2 ] && [ -n "$2" ]; then
local dest_file_name=$2
else
local dest_file_name="$db_name"
fi
save_cmd_all_ns "sonic-db-dump -n '$db_name' -y" "$dest_file_name.json"
save_cmd_all_ns "sonic-db-dump -n '$db_name' -y" "$dest_file_name.json" false $cleanup_method
}

###############################################################################
Expand Down Expand Up @@ -1259,6 +1281,9 @@ main() {
rm $rm_list
fi

# Remove secret from /etc files before tar
remove_secret_from_etc_files $TARDIR

start_t=$(date +%s%3N)
($TAR $V --warning=no-file-removed -rhf $TARFILE -C $DUMPDIR --mode=+rw \
--exclude="etc/alternatives" \
Expand All @@ -1271,6 +1296,13 @@ main() {
--exclude="*snmpd.conf*" \
--exclude="/etc/mlnx" \
--exclude="/etc/mft" \
--exclude="*/etc/sonic/*.cer" \
--exclude="*/etc/sonic/*.crt" \
--exclude="*/etc/sonic/*.pem" \
--exclude="*/etc/sonic/*.key" \
--exclude="*/etc/ssl/*.pem" \
--exclude="*/etc/ssl/certs/*" \
--exclude="*/etc/ssl/private/*" \
$BASE/etc \
|| abort "${ERROR_TAR_FAILED}" "Tar append operation failed. Aborting for safety.") \
&& $RM $V -rf $TARDIR
Expand Down Expand Up @@ -1310,6 +1342,57 @@ main() {
exit $RETURN_CODE
}

###############################################################################
# Remove secret from pipeline inout and output result to pipeline.
# Globals:
# None
# Arguments:
# None
# Returns:
# None
###############################################################################
remove_secret_from_config_db_dump() {
# Remove tacacs & radius passkey and snmp community from config DB
sed -E 's/\"passkey\"\s*:\s*\"([^\"]*)\"/\"passkey\":\"****\"/g; /SNMP_COMMUNITY/,/\s{2,4}\},/d'
}

###############################################################################
# Remove secret from dump files.
# Globals:
# Arguments:
# dumppath: the dump file path.
# Returns:
# None
###############################################################################
remove_secret_from_etc_files() {
local dumppath=$1
echo "Remove secret from etc files."
# Remove tacacs passkey from tacplus_nss.conf
local secret_regex='s/(secret=)([^,|\S]*)(.*)/\1****\3/g'
sed -i -E $secret_regex $dumppath/etc/tacplus_nss.conf

# Remove radius passkey from radius_nss.conf
sed -i -E $secret_regex $dumppath/etc/radius_nss.conf

# Remove tacacs passkey from common-auth-sonic
sed -i -E 's/(secret=)(\S*)/\1****/g' $dumppath/etc/pam.d/common-auth-sonic

# Remove tacacs passkey from pam_radius_auth.conf
sed -i -E 's/^([^#]\S*\s*)(\S*)/\1****/g' $dumppath/etc/pam_radius_auth.conf

# Remove radius passkey from per-server conf file /etc/pam_radius_auth.d/{ip}_{port}.conf
for filename in $dumppath/etc/pam_radius_auth.d/*.conf; do
sed -i -E 's/^([^#]\S*\s*)(\S*)/\1****/g' $filename
done

# Remove snmp community string from snmp.yml
sed -i -E 's/(\s*snmp_\S*community\s*:\s*)(\S*)/\1****/g' $dumppath/etc/sonic/snmp.yml

# Remove secret from /etc/sonic/config_db.json
cat $dumppath/etc/sonic/config_db.json | remove_secret_from_config_db_dump > $dumppath/etc/sonic/config_db.json.temp
mv $dumppath/etc/sonic/config_db.json.temp $dumppath/etc/sonic/config_db.json
}

###############################################################################
# Terminates generate_dump early just in case we have issues.
# Globals:
Expand Down

0 comments on commit 9f123c0

Please sign in to comment.