Skip to content

Commit

Permalink
Fixup based on the below PR comments.
Browse files Browse the repository at this point in the history
#2972
SAI DUMP based on the route table size

* [saidump]
• Saidump for DNX-SAI sonic-net/sonic-buildimage#13561

Solution and modification:
To use the Redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it.

(1) Updated platform/broadcom/docker-syncd-brcm-dnx/Dockerfile.j2, install Python library rdbtools into the syncd containter.
(2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format.
(3) Updated sonic-buildimage/build_debian.sh, to add a new script file: files/scripts/saidump.sh into the host. This shell file does the below steps:
  For each ASIC0, such as ASIC0,

  1. Save the Redis data.
  sudo sonic-db-cli -n asic$1 SAVE > /dev/null

  2. Move dump files to /var/run/redisX/
  docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/"

  3. Run rdb command to convert the dump files into JSON files
  docker exec syncd$1 sh -c "rdb --command json /var/run/redis$1/dump.rdb | tee /var/run/redis$1/dump.json > /dev/null"

  4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output.
  docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json -m 100"

  5. clear
  sudo rm -f /var/run/redis$1/dump.rdb
  sudo rm -f /var/run/redis$1/dump.json

(4) Update sonic-buildimage/src/sonic-utilities/scrip
  • Loading branch information
JunhongMao committed Sep 19, 2023
1 parent 2450c48 commit d7ae34d
Showing 1 changed file with 89 additions and 28 deletions.
117 changes: 89 additions & 28 deletions scripts/generate_dump
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ SKIP_BCMCMD=0
SAVE_STDERR=true
RETURN_CODE=$EXT_SUCCESS
DEBUG_DUMP=false
ROUTE_TAB_LIMIT_DIRECT_ITERATION=6400

# lock dirs/files
LOCKDIR="/tmp/techsupport-lock"
Expand Down Expand Up @@ -863,45 +864,109 @@ save_redis() {
}

###############################################################################
# SAI DUMP from syncd
# GET ROUTE table size by ASIC id and ip version
# Globals:
# NUM_ASICS
# TIMEOUT_MIN
# TIMEOUT_EXIT_CODE
# Arguments:
# None
# asic id
# IP version
# Returns:
# None
###############################################################################
save_saidump() {
trap 'handle_error $? $LINENO' ERR
if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
save_cmd "docker exec syncd saidump" "saidump"
# route
###############################################################################
get_route_table_size_by_asic_id_and_ipver() {
local asic_id="$1"
local ip_ver="$2"
local filepath="/tmp/route_summary.txt"
RC=0

if [ $ip_ver = "ipv4" ]; then
cmd="vtysh -n $asic_id -c 'show ip route summary json'"
elif [ $ip_ver = "ipv6" ]; then
cmd="vtysh -n $asic_id -c 'show ipv6 route summary json'"
else
for (( i=0; i<$NUM_ASICS; i++ ))
do
save_cmd "docker exec syncd$i saidump" "saidump$i"
done
echo "Wrong argument $ip_ver."
return 255
fi


local timeout_cmd="timeout --foreground ${TIMEOUT_MIN}m"
local cmds="$cmd > '$filepath'"

eval "${timeout_cmd} bash -c \"${cmds}\"" || RC=$?
if [ $RC -eq $TIMEOUT_EXIT_CODE ]; then
echo "Command: $cmds timedout after ${TIMEOUT_MIN} minutes."
return $RC
elif [ $RC -ne 0 ]; then
echo "Command: $cmds failed with RC $RC"
return $RC
fi

local route_tab_size=$(python3 -c "\
import json
with open('$filepath') as json_file:
data = json.load(json_file)
print(data['routesTotal'])")
rm $filepath
echo "$route_tab_size"
}

###############################################################################
# SAI DUMP from syncd by Redis Save command
# SAI DUMP based on the route table size
# if the route table has more than ROUTE_TAB_LIMIT_DIRECT_ITERATION
# then dump by Redis Save command,
# otherwize, dump it by directly iteration the Redis
#
# Globals:
# NUM_ASICS
# ROUTE_TAB_LIMIT_DIRECT_ITERATION
# Arguments:
# None
# Returns:
# None
###############################################################################
save_saidump_by_redis_save_cmd() {
save_saidump_by_route_size() {
trap 'handle_error $? $LINENO' ERR
if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
save_cmd "saidump.sh" "saidump"
else
for (( i=0; i<$NUM_ASICS; i++ ))
do
save_cmd "saidump.sh $i" "saidump$i"
done
fi

for (( i=0; i<$NUM_ASICS; i++ ))
do
route_size_ipv4=`get_route_table_size_by_asic_id_and_ipver $i ipv4`
ret=$?

if [ $ret -ne 0 ]; then
echo "Get route table's size by asicid $i and ipv4 failed."
return $ret
fi

route_size_ipv6=`get_route_table_size_by_asic_id_and_ipver $i ipv6`
ret=$?

if [ $ret -ne 0 ]; then
echo "Get route table's size by asicid $i and ipv6 failed."
return $ret
fi

route_size=`expr $route_size_ipv4 + $route_size_ipv6`
echo "The route table's size is $route_size(ipv4 $route_size_ipv4, ipv6 $route_size_ipv6)"

if [[ $route_size -gt $ROUTE_TAB_LIMIT_DIRECT_ITERATION ]]; then
echo "Dump by using Redis SAVE."

if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
save_cmd "saidump.sh" "saidump"
else
save_cmd "saidump.sh $i" "saidump$i"
fi
else
echo "Dump by using direct iteration of Redis DB."

if [[ ( "$NUM_ASICS" == 1 ) ]] ; then
save_cmd "docker exec syncd saidump" "saidump"
else
save_cmd "docker exec syncd$i saidump" "saidump$i"
fi
fi
done
}

###############################################################################
Expand Down Expand Up @@ -1810,11 +1875,7 @@ main() {
save_cmd "ps -AwwL -o user,pid,lwp,ppid,nlwp,pcpu,pri,nice,vsize,rss,tty,stat,wchan:12,start,bsdtime,command" "ps.extended" &
wait

if [[ "$device_type" != "SpineRouter" ]]; then
save_saidump
else
save_saidump_by_redis_save_cmd
fi
save_saidump_by_route_size

if [ "$asic" = "barefoot" ]; then
collect_barefoot
Expand Down

0 comments on commit d7ae34d

Please sign in to comment.