Skip to content

Commit

Permalink
[Mellanox] Improve FW upgrade logging (sonic-net#13465)
Browse files Browse the repository at this point in the history
- Why I did it
To improve ASIC FW upgrade logging and have information about the cause of FW update failure in the log.

- How I did it
Added syslog logger support

In case the FW update has failed the update tool will give the cause of the failure in the output in the last line, starting with "Fail".
When running the tool, in case of a failed update, we will parse the output to retrieve the cause and log it.

Device #1:
 ----------
 
 Device Type:      ConnectX6DX
   Part Number:      MCX623106AN-CDA_Ax
   Description:      ConnectX-6 Dx EN adapter card; 100GbE; Dual-port QSFP56; PCIe 4.0/3.0 x16;
   PSID:             MT_0000000359
   PCI Device Name:  /dev/mst/mt4125_pciconf0
   Base GUID:        0c42a103007d22d4
   Base MAC:         0c42a17d22d4
   Versions:         Current        Available     
      FW             22.32.0498     22.32.0498    
      PXE            3.6.0500       3.6.0500      
      UEFI           14.25.0015     14.25.0015    
 
 Status:           Forced update required
 
---------
 Found 1 device(s) requiring firmware update...
 
Device #1: Updating FW ...     
 FSMST_INITIALIZE -   OK          
 Writing Boot image component -   OK          
 Fail : The Digest in the signature is wrong

- How to verify it
mlnx-fw-upgrade.sh --upgrade
  • Loading branch information
DavidZagury authored Jan 25, 2023
1 parent 9a49aec commit 4cc84c6
Showing 1 changed file with 43 additions and 3 deletions.
46 changes: 43 additions & 3 deletions platform/mellanox/mlnx-fw-upgrade.j2
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ declare -rA FW_FILE_MAP=( \
)

IMAGE_UPGRADE="${NO_PARAM}"
SYSLOG_LOGGER="${NO_PARAM}"
VERBOSE_LEVEL="${VERBOSE_MIN}"

function PrintHelp() {
Expand All @@ -48,7 +49,8 @@ function PrintHelp() {
echo
echo "OPTIONS:"
echo " -u, --upgrade Upgrade ASIC firmware using next boot image (useful after SONiC-To-SONiC update)"
echo " -v, --verbose Verbose mode"
echo " -s, --syslog Use syslog logger (enabled when -u|--upgrade)"
echo " -v, --verbose Verbose mode (enabled when -u|--upgrade)"
echo " -h, --help Print help"
echo
echo "Examples:"
Expand All @@ -63,10 +65,14 @@ function ParseArguments() {
case "$1" in
-u|--upgrade)
IMAGE_UPGRADE="${YES_PARAM}"
SYSLOG_LOGGER="${YES_PARAM}"
;;
-v|--verbose)
VERBOSE_LEVEL="${VERBOSE_MAX}"
;;
-s|--syslog)
SYSLOG_LOGGER="${YES_PARAM}"
;;
-h|--help)
PrintHelp
exit "${EXIT_SUCCESS}"
Expand All @@ -79,25 +85,42 @@ function ParseArguments() {
function LogError() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_ERROR}" ]]; then
echo "ERROR: $*"
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
fi

if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "ERROR" -t "${SCRIPT_NAME}" "$*"
fi
}

function LogWarning() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_WARNING}" ]]; then
echo "WARNING: $*"
fi

if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "WARNING" -t "${SCRIPT_NAME}" "$*"
fi
}

function LogNotice() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_NOTICE}" ]]; then
echo "NOTICE: $*"
fi

if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "NOTICE" -t "${SCRIPT_NAME}" "$*"
fi
}

function LogInfo() {
if [[ "${VERBOSE_LEVEL}" -ge "${VERBOSE_INFO}" ]]; then
echo "INFO: $*"
fi

if [[ "${SYSLOG_LOGGER}" = "${YES_PARAM}" ]]; then
logger -p "INFO" -t "${SCRIPT_NAME}" "$*"
fi
}

function ExitFailure() {
Expand Down Expand Up @@ -186,6 +209,23 @@ function RunCmd() {
fi
}

function RunFwUpdateCmd() {
local ERROR_CODE="${EXIT_SUCCESS}"
local COMMAND="${BURN_CMD} $@"

if [[ "${VERBOSE_LEVEL}" -eq "${VERBOSE_MAX}" ]]; then
output=$(eval "${COMMAND}")
else
output=$(eval "${COMMAND}") >/dev/null 2>&1
fi

ERROR_CODE="$?"
if [[ "${ERROR_CODE}" != "${EXIT_SUCCESS}" ]]; then
failure_msg="${output#*Fail : }"
ExitFailure "FW Update command: ${COMMAND} failed with error: ${failure_msg}"
fi
}

function UpgradeFW() {
local -r _FS_MOUNTPOINT="$1"

Expand Down Expand Up @@ -229,9 +269,9 @@ function UpgradeFW() {
local -r _MST_DEVICE="$(GetMstDevice)"
if [[ "${_MST_DEVICE}" = "${UNKN_MST}" ]]; then
LogWarning "could not find fastest mst device, using default device"
RunCmd "${BURN_CMD} -i ${_FW_FILE}"
RunFwUpdateCmd "-i ${_FW_FILE}"
else
RunCmd "${BURN_CMD} -d ${_MST_DEVICE} -i ${_FW_FILE}"
RunFwUpdateCmd "-d ${_MST_DEVICE} -i ${_FW_FILE}"
fi
fi
}
Expand Down

0 comments on commit 4cc84c6

Please sign in to comment.