Skip to content

Commit

Permalink
handle json load exception in bgpmon (#17856)
Browse files Browse the repository at this point in the history
Why I did it
ICM reported due to "BGPMon Process exited" which was caused by json load exception.

Work item tracking
Microsoft ADO (number only):
25916773
How I did it
Add an exception handle during json load.

How to verify it
Verified locally, add debug log to modify the output string of cmd to make it not with json formation, then check the syslog.
  • Loading branch information
lipxu authored Jan 29, 2024
1 parent 5516381 commit c23c8af
Showing 1 changed file with 39 additions and 12 deletions.
51 changes: 39 additions & 12 deletions src/sonic-bgpcfgd/bgpmon/bgpmon.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ def __init__(self):
self.db.connect(self.db.STATE_DB, False)
self.pipe = swsscommon.RedisPipeline(self.db.get_redis_client(self.db.STATE_DB))
self.db.delete_all_by_pattern(self.db.STATE_DB, "NEIGH_STATE_TABLE|*" )
self.MAX_RETRY_ATTEMPTS = 3

# A quick way to check if there are anything happening within BGP is to
# check its log file has any activities. This is by checking its modified
Expand Down Expand Up @@ -77,18 +78,44 @@ def update_new_peer_states(self, peer_dict):
# Get a new snapshot of BGP neighbors and store them in the "new" location
def get_all_neigh_states(self):
cmd = ["vtysh", "-c", 'show bgp summary json']
rc, output = getstatusoutput_noshell(cmd)
if rc:
syslog.syslog(syslog.LOG_ERR, "*ERROR* Failed with rc:{} when execute: {}".format(rc, cmd))
return

peer_info = json.loads(output)
# cmd ran successfully, safe to Clean the "new" set/dict for new snapshot
self.new_peer_l.clear()
self.new_peer_state.clear()
for key, value in peer_info.items():
if key == "ipv4Unicast" or key == "ipv6Unicast":
self.update_new_peer_states(value)
retry_attempt = 0

while retry_attempt < self.MAX_RETRY_ATTEMPTS:
try:
rc, output = getstatusoutput_noshell(cmd)
if rc:
syslog.syslog(syslog.LOG_ERR, "*ERROR* Failed with rc:{} when execute: {}".format(rc, cmd))
return
if len(output) == 0:
syslog.syslog(syslog.LOG_WARNING, "*WARNING* output none when execute: {}".format(cmd))
return

peer_info = json.loads(output)
# cmd ran successfully, safe to Clean the "new" set/dict for new snapshot
self.new_peer_l.clear()
self.new_peer_state.clear()
for key, value in peer_info.items():
if key == "ipv4Unicast" or key == "ipv6Unicast":
self.update_new_peer_states(value)
return

except json.JSONDecodeError as decode_error:
# Log the exception and retry if within the maximum attempts
retry_attempt += 1
syslog.syslog(syslog.LOG_WARNING, "*WARNING* JSONDecodeError: {} when execute: {} Retry attempt: {}".format(decode_error, cmd, retry_attempt))
time.sleep(1)
continue
except Exception as e:
# Log other exceptions and return failure
retry_attempt += 1
syslog.syslog(syslog.LOG_WARNING, "*WARNING* An unexpected error occurred: {} when execute: {} Retry attempt: {}".format(e, cmd, retry_attempt))
time.sleep(1)
continue

# Log an error if the maximum retry attempts are reached
syslog.syslog(syslog.LOG_ERR, "*ERROR* Maximum retry attempts reached. Failed to execute: {} Output: {}".format(cmd, output))
sys.exit(1)


# This method will take the caller's dictionary which contains the peer state operation
# That need to be updated in StateDB using Redis pipeline.
Expand Down

0 comments on commit c23c8af

Please sign in to comment.