Skip to content

Commit

Permalink
[memory_checker] Do not check memory usage of containers which are no…
Browse files Browse the repository at this point in the history
…t created (#11129)

Signed-off-by: Yong Zhao yozhao@microsoft.com

Why I did it
This PR aims to fix an issue (#10088) by enhancing the script memory_checker.

Specifically, if container is not created successfully during device is booted/rebooted, then memory_checker do not need check its memory usage.

How I did it
In the script memory_checker, a function is added to get names of running containers. If the specified container name is not in current running container list, then this script will exit without checking its memory usage.

How to verify it
I tested on a lab device by following the steps:

Stops telemetry container with command sudo systemctl stop telemetry.service

Removes telemetry container with command docker rm telemetry

Checks whether the script memory_checker ran by Monit will generate the syslog message saying it will exit without checking memory usage of telemetry.
  • Loading branch information
yozhao101 authored and yxieca committed Jun 19, 2022
1 parent 36b54da commit d63d16b
Showing 1 changed file with 32 additions and 2 deletions.
34 changes: 32 additions & 2 deletions files/image_config/monit/memory_checker
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ import sys
import syslog
import re

import docker


def get_command_result(command):
"""Executes the command and return the resulting output.
Expand Down Expand Up @@ -86,14 +88,36 @@ def check_memory_usage(container_name, threshold_value):
print("[{}]: Memory usage ({} Bytes) is larger than the threshold ({} Bytes)!"
.format(container_name, mem_usage_bytes, threshold_value))
syslog.syslog(syslog.LOG_INFO, "[{}]: Memory usage ({} Bytes) is larger than the threshold ({} Bytes)!"
.format(container_name, mem_usage_bytes, threshold_value))
.format(container_name, mem_usage_bytes, threshold_value))
sys.exit(3)
else:
syslog.syslog(syslog.LOG_ERR, "[memory_checker] Failed to retrieve memory value from '{}'"
.format(mem_usage))
sys.exit(4)


def get_running_container_names():
"""Retrieves names of running containers by talking to the docker daemon.
Args:
None.
Returns:
running_container_names: A list indicates names of running containers.
"""
try:
docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock')
running_container_list = docker_client.containers.list(filters={"status": "running"})
running_container_names = [ container.name for container in running_container_list ]
except (docker.errors.APIError, docker.errors.DockerException) as err:
syslog.syslog(syslog.LOG_ERR,
"Failed to retrieve the running container list from docker daemon! Error message is: '{}'"
.format(err))
sys.exit(5)

return running_container_names


def main():
parser = argparse.ArgumentParser(description="Check memory usage of a container \
and an alerting message will be written into syslog if memory usage \
Expand All @@ -104,7 +128,13 @@ def main():
parser.add_argument("threshold_value", type=int, help="threshold value in bytes")
args = parser.parse_args()

check_memory_usage(args.container_name, args.threshold_value)
running_container_names = get_running_container_names()
if args.container_name in running_container_names:
check_memory_usage(args.container_name, args.threshold_value)
else:
syslog.syslog(syslog.LOG_INFO,
"[memory_checker] Exits without checking memory usage since container '{}' is not running!"
.format(args.container_name))


if __name__ == "__main__":
Expand Down

0 comments on commit d63d16b

Please sign in to comment.