Skip to content

Commit

Permalink
Multi DB with namespace support, Introducing the database_global.json… (
Browse files Browse the repository at this point in the history
#4477)

* Multi DB with namespace support, Introducing the database_global.json file
for supporting accessing DB's in other namespaces for service running in
linux host

* Updates based on comments

* Adding the j2 templates for database_config and database_global files.

* Updating to retrieve the redis DIR's to be mounted from database_global.json file.

* Additional check to see if asic.conf file exists before sourcing it.

* Updates based on PR comments discussion.

* Review comments update

* Updates to the argument "-n" for namespace used in both context of parsing minigraph and multi DB access.

* Update with the attribute "persistence_for_warm_boot" that was added to database_config.json file earlier.

* Removing the database_config.json file to avioid confusion in future.
We use the database_config.json.j2 file to generate database_config.json files dynamically.

* Update the comments for sudo usage in docker_image_ctrl.j2

* Update with the new logic in PING PONG tests using sonic-db-cli. With this we wait till the
PONG response is received when redis server is up.

* Similar changes in swss and syncd scripts for the PING tests with sonic-db-cli

* Updated with a missing , in the database_config.json.j2 file, Do pip install of j2cli in docker-base-buster.
  • Loading branch information
judyjoseph authored and abdosi committed May 10, 2020
1 parent 87f3592 commit c808640
Show file tree
Hide file tree
Showing 9 changed files with 172 additions and 45 deletions.
4 changes: 4 additions & 0 deletions dockers/docker-base-stretch/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ RUN apt-get update && \
perl \
procps \
python \
python-pip \
rsyslog \
vim-tiny \
# Install dependencies of supervisor
Expand All @@ -63,6 +64,9 @@ RUN apt-get -y install \
net-tools
{% endif %}

# For templating
RUN pip install j2cli

RUN mkdir -p /etc/supervisor /var/log/supervisor

RUN apt-get -y purge \
Expand Down
3 changes: 2 additions & 1 deletion dockers/docker-database/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ RUN apt-get clean -y && \

COPY ["supervisord.conf.j2", "/usr/share/sonic/templates/"]
COPY ["docker-database-init.sh", "/usr/local/bin/"]
COPY ["database_config.json", "/etc/default/sonic-db/"]
COPY ["database_config.json.j2", "/usr/share/sonic/templates/"]
COPY ["database_global.json.j2", "/usr/share/sonic/templates/"]
COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
COPY ["critical_processes", "/etc/supervisor"]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"redis":{
"hostname" : "127.0.0.1",
"port" : 6379,
"unix_socket_path" : "/var/run/redis/redis.sock"
"unix_socket_path" : "/var/run/redis{{NAMESPACE_ID}}/redis.sock",
"persistence_for_warm_boot" : "yes"
}
},
"DATABASES" : {
Expand Down
21 changes: 21 additions & 0 deletions dockers/docker-database/database_global.json.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{% set namespace_cnt = NAMESPACE_COUNT|int %}
{
"INCLUDES" : [
{
"include" : "../../redis/sonic-db/database_config.json"
},
{% if namespace_cnt > 1 %}
{% for ns in range(namespace_cnt) %}
{
"namespace" : "{{NAMESPACE_PREFIX}}{{ns}}",
"include" : "../../redis{{ns}}/sonic-db/database_config.json"
{% if ns == namespace_cnt-1 %}
}
{% else %}
},
{% endif %}
{% endfor %}
],
"VERSION" : "1.0"
}
{% endif %}
21 changes: 17 additions & 4 deletions dockers/docker-database/docker-database-init.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,26 @@
#!/usr/bin/env bash

mkdir -p /var/run/redis/sonic-db
if [ -f /etc/sonic/database_config.json ]; then
cp /etc/sonic/database_config.json /var/run/redis/sonic-db
REDIS_DIR=/var/run/redis$NAMESPACE_ID
mkdir -p $REDIS_DIR/sonic-db

if [ -f /etc/sonic/database_config$NAMESPACE_ID.json ]; then
cp /etc/sonic/database_config$NAMESPACE_ID.json $REDIS_DIR/sonic-db/database_config.json
else
cp /etc/default/sonic-db/database_config.json /var/run/redis/sonic-db
j2 /usr/share/sonic/templates/database_config.json.j2 > $REDIS_DIR/sonic-db/database_config.json
fi

mkdir -p /etc/supervisor/conf.d/

# copy/generate the database_global.json file if this is global database service in multi asic platform.
if [[ $NAMESPACE_ID == "" ]] && [[ $NAMESPACE_COUNT -gt 1 ]]
then
if [ -f /etc/sonic/database_global.json ]; then
cp /etc/sonic/database_global.json $REDIS_DIR/sonic-db/database_global.json
else
j2 /usr/share/sonic/templates/database_global.json.j2 > $REDIS_DIR/sonic-db/database_global.json
fi
fi

# generate all redis server supervisord configuration file
sonic-cfggen -j /var/run/redis/sonic-db/database_config.json -t /usr/share/sonic/templates/supervisord.conf.j2 > /etc/supervisor/conf.d/supervisord.conf

Expand Down
101 changes: 86 additions & 15 deletions files/build_templates/docker_image_ctl.j2
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ function preStartAction()
docker cp /tmp/dump.rdb database$DEV:/var/lib/redis/
fi
{%- elif docker_container_name == "snmp" %}
sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB HSET 'DEVICE_METADATA|localhost' chassis_serial_number $(decode-syseeprom -s)
$SONIC_DB_CLI STATE_DB HSET 'DEVICE_METADATA|localhost' chassis_serial_number $(decode-syseeprom -s)
{%- else %}
: # nothing
{%- endif %}
Expand All @@ -77,9 +77,9 @@ function postStartAction()
docker exec -i database$DEV sysctl -w net.ipv6.conf.all.disable_ipv6=0
link_namespace $DEV
fi

# Wait until redis starts
# TODO: should use $SONIC_DB_CLI if Judy's PR 4477 is in first, otherwise PR 4477 should change this part
until [[ $(/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli PING | grep -c PONG) -gt 0 ]]; do
until [[ $($SONIC_DB_CLI PING | grep -c PONG) -gt 0 ]]; do
sleep 1;
done

Expand All @@ -89,23 +89,25 @@ function postStartAction()
# If there is a config_db.json dump file, load it.
if [ -r /etc/sonic/config_db$DEV.json ]; then
if [ -r /etc/sonic/init_cfg.json ]; then
sonic-netns-exec "$NET_NS" sonic-cfggen -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db$DEV.json --write-to-db
$SONIC_CFGGEN -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db$DEV.json --write-to-db
else
sonic-netns-exec "$NET_NS" sonic-cfggen -j /etc/sonic/config_db$DEV.json --write-to-db
$SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db
fi
fi

if [[ "$BOOT_TYPE" == "fast" ]]; then
# set the key to expire in 3 minutes
/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
$SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
fi

/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
$SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
fi

if [[ -x /usr/bin/db_migrator.py ]]; then
# Migrate the DB to the latest schema version if needed
/usr/bin/db_migrator.py -o migrate
if [ -z "$DEV" ]; then
/usr/bin/db_migrator.py -o migrate
fi
fi
{%- elif docker_container_name == "swss" %}
docker exec swss$DEV rm -f /ready # remove cruft
Expand Down Expand Up @@ -137,14 +139,20 @@ start() {
BOOT_TYPE=`getBootType`

# Obtain our platform as we will mount directories with these names in each docker
PLATFORM=`sonic-netns-exec "$NET_NS" sonic-cfggen -H -v DEVICE_METADATA.localhost.platform`
PLATFORM=`$SONIC_CFGGEN -H -v DEVICE_METADATA.localhost.platform`

# Parse the device specific asic conf file, if it exists
ASIC_CONF=/usr/share/sonic/device/$PLATFORM/asic.conf
if [ -f "$ASIC_CONF" ]; then
source $ASIC_CONF
fi

{%- if docker_container_name == "database" %}
# Don't mount HWSKU in {{docker_container_name}} container.
HWSKU=""
{%- else %}
# Obtain our HWSKU as we will mount directories with these names in each docker
HWSKU=`sonic-netns-exec "$NET_NS" sonic-cfggen -d -v 'DEVICE_METADATA["localhost"]["hwsku"]'`
HWSKU=`$SONIC_CFGGEN -d -v 'DEVICE_METADATA["localhost"]["hwsku"]'`
{%- endif %}

DOCKERCHECK=`docker inspect --type container {{docker_container_name}}$DEV 2>/dev/null`
Expand Down Expand Up @@ -173,18 +181,64 @@ start() {

{%- if docker_container_name == "database" %}
echo "Creating new {{docker_container_name}}$DEV container"
if [ -z "$DEV" ]; then
# if database_global exists in old_config, use it; otherwise use the default one in new image
if [ -f /etc/sonic/old_config/database_global.json ]; then
echo "Use database_global.json from old system..."
mv /etc/sonic/old_config/database_global.json /etc/sonic/
fi
fi
# if database_config exists in old_config, use it; otherwise use the default one in new image
if [ -f /etc/sonic/old_config/database_config.json ]; then
if [ -f /etc/sonic/old_config/database_config$DEV.json ]; then
echo "Use database_config.json from old system..."
mv /etc/sonic/old_config/database_config.json /etc/sonic/
mv /etc/sonic/old_config/database_config$DEV.json /etc/sonic/
fi
{%- else %}
echo "Creating new {{docker_container_name}}$DEV container with HWSKU $HWSKU"
{%- endif %}

# In Multi ASIC platforms the global database config file database_global.json will exist.
# Parse the file and get the include path for the database_config.json files used in
# various namesapces. The database_config paths are relative to the DIR of SONIC_DB_GLOBAL_JSON.
SONIC_DB_GLOBAL_JSON="/var/run/redis/sonic-db/database_global.json"
if [ -f "$SONIC_DB_GLOBAL_JSON" ]; then
# TODO Create a separate python script with the below logic and invoke it here.
redis_dir_list=`/usr/bin/python -c "import sys; import os; import json; f=open(sys.argv[1]); \
global_db_dir = os.path.dirname(sys.argv[1]); data=json.load(f); \
print(\" \".join([os.path.normpath(global_db_dir+'/'+elem['include']).partition('sonic-db')[0]\
for elem in data['INCLUDES'] if 'namespace' in elem])); f.close()" $SONIC_DB_GLOBAL_JSON`
fi

if [ -z "$DEV" ]; then
NET="host"

# For Multi-ASIC platform we have to mount the redis paths for database instances running in different
# namespaces, into the single instance dockers like snmp, pmon on linux host. These global dockers
# will need to get/set tables from databases in different namespaces.
# /var/run/redis0 ---> mounted as --> /var/run/redis0
# /var/run/redis1 ---> mounted as --> /var/run/redis1 .. etc
# The below logic extracts the base DIR's where database_config.json's for various namespaces exist.
# redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"

{%- if docker_container_name != "database" %}
if [ -n "$redis_dir_list" ]; then
for redis_dir in $redis_dir_list
do
REDIS_MNT=$REDIS_MNT" -v $redis_dir:$redis_dir:rw "
done
fi
{%- endif %}
else
# This part of code is applicable for Multi-ASIC platforms. Here we mount the namespace specific
# redis directory into the docker running in that namespace. Below eg: is for namespace "asic1"
# /var/run/redis1 ---> mounted as --> /var/run/redis1
# redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"
if [ -n "$redis_dir_list" ]; then
id=`expr $DEV + 1`
redis_dir=`echo $redis_dir_list | cut -d " " -f $id`
REDIS_MNT=" -v $redis_dir:$redis_dir:rw "
fi

{%- if docker_container_name == "database" %}
NET="bridge"
{%- else %}
Expand Down Expand Up @@ -232,6 +286,7 @@ start() {
-v /etc/sonic/frr/$DEV:/etc/frr:rw \
{%- endif %}
-v /var/run/redis$DEV:/var/run/redis:rw \
$REDIS_MNT \
-v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \
{%- if docker_container_name != "database" %}
-v /usr/share/sonic/device/$PLATFORM/$HWSKU/$DEV:/usr/share/sonic/hwsku:ro \
Expand All @@ -240,6 +295,11 @@ start() {
--tmpfs /tmp \
{%- endif %}
--tmpfs /var/tmp \
{%- if docker_container_name == "database" %}
--env "NAMESPACE_ID"="$DEV" \
--env "NAMESPACE_PREFIX"="$NAMESPACE_PREFIX" \
--env "NAMESPACE_COUNT"=$NUM_ASIC \
{%- endif %}
--name={{docker_container_name}}$DEV {{docker_image_name}}:latest || {
echo "Failed to docker run" >&1
exit 4
Expand All @@ -265,10 +325,21 @@ stop() {

OP=$1
DEV=$2 # namespace/device number to operate on
NAMESPACE_PREFIX="asic"
if [ "$DEV" ]; then
NET_NS="asic$DEV" #name of the network namespace
else
NET_NS=""
NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace

# While using -n (namespace) argument, sonic-cfggen/sonic-db-cli uses redis UNIX socket
# for accessing redis DB in a namespace. This unix socket has permission restrictions since
# it is created by systemd database.servce started with [User] as [root].
# sudo is needed here for services which are started by systemd with [User] as [admin]
# and needs to override this unix socket permission restrictions.
SONIC_CFGGEN="sudo sonic-cfggen -n $NET_NS"
SONIC_DB_CLI="sudo sonic-db-cli -n $NET_NS"
else
NET_NS=""
SONIC_CFGGEN="sonic-cfggen"
SONIC_DB_CLI="sonic-db-cli"
fi

case "$1" in
Expand Down
28 changes: 15 additions & 13 deletions files/scripts/swss.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ function unlock_service_state_change()

function check_warm_boot()
{
SYSTEM_WARM_START=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
SERVICE_WARM_START=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable`
SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable`
if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then
WARM_BOOT="true"
else
Expand All @@ -40,7 +40,7 @@ function check_warm_boot()
function validate_restore_count()
{
if [[ x"$WARM_BOOT" == x"true" ]]; then
RESTORE_COUNT=`sonic-netns-exec "$NET_NS" sonic-db-cli STATE_DB hget "WARM_RESTART_TABLE|orchagent" restore_count`
RESTORE_COUNT=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_TABLE|orchagent" restore_count`
# We have to make sure db data has not been flushed.
if [[ -z "$RESTORE_COUNT" ]]; then
WARM_BOOT="false"
Expand All @@ -51,13 +51,12 @@ function validate_restore_count()
function wait_for_database_service()
{
# Wait for redis server start before database clean
# TODO: should use $SONIC_DB_CLI if Judy's PR 4477 is in first, otherwise PR 4477 should change this part
until [[ $(/usr/bin/sonic-netns-exec "$NET_NS" sonic-db-cli PING | grep -c PONG) -gt 0 ]]; do
until [[ $($SONIC_DB_CLI PING | grep -c PONG) -gt 0 ]]; do
sleep 1;
done

# Wait for configDB initialization
until [[ $(sonic-netns-exec "$NET_NS" sonic-db-cli CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
until [[ $($SONIC_DB_CLI CONFIG_DB GET "CONFIG_DB_INITIALIZED") ]];
do sleep 1;
done
}
Expand All @@ -67,7 +66,7 @@ function wait_for_database_service()
# $2 the string of a list of table prefixes
function clean_up_tables()
{
sonic-netns-exec "$NET_NS" sonic-db-cli $1 EVAL "
$SONIC_DB_CLI $1 EVAL "
local tables = {$2}
for i = 1, table.getn(tables) do
local matches = redis.call('KEYS', tables[i])
Expand Down Expand Up @@ -135,10 +134,10 @@ start() {
# Don't flush DB during warm boot
if [[ x"$WARM_BOOT" != x"true" ]]; then
debug "Flushing APP, ASIC, COUNTER, CONFIG, and partial STATE databases ..."
sonic-netns-exec "$NET_NS" sonic-db-cli APPL_DB FLUSHDB
sonic-netns-exec "$NET_NS" sonic-db-cli ASIC_DB FLUSHDB
sonic-netns-exec "$NET_NS" sonic-db-cli COUNTERS_DB FLUSHDB
sonic-netns-exec "$NET_NS" sonic-db-cli FLEX_COUNTER_DB FLUSHDB
$SONIC_DB_CLI APPL_DB FLUSHDB
$SONIC_DB_CLI ASIC_DB FLUSHDB
$SONIC_DB_CLI COUNTERS_DB FLUSHDB
$SONIC_DB_CLI FLEX_COUNTER_DB FLUSHDB
clean_up_tables STATE_DB "'PORT_TABLE*', 'MGMT_PORT_TABLE*', 'VLAN_TABLE*', 'VLAN_MEMBER_TABLE*', 'LAG_TABLE*', 'LAG_MEMBER_TABLE*', 'INTERFACE_TABLE*', 'MIRROR_SESSION*', 'VRF_TABLE*', 'FDB_TABLE*'"
fi

Expand Down Expand Up @@ -209,10 +208,13 @@ SERVICE="swss"
PEER="syncd"
DEBUGLOG="/tmp/swss-syncd-debug$DEV.log"
LOCKFILE="/tmp/swss-syncd-lock$DEV"
NAMESPACE_PREFIX="asic"
if [ "$DEV" ]; then
NET_NS="asic$DEV" #name of the network namespace
NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
else
NET_NS=""
NET_NS=""
SONIC_DB_CLI="sonic-db-cli"
fi

case "$1" in
Expand Down
Loading

0 comments on commit c808640

Please sign in to comment.