Fix race condition on seeding database on startup. Improve startup logs.

If nginx got started before MongoDB/Mora was fully responsive, there was
a possibility seeding the initial database data would fail. This
improves the seeding process by waiting for MongoDB to be fully up
before proceeding with the seeding. Seeding will be re-attempted until
it succeeds.

Some related improvements were made to the similar Elasticsearch setup
process. We were already waiting there to ensure Elasticsearch was up
before running setup, but if Elasticsearch timed out, then setup wasn't
re-attempted for another 1 hour. This improves things so that setup is
reattempted on startup until it succeeds.

This also quiets the startup logs by only logging events when MongoDB or
Elasticsearch aren't ready after the 60 second timeout period (rather
than on each connection attempt).

src/api-umbrella/proxy/jobs/elasticsearch_setup.lua

@@ -17,7 +17,7 @@ local function wait_for_elasticsearch()
   repeat
     local res, err = httpc:request_uri(elasticsearch_host .. "/_cluster/health")
     if err then
-      ngx.log(ngx.ERR, "failed to fetch cluster health from elasticsearch: ", err)
+      ngx.log(ngx.NOTICE, "failed to fetch cluster health from elasticsearch (this is expected if elasticsearch is starting up at the same time): ", err)
     elseif res.body then
       local elasticsearch_health = cjson.decode(res.body)
       if elasticsearch_health["status"] == "yellow" or elasticsearch_health["status"] == "green" then
@@ -30,6 +30,12 @@ local function wait_for_elasticsearch()
       wait_time = wait_time + sleep_time
     end
   until elasticsearch_alive or wait_time > max_time
+
+  if elasticsearch_alive then
+    return true, nil
+  else
+    return false, "elasticsearch was not ready within " .. max_time  .."s"
+  end
 end
 
 local function create_templates()
@@ -101,14 +107,20 @@ local function create_aliases()
   end
 end
 
-local function do_check()
-  wait_for_elasticsearch()
-  create_templates()
-  create_aliases()
+local function setup()
+  local _, err = wait_for_elasticsearch()
+  if not err then
+    create_templates()
+    create_aliases()
+  else
+    ngx.log(ngx.ERR, "timed out waiting for eleasticsearch before setup, rerunning...")
+    ngx.sleep(5)
+    setup()
+  end
 end
 
 function _M.spawn()
-  interval_lock.repeat_with_mutex('elasticsearch_index_setup', delay, do_check)
+  interval_lock.repeat_with_mutex('elasticsearch_index_setup', delay, setup)
 end
 
 return _M

src/api-umbrella/proxy/startup/seed_database.lua

@@ -1,11 +1,37 @@
 local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays"
-local lock = require "resty.lock"
+local interval_lock = require "api-umbrella.utils.interval_lock"
 local mongo = require "api-umbrella.utils.mongo"
 local random_token = require "api-umbrella.utils.random_token"
 local uuid = require "resty.uuid"
 
 local nowMongoDate = { ["$date"] = { ["$numberLong"] = tostring(os.time() * 1000) } }
 
+local function wait_for_mongodb()
+  local mongodb_alive = false
+  local wait_time = 0
+  local sleep_time = 0.5
+  local max_time = 14
+  repeat
+    local _, err = mongo.collections()
+    if err then
+      ngx.log(ngx.NOTICE, "failed to establish connection to mongodb (this is expected if mongodb is starting up at the same time): ", err)
+    else
+      mongodb_alive = true
+    end
+
+    if not mongodb_alive then
+      ngx.sleep(sleep_time)
+      wait_time = wait_time + sleep_time
+    end
+  until mongodb_alive or wait_time > max_time
+
+  if mongodb_alive then
+    return true, nil
+  else
+    return false, "elasticsearch was not ready within " .. max_time  .."s"
+  end
+end
+
 local function seed_api_keys()
   local keys = {
     -- static.site.ajax@internal.apiumbrella
@@ -208,19 +234,24 @@ local function seed_admin_permissions()
 end
 
 local function seed()
-  local seed_lock = lock:new("locks", { ["timeout"] = 0 })
-  local _, lock_err = seed_lock:lock("seed_database")
-  if lock_err then
-    return
+  local _, err = wait_for_mongodb()
+  if not err then
+    seed_api_keys()
+    seed_initial_superusers()
+    seed_admin_permissions()
+  else
+    ngx.log(ngx.ERR, "timed out waiting for mongodb before seeding, rerunning...")
+    ngx.sleep(5)
+    seed()
   end
+end
 
-  seed_api_keys()
-  seed_initial_superusers()
-  seed_admin_permissions()
+local function seed_once()
+  interval_lock.mutex_exec("seed_database", seed)
 end
 
 return function()
-  local ok, err = ngx.timer.at(0, seed)
+  local ok, err = ngx.timer.at(0, seed_once)
   if not ok then
     ngx.log(ngx.ERR, "failed to create timer: ", err)
     return

src/api-umbrella/utils/mongo.lua

@@ -120,6 +120,12 @@ function _M.find(collection, query_options)
   return results, err
 end
 
+function _M.collections()
+  local collection = ""
+  local query_options = {}
+  return _M.find(collection, query_options)
+end
+
 function _M.first(collection, query_options)
   if not query_options then
     query_options = {}