diff --git a/src/api-umbrella/proxy/jobs/elasticsearch_setup.lua b/src/api-umbrella/proxy/jobs/elasticsearch_setup.lua index 254cba8d1..6015ba578 100644 --- a/src/api-umbrella/proxy/jobs/elasticsearch_setup.lua +++ b/src/api-umbrella/proxy/jobs/elasticsearch_setup.lua @@ -17,7 +17,7 @@ local function wait_for_elasticsearch() repeat local res, err = httpc:request_uri(elasticsearch_host .. "/_cluster/health") if err then - ngx.log(ngx.ERR, "failed to fetch cluster health from elasticsearch: ", err) + ngx.log(ngx.NOTICE, "failed to fetch cluster health from elasticsearch (this is expected if elasticsearch is starting up at the same time): ", err) elseif res.body then local elasticsearch_health = cjson.decode(res.body) if elasticsearch_health["status"] == "yellow" or elasticsearch_health["status"] == "green" then @@ -30,6 +30,12 @@ local function wait_for_elasticsearch() wait_time = wait_time + sleep_time end until elasticsearch_alive or wait_time > max_time + + if elasticsearch_alive then + return true, nil + else + return false, "elasticsearch was not ready within " .. max_time .."s" + end end local function create_templates() @@ -101,14 +107,20 @@ local function create_aliases() end end -local function do_check() - wait_for_elasticsearch() - create_templates() - create_aliases() +local function setup() + local _, err = wait_for_elasticsearch() + if not err then + create_templates() + create_aliases() + else + ngx.log(ngx.ERR, "timed out waiting for eleasticsearch before setup, rerunning...") + ngx.sleep(5) + setup() + end end function _M.spawn() - interval_lock.repeat_with_mutex('elasticsearch_index_setup', delay, do_check) + interval_lock.repeat_with_mutex('elasticsearch_index_setup', delay, setup) end return _M diff --git a/src/api-umbrella/proxy/startup/seed_database.lua b/src/api-umbrella/proxy/startup/seed_database.lua index 3f4bb4ffb..30b6867e1 100644 --- a/src/api-umbrella/proxy/startup/seed_database.lua +++ b/src/api-umbrella/proxy/startup/seed_database.lua @@ -1,11 +1,37 @@ local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays" -local lock = require "resty.lock" +local interval_lock = require "api-umbrella.utils.interval_lock" local mongo = require "api-umbrella.utils.mongo" local random_token = require "api-umbrella.utils.random_token" local uuid = require "resty.uuid" local nowMongoDate = { ["$date"] = { ["$numberLong"] = tostring(os.time() * 1000) } } +local function wait_for_mongodb() + local mongodb_alive = false + local wait_time = 0 + local sleep_time = 0.5 + local max_time = 14 + repeat + local _, err = mongo.collections() + if err then + ngx.log(ngx.NOTICE, "failed to establish connection to mongodb (this is expected if mongodb is starting up at the same time): ", err) + else + mongodb_alive = true + end + + if not mongodb_alive then + ngx.sleep(sleep_time) + wait_time = wait_time + sleep_time + end + until mongodb_alive or wait_time > max_time + + if mongodb_alive then + return true, nil + else + return false, "elasticsearch was not ready within " .. max_time .."s" + end +end + local function seed_api_keys() local keys = { -- static.site.ajax@internal.apiumbrella @@ -208,19 +234,24 @@ local function seed_admin_permissions() end local function seed() - local seed_lock = lock:new("locks", { ["timeout"] = 0 }) - local _, lock_err = seed_lock:lock("seed_database") - if lock_err then - return + local _, err = wait_for_mongodb() + if not err then + seed_api_keys() + seed_initial_superusers() + seed_admin_permissions() + else + ngx.log(ngx.ERR, "timed out waiting for mongodb before seeding, rerunning...") + ngx.sleep(5) + seed() end +end - seed_api_keys() - seed_initial_superusers() - seed_admin_permissions() +local function seed_once() + interval_lock.mutex_exec("seed_database", seed) end return function() - local ok, err = ngx.timer.at(0, seed) + local ok, err = ngx.timer.at(0, seed_once) if not ok then ngx.log(ngx.ERR, "failed to create timer: ", err) return diff --git a/src/api-umbrella/utils/mongo.lua b/src/api-umbrella/utils/mongo.lua index f3d26fceb..04f225507 100644 --- a/src/api-umbrella/utils/mongo.lua +++ b/src/api-umbrella/utils/mongo.lua @@ -120,6 +120,12 @@ function _M.find(collection, query_options) return results, err end +function _M.collections() + local collection = "" + local query_options = {} + return _M.find(collection, query_options) +end + function _M.first(collection, query_options) if not query_options then query_options = {}