Preload all lua modules in init phase. Get rid of init globals.

Preloading all the lua modules should ensure better memory usage. And
since required modules only ever get required once, we can actually get
rid of some of our globals, which should be better/safer.

This also adds a custom linting task that walks through the Lua source
files and ensure all of the requires are pre-loaded in in the init
phase, so this list is easier to maintain.

See https://github.com/openresty/lua-nginx-module#init_by_lua for more
details about COW memory handling of requiring in init phase.

.luacheckrc

@@ -3,10 +3,6 @@ std = "ngx_lua"
 globals = {
   "DEBUG",
   "WORKER_GROUP_ID",
-  "config",
-  "elasticsearch_templates",
-  "inspect",
-  "user_agent_parser_data",
 }
 
 max_line_length = false

Taskfile.yml

@@ -643,17 +643,30 @@ tasks:
       - ./build/work/stamp/test-deps/elasticsearch6
     method: checksum
 
-  test-deps:luacheck:
+  test-deps:lua:luacheck:
     deps:
       - deps:luarocks
     cmds:
-      - ./tasks/test-deps/luacheck
+      - ./tasks/test-deps/lua/luacheck
     sources:
       - ./build/work/stamp/deps/luarocks
-      - ./tasks/test-deps/luacheck
+      - ./tasks/test-deps/lua/luacheck
       - ./tasks/helpers.sh
     generates:
-      - ./build/work/stamp/test-deps/luacheck
+      - ./build/work/stamp/test-deps/lua/luacheck
+    method: checksum
+
+  test-deps:lua:penlight:
+    deps:
+      - deps:luarocks
+    cmds:
+      - ./tasks/test-deps/lua/penlight
+    sources:
+      - ./build/work/stamp/deps/luarocks
+      - ./tasks/test-deps/lua/penlight
+      - ./tasks/helpers.sh
+    generates:
+      - ./build/work/stamp/test-deps/lua/penlight
     method: checksum
 
   test-deps:mailhog:
@@ -721,7 +734,8 @@ tasks:
       - task: test-deps:bundle
       - task: test-deps:elasticsearch5
       - task: test-deps:elasticsearch6
-      - task: test-deps:luacheck
+      - task: test-deps:lua:luacheck
+      - task: test-deps:lua:penlight
       - task: test-deps:mailhog
       - task: test-deps:mongo-orchestration
       - task: test-deps:openldap
@@ -737,10 +751,16 @@ tasks:
 
   lint:lua:
     deps:
-      - test-deps:luacheck
+      - test-deps:lua:luacheck
     cmds:
       - ./tasks/lint/lua
 
+  lint:resty:
+    deps:
+      - test-deps:lua:penlight
+    cmds:
+      - ./tasks/lint/resty/run
+
   lint:ruby:
     deps:
       - test-deps:bundle
@@ -757,6 +777,7 @@ tasks:
     cmds:
       - task: lint:js
       - task: lint:lua
+      - task: lint:resty
       - task: lint:ruby
       - task: lint:shell
 

scripts/elasticsearch-v2-migrate/migrate.lua

@@ -1,5 +1,4 @@
-config = require "api-umbrella.proxy.models.file_config"
-require "api-umbrella.proxy.startup.init_user_agent_parser_data"
+local config = require "api-umbrella.proxy.models.file_config"
 
 -- local Date = require "pl.Date"
 local argparse = require "argparse"

scripts/import_access_logs/import.lua

@@ -1,7 +1,5 @@
-config = require "api-umbrella.proxy.models.file_config"
-require "api-umbrella.proxy.startup.init_user_agent_parser_data"
+local config = require "api-umbrella.proxy.models.file_config"
 
-inspect = require "inspect"
 local cjson = require "cjson"
 local elasticsearch_encode_json = require "api-umbrella.utils.elasticsearch_encode_json"
 local http = require "resty.http"

src/api-umbrella/auto-ssl/hooks/init.lua

@@ -1,7 +1,6 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local path = require "pl.path"
 
-config = require "api-umbrella.proxy.models.file_config"
-
 auto_ssl = (require "resty.auto-ssl").new({
   dir = path.join(config["etc_dir"], "auto-ssl"),
   hook_server_port = config["auto_ssl"]["hook_server"]["port"],

src/api-umbrella/auto-ssl/storage_adapters/mongodb.lua

@@ -1,5 +1,5 @@
 local aes = require "resty.aes"
-local inspect = require "inspect"
+local config = require "api-umbrella.proxy.models.file_config"
 local mongo = require "api-umbrella.utils.mongo"
 local path = require "pl.path"
 local resty_random = require "resty.random"
@@ -54,7 +54,7 @@ function _M.get(_, key)
       local aes_instance = assert(aes:new(ENCRYPTION_SECRET, nil, aes.cipher(256, "cbc"), { iv = res["encryption_iv"] }))
       res = aes_instance:decrypt(ngx.decode_base64(res["encrypted_value"]))
       if not res then
-        ngx.log(ngx.ERR, "auto-ssl: decryption failed: ", inspect(err))
+        ngx.log(ngx.ERR, "auto-ssl: decryption failed: ", (tostring(err) or ""))
         err = "decryption failed"
       end
     else

src/api-umbrella/elasticsearch-aws-signing-proxy/init.lua

@@ -1,2 +1 @@
-inspect = require "inspect"
-config = require "api-umbrella.proxy.models.file_config"
+require "api-umbrella.proxy.models.file_config"

src/api-umbrella/elasticsearch-aws-signing-proxy/proxy.lua

@@ -1,3 +1,5 @@
+local config = require "api-umbrella.proxy.models.file_config"
+
 local username = config["elasticsearch"]["aws_signing_proxy"]["username"]
 if not username then
   ngx.say("elasticsearch.aws_signing_proxy.username must be configured in /etc/api-umbrella/api-umbrella.yml")

src/api-umbrella/http-api/health.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local elasticsearch_query = require("api-umbrella.utils.elasticsearch").query
 local http = require "resty.http"
 local json_encode = require "api-umbrella.utils.json_encode"

src/api-umbrella/proxy/startup/init_elasticsearch_templates_data.lua → src/api-umbrella/proxy/elasticsearch_templates_data.lua

@@ -1,4 +1,7 @@
 local cjson = require "cjson"
+local config = require "api-umbrella.proxy.models.file_config"
+
+local elasticsearch_templates
 
 local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/elasticsearch_templates_v" .. config["elasticsearch"]["template_version"]
 if config["elasticsearch"]["api_version"] >= 5 then
@@ -41,3 +44,5 @@ else
 
   f:close()
 end
+
+return elasticsearch_templates

src/api-umbrella/proxy/error_handler.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays"
 local httpsify_current_url = require "api-umbrella.utils.httpsify_current_url"
 local is_hash = require "api-umbrella.utils.is_hash"

src/api-umbrella/proxy/hooks/init.lua

@@ -1,9 +1,7 @@
 DEBUG = false
 
 -- Pre-load modules.
-require "resty.txid"
-
-inspect = require "inspect"
+require "api-umbrella.proxy.hooks.init_preload_modules"
 
 -- Generate a unique ID to represent this group of worker processes. This value
 -- will be the same amongst all the subsequently inited workers, but the value
@@ -25,11 +23,6 @@ elseif incr_err then
   ngx.log(ngx.ERR, "worker_group_id incr err: ", incr_err)
 end
 
-config = require "api-umbrella.proxy.models.file_config"
-
-require "api-umbrella.proxy.startup.init_elasticsearch_templates_data"
-require "api-umbrella.proxy.startup.init_user_agent_parser_data"
-
 ngx.shared.stats:delete("distributed_last_fetched_at")
 ngx.shared.api_users:delete("last_fetched_at")
 ngx.shared.active_config:set("elasticsearch_templates_created", false)

src/api-umbrella/proxy/hooks/init_preload_modules.lua

@@ -0,0 +1,69 @@
+-- Pre-load modules.
+require "api-umbrella.proxy.distributed_rate_limit_queue"
+require "api-umbrella.proxy.elasticsearch_templates_data"
+require "api-umbrella.proxy.error_handler"
+require "api-umbrella.proxy.jobs.distributed_rate_limit_puller"
+require "api-umbrella.proxy.jobs.distributed_rate_limit_pusher"
+require "api-umbrella.proxy.jobs.elasticsearch_setup"
+require "api-umbrella.proxy.jobs.load_api_users"
+require "api-umbrella.proxy.jobs.load_db_config"
+require "api-umbrella.proxy.log_utils"
+require "api-umbrella.proxy.middleware.api_key_validator"
+require "api-umbrella.proxy.middleware.api_matcher"
+require "api-umbrella.proxy.middleware.api_settings"
+require "api-umbrella.proxy.middleware.https_validator"
+require "api-umbrella.proxy.middleware.ip_validator"
+require "api-umbrella.proxy.middleware.rate_limit"
+require "api-umbrella.proxy.middleware.referer_validator"
+require "api-umbrella.proxy.middleware.rewrite_request"
+require "api-umbrella.proxy.middleware.rewrite_response"
+require "api-umbrella.proxy.middleware.role_validator"
+require "api-umbrella.proxy.middleware.user_settings"
+require "api-umbrella.proxy.middleware.website_matcher"
+require "api-umbrella.proxy.models.active_config"
+require "api-umbrella.proxy.models.db_config"
+require "api-umbrella.proxy.models.file_config"
+require "api-umbrella.proxy.startup.seed_database"
+require "api-umbrella.proxy.user_agent_parser"
+require "api-umbrella.proxy.user_agent_parser_data"
+require "api-umbrella.proxy.user_store"
+require "api-umbrella.proxy.utils"
+require "api-umbrella.utils.array_last"
+require "api-umbrella.utils.deep_merge_overwrite_arrays"
+require "api-umbrella.utils.elasticsearch"
+require "api-umbrella.utils.escape_regex"
+require "api-umbrella.utils.escape_uri_non_ascii"
+require "api-umbrella.utils.flatten_headers"
+require "api-umbrella.utils.host_normalize"
+require "api-umbrella.utils.httpsify_current_url"
+require "api-umbrella.utils.interval_lock"
+require "api-umbrella.utils.invert_table"
+require "api-umbrella.utils.is_hash"
+require "api-umbrella.utils.matches_hostname"
+require "api-umbrella.utils.mongo"
+require "api-umbrella.utils.mustache_unescape"
+require "api-umbrella.utils.nillify_json_nulls"
+require "api-umbrella.utils.nillify_yaml_nulls"
+require "api-umbrella.utils.random_token"
+require "api-umbrella.utils.redirect_matches_to_https"
+require "cjson"
+require "cmsgpack"
+require "iconv"
+require "icu-date"
+require "libcidr-ffi"
+require "lustache"
+require "lyaml"
+require "pl.path"
+require "pl.seq"
+require "pl.stringx"
+require "pl.tablex"
+require "pl.types"
+require "pl.utils"
+require "resty.logger.socket"
+require "resty.lrucache.pureffi"
+require "resty.sha256"
+require "resty.string"
+require "resty.txid"
+require "resty.uuid"
+require "shcache"
+require "socket.url"

src/api-umbrella/proxy/hooks/rewrite.lua

@@ -1,4 +1,5 @@
 local api_matcher = require "api-umbrella.proxy.middleware.api_matcher"
+local config = require "api-umbrella.proxy.models.file_config"
 local error_handler = require "api-umbrella.proxy.error_handler"
 local host_normalize = require "api-umbrella.utils.host_normalize"
 local redirect_matches_to_https = require "api-umbrella.utils.redirect_matches_to_https"

src/api-umbrella/proxy/hooks/set_x_api_umbrella_request_id.lua

@@ -0,0 +1,2 @@
+local txid = require "resty.txid"
+return txid()

src/api-umbrella/proxy/jobs/distributed_rate_limit_pusher.lua

@@ -1,6 +1,7 @@
 local _M = {}
 
 local array_last = require "api-umbrella.utils.array_last"
+local config = require "api-umbrella.proxy.models.file_config"
 local distributed_rate_limit_queue = require "api-umbrella.proxy.distributed_rate_limit_queue"
 local mongo = require "api-umbrella.utils.mongo"
 local plutils = require "pl.utils"

src/api-umbrella/proxy/jobs/elasticsearch_setup.lua

@@ -1,6 +1,8 @@
 local _M = {}
 
+local config = require "api-umbrella.proxy.models.file_config"
 local elasticsearch_query = require("api-umbrella.utils.elasticsearch").query
+local elasticsearch_templates = require "api-umbrella.proxy.elasticsearch_templates_data"
 local interval_lock = require "api-umbrella.utils.interval_lock"
 
 local delay = 3600  -- in seconds

src/api-umbrella/proxy/log_utils.lua

@@ -1,4 +1,5 @@
 local cjson = require "cjson"
+local config = require "api-umbrella.proxy.models.file_config"
 local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii"
 local iconv = require "iconv"
 local icu_date = require "icu-date"

src/api-umbrella/proxy/middleware/api_key_validator.lua

@@ -1,5 +1,6 @@
-local user_store = require "api-umbrella.proxy.user_store"
+local config = require "api-umbrella.proxy.models.file_config"
 local types = require "pl.types"
+local user_store = require "api-umbrella.proxy.user_store"
 
 local get_user = user_store.get
 local is_empty = types.is_empty

src/api-umbrella/proxy/middleware/api_matcher.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local matches_hostname = require "api-umbrella.utils.matches_hostname"
 local stringx = require "pl.stringx"
 local utils = require "api-umbrella.proxy.utils"

src/api-umbrella/proxy/middleware/api_settings.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays"
 local seq = require "pl.seq"
 local tablex = require "pl.tablex"

src/api-umbrella/proxy/middleware/rate_limit.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local distributed_rate_limit_queue = require "api-umbrella.proxy.distributed_rate_limit_queue"
 
 local function bucket_keys(settings, user, limit, current_time)

src/api-umbrella/proxy/middleware/rewrite_request.lua

@@ -1,4 +1,4 @@
-local inspect = require "inspect"
+local config = require "api-umbrella.proxy.models.file_config"
 local lustache = require "lustache"
 local plutils = require "pl.utils"
 local stringx = require "pl.stringx"
@@ -115,7 +115,7 @@ local function set_headers(settings)
         if ok then
           value = output
         else
-          ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. inspect(output))
+          ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. (tostring(output) or ""))
         end
       else
         value = header["value"]
@@ -236,7 +236,7 @@ local function url_rewrites(api)
             if ok then
               new_uri = output
             else
-              ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. inspect(output))
+              ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. (tostring(output) or ""))
             end
 
             if rewrite["_backend_replacement_args"] then
@@ -248,7 +248,7 @@ local function url_rewrites(api)
               if ok then
                 new_uri = new_uri .. "?" .. output
               else
-                ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. inspect(output))
+                ngx.log(ngx.ERR, "Mustache rendering error while rendering error template: " .. (tostring(output) or ""))
               end
             end
           end

src/api-umbrella/proxy/middleware/rewrite_response.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local stringx = require "pl.stringx"
 local url = require "socket.url"
 local utils = require "api-umbrella.proxy.utils"

src/api-umbrella/proxy/middleware/website_matcher.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local matches_hostname = require "api-umbrella.utils.matches_hostname"
 
 return function(active_config)

src/api-umbrella/proxy/models/active_config.lua

@@ -1,13 +1,14 @@
 local cidr = require "libcidr-ffi"
 local cjson = require "cjson"
+local config = require "api-umbrella.proxy.models.file_config"
 local escape_regex = require "api-umbrella.utils.escape_regex"
 local host_normalize = require "api-umbrella.utils.host_normalize"
 local mustache_unescape = require "api-umbrella.utils.mustache_unescape"
 local plutils = require "pl.utils"
 local random_token = require "api-umbrella.utils.random_token"
+local startswith = require("pl.stringx").startswith
 local tablex = require "pl.tablex"
 local utils = require "api-umbrella.proxy.utils"
-local startswith = require("pl.stringx").startswith
 
 local append_array = utils.append_array
 local cache_computed_settings = utils.cache_computed_settings

src/api-umbrella/proxy/startup/seed_database.lua

@@ -1,3 +1,4 @@
+local config = require "api-umbrella.proxy.models.file_config"
 local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays"
 local interval_lock = require "api-umbrella.utils.interval_lock"
 local mongo = require "api-umbrella.utils.mongo"

src/api-umbrella/proxy/user_agent_parser.lua

@@ -1,6 +1,7 @@
 local lrucache = require "resty.lrucache.pureffi"
+local data = require "api-umbrella.proxy.user_agent_parser_data"
 
-local data = user_agent_parser_data
+-- local data = user_agent_parser_data
 local cache = lrucache.new(500)
 
 return function(user_agent)

src/api-umbrella/proxy/startup/init_user_agent_parser_data.lua → src/api-umbrella/proxy/user_agent_parser_data.lua

@@ -1,5 +1,7 @@
 local cjson = require "cjson"
 
+local user_agent_parser_data
+
 local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/user_agent_data.json"
 local f, err = io.open(path, "rb")
 if err then
@@ -17,3 +19,5 @@ else
 
   f:close()
 end
+
+return user_agent_parser_data