Update to use GeoIP2 database.

The legacy databases we were using were discontinued
(https://support.maxmind.com/geolite-legacy-discontinuation-notice/), so
we need to use a different nginx plugin to integrate with the newer
GeoLite2 databases.

#440

Taskfile.yml

@@ -15,9 +15,10 @@
 #   within the dependency chain. We use these stamp files so that the
 #   checksumming approach properly invalidates the entire chain of dependencies
 #   when build files change (whereas just using "deps" in go-task doesn't fully
-#   work for multiple levels of dependencies). So, for example, if the libgeoip
-#   dependency changes, rebuilding it properly cascades and triggers a rebuild
-#   of openresty (1st level dependency) and luarocks (2nd level dependency).
+#   work for multiple levels of dependencies). So, for example, if the
+#   libmaxminddb dependency changes, rebuilding it properly cascades and
+#   triggers a rebuild of openresty (1st level dependency) and luarocks (2nd
+#   level dependency).
 # - For colorized output we use the "unbuffer" tool to force color output on
 #   some commands, since Task doesn't easily support detecting colorized output
 #   (as of the output changes in 2.0.3):
@@ -91,14 +92,14 @@ tasks:
       - ./build/work/stamp/deps/libfastjson
     method: checksum
 
-  deps:libgeoip:
+  deps:libmaxminddb:
     cmds:
-      - ./tasks/deps/libgeoip
+      - ./tasks/deps/libmaxminddb
     sources:
-      - ./tasks/deps/libgeoip
+      - ./tasks/deps/libmaxminddb
       - ./tasks/helpers.sh
     generates:
-      - ./build/work/stamp/deps/libgeoip
+      - ./build/work/stamp/deps/libmaxminddb
     method: checksum
 
   deps:luarocks:
@@ -136,11 +137,11 @@ tasks:
 
   deps:openresty:
     deps:
-      - deps:libgeoip
+      - deps:libmaxminddb
     cmds:
       - ./tasks/deps/openresty
     sources:
-      - ./build/work/stamp/deps/libgeoip
+      - ./build/work/stamp/deps/libmaxminddb
       - ./tasks/deps/openresty
       - ./tasks/helpers.sh
     generates:
@@ -299,19 +300,6 @@ tasks:
       - ./build/work/stamp/app-deps/lua/cmsgpack
     method: checksum
 
-  app-deps:lua:iconv:
-    deps:
-      - deps:luarocks
-    cmds:
-      - ./tasks/app-deps/lua/iconv
-    sources:
-      - ./build/work/stamp/deps/luarocks
-      - ./tasks/app-deps/lua/iconv
-      - ./tasks/helpers.sh
-    generates:
-      - ./build/work/stamp/app-deps/lua/iconv
-    method: checksum
-
   app-deps:lua:icu-date:
     deps:
       - deps:luarocks
@@ -517,7 +505,6 @@ tasks:
       - task: app-deps:admin-ui:yarn
       - task: app-deps:lua:argparse
       - task: app-deps:lua:cmsgpack
-      - task: app-deps:lua:iconv
       - task: app-deps:lua:icu-date
       - task: app-deps:lua:inspect
       - task: app-deps:lua:libcidr-ffi

scripts/rake/outdated_packages.rb

@@ -36,18 +36,15 @@ class OutdatedPackages
     "libfastjson" => {
       :git => "https://github.com/rsyslog/libfastjson.git",
     },
-    "libgeoip" => {
-      :git => "https://github.com/maxmind/geoip-api-c.git",
+    "libmaxminddb" => {
+      :git => "https://github.com/maxmind/libmaxminddb.git",
     },
     "lua_argparse" => {
       :luarock => "argparse",
     },
     "lua_cmsgpack" => {
       :luarock => "lua-cmsgpack",
     },
-    "lua_iconv" => {
-      :luarock => "lua-iconv",
-    },
     "lua_icu_date" => {
       :git => "https://github.com/GUI/lua-icu-date.git",
       :git_ref => "master",
@@ -114,6 +111,9 @@ class OutdatedPackages
       :git => "https://github.com/emicklei/mora.git",
       :git_ref => "master",
     },
+    "ngx_http_geoip2_module" => {
+      :git => "https://github.com/leev/ngx_http_geoip2_module.git",
+    },
     "nodejs" => {
       :git => "https://github.com/nodejs/node.git",
       :constraint => "~> 10.13",

src/api-umbrella/cli/setup.lua

@@ -130,9 +130,9 @@ local function ensure_geoip_db()
   -- If the city db path doesn't exist, copy it from the package installation
   -- location to the runtime location (this path will then be overwritten by
   -- the auto-updater so we don't touch the original packaged file).
-  local city_db_path = path.join(config["db_dir"], "geoip/city-v6.dat")
+  local city_db_path = path.join(config["db_dir"], "geoip/GeoLite2-City.mmdb")
   if not path.exists(city_db_path) then
-    local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/city-v6.dat")
+    local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/GeoLite2-City.mmdb")
     dir.makepath(path.dirname(city_db_path))
     file.copy(default_city_db_path, city_db_path)
     chmod(city_db_path, tonumber("0640", 8))

src/api-umbrella/proxy/hooks/init_preload_modules.lua

@@ -56,7 +56,6 @@ require "api-umbrella.utils.round"
 require "api-umbrella.utils.xpcall_error_handler"
 require "cjson"
 require "cmsgpack"
-require "iconv"
 require "icu-date"
 require "libcidr-ffi"
 require "lustache"

src/api-umbrella/proxy/log_utils.lua

@@ -1,6 +1,5 @@
 local config = require "api-umbrella.proxy.models.file_config"
 local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii"
-local iconv = require "iconv"
 local icu_date = require "icu-date"
 local json_encode = require "api-umbrella.utils.json_encode"
 local logger = require "resty.logger.socket"
@@ -232,34 +231,14 @@ function _M.cache_new_city_geocode(data)
 end
 
 function _M.set_request_ip_geo_fields(data, ngx_var)
-  -- The GeoIP module returns ISO-8859-1 encoded city names, but we need UTF-8
-  -- for inserting into ElasticSearch.
-  local geoip_city = ngx_var.geoip_city
-  if geoip_city then
-    local encoding_converter = iconv.new("utf-8//IGNORE", "iso-8859-1")
-    local geoip_city_encoding_err
-    geoip_city, geoip_city_encoding_err  = encoding_converter:iconv(geoip_city)
-    if geoip_city_encoding_err then
-      ngx.log(ngx.ERR, "encoding error for geoip city: ", geoip_city_encoding_err, geoip_city)
-    end
-  end
-
-  -- The geoip database returns "00" for unknown regions sometimes:
-  -- http://maxmind.com/download/geoip/kml/index.html Remove these and treat
-  -- these as nil.
-  local geoip_region = ngx_var.geoip_region
-  if geoip_region == "00" then
-    geoip_region = nil
-  end
-
-  data["request_ip_city"] = geoip_city
-  data["request_ip_country"] = ngx_var.geoip_city_country_code
-  data["request_ip_region"] = geoip_region
+  data["request_ip_city"] = ngx_var.geoip2_data_city_name
+  data["request_ip_country"] = ngx_var.geoip2_data_country_code
+  data["request_ip_region"] = ngx_var.geoip2_data_subdivision_code
 
-  local geoip_latitude = ngx_var.geoip_latitude
+  local geoip_latitude = ngx_var.geoip2_data_latitude
   if geoip_latitude then
     data["request_ip_lat"] = tonumber(geoip_latitude)
-    data["request_ip_lon"] = tonumber(ngx_var.geoip_longitude)
+    data["request_ip_lon"] = tonumber(ngx_var.geoip2_data_longitude)
   end
 end
 

tasks/deps/geolitecity

@@ -7,15 +7,18 @@ set +x
 # If the file exists and was created within the past day, then skip doing
 # anything else. This approach ensures that new files are downloaded, but only
 # once per day.
-if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat" -mmin -1440) ]]; then
+if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb" -mmin -1440) ]]; then
   exit 0
 fi
 
 set -x
 
 task_working_dir
-curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz
-gunzip -c GeoLiteCityv6.dat.gz > GeoLiteCityv6.dat
-install -D -m 644 ./GeoLiteCityv6.dat "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat"
+curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
+dir="GeoLite2-City"
+rm -rf "$dir"
+mkdir -p "$dir"
+tar -xf GeoLite2-City.tar.gz -C "$dir" --strip-components 1
+install -D -m 644 ./GeoLite2-City/GeoLite2-City.mmdb "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb"
 
 stamp

tasks/deps/libmaxminddb

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+libmaxminddb_version="1.3.2"
+libmaxminddb_hash="67a861965b30d045bf29a2126bcc05ed"
+
+set -e -u -x
+source ./tasks/helpers.sh
+
+task_working_dir
+download "https://github.com/maxmind/libmaxminddb/releases/download/$libmaxminddb_version/libmaxminddb-$libmaxminddb_version.tar.gz" "md5" "$libmaxminddb_hash"
+extract_download "libmaxminddb-$libmaxminddb_version.tar.gz"
+
+cd "libmaxminddb-$libmaxminddb_version"
+./configure --prefix="$INSTALL_PREFIX_EMBEDDED"
+make -j"$NPROC"
+make install DESTDIR="$STAGE_DIR"
+find "$STAGE_EMBEDDED_DIR/bin/" -name "mmdblookup" -exec chrpath -d {} \;
+
+stamp

tasks/deps/openresty

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+ngx_http_geoip2_module_version="3.2"
+ngx_http_geoip2_module_hash="2e60d26460d178068f7a4e9a4e64990a"
 openresty_version="1.13.6.2"
 openresty_hash="d95bc4bbe15e4b045a0593b4ecc0db38"
 openssl_version="1.0.2q"
@@ -21,6 +23,9 @@ extract_download "openssl-$openssl_version.tar.gz"
 download "https://openresty.org/download/openresty-$openresty_version.tar.gz" "md5" "$openresty_hash"
 extract_download "openresty-$openresty_version.tar.gz"
 
+download "https://github.com/leev/ngx_http_geoip2_module/archive/$ngx_http_geoip2_module_version.tar.gz" "md5" "$ngx_http_geoip2_module_hash"
+extract_download "$ngx_http_geoip2_module_version.tar.gz"
+
 cd "openresty-$openresty_version"
 patch -p1 < "$SOURCE_DIR/build/patches/openresty-opm.patch"
 patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch"
@@ -36,12 +41,12 @@ patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch"
   --with-pcre-opt=-g \
   --with-pcre-conf-opt=--enable-unicode-properties \
   --with-pcre-jit \
-  --with-http_geoip_module \
   --with-http_gunzip_module \
   --with-http_gzip_static_module \
   --with-http_realip_module \
   --with-http_ssl_module \
-  --with-http_stub_status_module
+  --with-http_stub_status_module \
+  --add-module="../ngx_http_geoip2_module-$ngx_http_geoip2_module_version"
 make -j"$NPROC"
 make install DESTDIR="$STAGE_DIR"
 chrpath -d "$STAGE_EMBEDDED_DIR/openresty/nginx/sbin/nginx"

templates/etc/nginx/router.conf.mustache

@@ -71,7 +71,13 @@ http {
     resolver_timeout 12s;
   {{/dns_resolver._nameservers_nginx}}
 
-  geoip_city {{db_dir}}/geoip/city-v6.dat;
+  geoip2 {{db_dir}}/geoip/GeoLite2-City.mmdb {
+    $geoip2_data_country_code country iso_code;
+    $geoip2_data_subdivision_code subdivisions 0 iso_code;
+    $geoip2_data_city_name city names en;
+    $geoip2_data_latitude location latitude;
+    $geoip2_data_longitude location longitude;
+  }
 
   geo $banned_ip {
     default 0;

test/processes/test_rpaths.rb

@@ -38,7 +38,7 @@ def test_binary_rpaths
       "/embedded/openresty/nginx/sbin/nginx",
       "/embedded/libexec/trafficserver/ssl_cert_loader.so",
       # LuaRock
-      "/embedded/apps/core/shared/vendor/lua/lib/lua/5.1/iconv.so",
+      "/embedded/apps/core/shared/vendor/lua/lib/lua/5.1/yaml.so",
       # Rubygem
       "/oj/oj.so",
     ].each do |expected_path_end|

test/proxy/logging/test_ip_geocoding.rb

@@ -80,13 +80,32 @@ def test_country_city_no_region
       :ip => "104.250.168.24",
       :country => "MC",
       :region => nil,
-      :city => "Monte-carlo",
+      :city => "Monte Carlo",
       :lat => 43.7333,
       :lon => 7.4167,
     })
   end
 
   def test_country_no_region_city
+    response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({
+      :headers => {
+        "X-Forwarded-For" => "1.1.1.1",
+      },
+    }))
+    assert_response_code(200, response)
+
+    record = wait_for_log(response)[:hit_source]
+    assert_geocode(record, {
+      :ip => "1.1.1.1",
+      :country => "AU",
+      :region => nil,
+      :city => nil,
+      :lat => -33.494,
+      :lon => 143.2104,
+    })
+  end
+
+  def test_no_country_region_city
     response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({
       :headers => {
         "X-Forwarded-For" => "67.43.156.1",
@@ -97,11 +116,11 @@ def test_country_no_region_city
     record = wait_for_log(response)[:hit_source]
     assert_geocode(record, {
       :ip => "67.43.156.1",
-      :country => "A1",
+      :country => nil,
       :region => nil,
       :city => nil,
-      :lat => 0.0,
-      :lon => 0.0,
+      :lat => nil,
+      :lon => nil,
     })
   end
 
@@ -118,7 +137,7 @@ def test_city_accent_chars
       :ip => "184.148.224.214",
       :country => "CA",
       :region => "QC",
-      :city => "Trois-rivières",
+      :city => "Trois-Rivières",
       :lat => 46.316,
       :lon => -72.6833,
     })
@@ -128,12 +147,19 @@ def test_city_accent_chars
 
   def assert_geocode(record, options)
     assert_geocode_log(record, options)
-    assert_geocode_cache(record, options)
+    if !options.fetch(:lat).nil? || !options.fetch(:lon).nil?
+      assert_geocode_cache(record, options)
+    end
   end
 
   def assert_geocode_log(record, options)
     assert_equal(options.fetch(:ip), record.fetch("request_ip"))
-    assert_equal(options.fetch(:country), record.fetch("request_ip_country"))
+    if(options.fetch(:country).nil?)
+      assert_nil(record["request_ip_country"])
+      refute(record.key?("request_ip_country"))
+    else
+      assert_equal(options.fetch(:country), record.fetch("request_ip_country"))
+    end
     if(options.fetch(:region).nil?)
       assert_nil(record["request_ip_region"])
       refute(record.key?("request_ip_region"))