diff --git a/Taskfile.yml b/Taskfile.yml index 975884415..d4e34a515 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -15,9 +15,10 @@ # within the dependency chain. We use these stamp files so that the # checksumming approach properly invalidates the entire chain of dependencies # when build files change (whereas just using "deps" in go-task doesn't fully -# work for multiple levels of dependencies). So, for example, if the libgeoip -# dependency changes, rebuilding it properly cascades and triggers a rebuild -# of openresty (1st level dependency) and luarocks (2nd level dependency). +# work for multiple levels of dependencies). So, for example, if the +# libmaxminddb dependency changes, rebuilding it properly cascades and +# triggers a rebuild of openresty (1st level dependency) and luarocks (2nd +# level dependency). # - For colorized output we use the "unbuffer" tool to force color output on # some commands, since Task doesn't easily support detecting colorized output # (as of the output changes in 2.0.3): @@ -91,14 +92,14 @@ tasks: - ./build/work/stamp/deps/libfastjson method: checksum - deps:libgeoip: + deps:libmaxminddb: cmds: - - ./tasks/deps/libgeoip + - ./tasks/deps/libmaxminddb sources: - - ./tasks/deps/libgeoip + - ./tasks/deps/libmaxminddb - ./tasks/helpers.sh generates: - - ./build/work/stamp/deps/libgeoip + - ./build/work/stamp/deps/libmaxminddb method: checksum deps:luarocks: @@ -136,11 +137,11 @@ tasks: deps:openresty: deps: - - deps:libgeoip + - deps:libmaxminddb cmds: - ./tasks/deps/openresty sources: - - ./build/work/stamp/deps/libgeoip + - ./build/work/stamp/deps/libmaxminddb - ./tasks/deps/openresty - ./tasks/helpers.sh generates: @@ -299,19 +300,6 @@ tasks: - ./build/work/stamp/app-deps/lua/cmsgpack method: checksum - app-deps:lua:iconv: - deps: - - deps:luarocks - cmds: - - ./tasks/app-deps/lua/iconv - sources: - - ./build/work/stamp/deps/luarocks - - ./tasks/app-deps/lua/iconv - - ./tasks/helpers.sh - generates: - - ./build/work/stamp/app-deps/lua/iconv - method: checksum - app-deps:lua:icu-date: deps: - deps:luarocks @@ -517,7 +505,6 @@ tasks: - task: app-deps:admin-ui:yarn - task: app-deps:lua:argparse - task: app-deps:lua:cmsgpack - - task: app-deps:lua:iconv - task: app-deps:lua:icu-date - task: app-deps:lua:inspect - task: app-deps:lua:libcidr-ffi diff --git a/scripts/rake/outdated_packages.rb b/scripts/rake/outdated_packages.rb index 63988dd52..677f707df 100644 --- a/scripts/rake/outdated_packages.rb +++ b/scripts/rake/outdated_packages.rb @@ -36,8 +36,8 @@ class OutdatedPackages "libfastjson" => { :git => "https://github.com/rsyslog/libfastjson.git", }, - "libgeoip" => { - :git => "https://github.com/maxmind/geoip-api-c.git", + "libmaxminddb" => { + :git => "https://github.com/maxmind/libmaxminddb.git", }, "lua_argparse" => { :luarock => "argparse", @@ -45,9 +45,6 @@ class OutdatedPackages "lua_cmsgpack" => { :luarock => "lua-cmsgpack", }, - "lua_iconv" => { - :luarock => "lua-iconv", - }, "lua_icu_date" => { :git => "https://github.com/GUI/lua-icu-date.git", :git_ref => "master", @@ -114,6 +111,9 @@ class OutdatedPackages :git => "https://github.com/emicklei/mora.git", :git_ref => "master", }, + "ngx_http_geoip2_module" => { + :git => "https://github.com/leev/ngx_http_geoip2_module.git", + }, "nodejs" => { :git => "https://github.com/nodejs/node.git", :constraint => "~> 10.13", diff --git a/src/api-umbrella/cli/setup.lua b/src/api-umbrella/cli/setup.lua index 1734eab5f..7151a1d33 100644 --- a/src/api-umbrella/cli/setup.lua +++ b/src/api-umbrella/cli/setup.lua @@ -130,9 +130,9 @@ local function ensure_geoip_db() -- If the city db path doesn't exist, copy it from the package installation -- location to the runtime location (this path will then be overwritten by -- the auto-updater so we don't touch the original packaged file). - local city_db_path = path.join(config["db_dir"], "geoip/city-v6.dat") + local city_db_path = path.join(config["db_dir"], "geoip/GeoLite2-City.mmdb") if not path.exists(city_db_path) then - local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/city-v6.dat") + local default_city_db_path = path.join(config["_embedded_root_dir"], "var/db/geoip/GeoLite2-City.mmdb") dir.makepath(path.dirname(city_db_path)) file.copy(default_city_db_path, city_db_path) chmod(city_db_path, tonumber("0640", 8)) diff --git a/src/api-umbrella/proxy/hooks/init_preload_modules.lua b/src/api-umbrella/proxy/hooks/init_preload_modules.lua index 51a73c9cd..cd988c70f 100644 --- a/src/api-umbrella/proxy/hooks/init_preload_modules.lua +++ b/src/api-umbrella/proxy/hooks/init_preload_modules.lua @@ -56,7 +56,6 @@ require "api-umbrella.utils.round" require "api-umbrella.utils.xpcall_error_handler" require "cjson" require "cmsgpack" -require "iconv" require "icu-date" require "libcidr-ffi" require "lustache" diff --git a/src/api-umbrella/proxy/log_utils.lua b/src/api-umbrella/proxy/log_utils.lua index 96fd2cb70..cbe417a58 100644 --- a/src/api-umbrella/proxy/log_utils.lua +++ b/src/api-umbrella/proxy/log_utils.lua @@ -1,6 +1,5 @@ local config = require "api-umbrella.proxy.models.file_config" local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii" -local iconv = require "iconv" local icu_date = require "icu-date" local json_encode = require "api-umbrella.utils.json_encode" local logger = require "resty.logger.socket" @@ -232,34 +231,14 @@ function _M.cache_new_city_geocode(data) end function _M.set_request_ip_geo_fields(data, ngx_var) - -- The GeoIP module returns ISO-8859-1 encoded city names, but we need UTF-8 - -- for inserting into ElasticSearch. - local geoip_city = ngx_var.geoip_city - if geoip_city then - local encoding_converter = iconv.new("utf-8//IGNORE", "iso-8859-1") - local geoip_city_encoding_err - geoip_city, geoip_city_encoding_err = encoding_converter:iconv(geoip_city) - if geoip_city_encoding_err then - ngx.log(ngx.ERR, "encoding error for geoip city: ", geoip_city_encoding_err, geoip_city) - end - end - - -- The geoip database returns "00" for unknown regions sometimes: - -- http://maxmind.com/download/geoip/kml/index.html Remove these and treat - -- these as nil. - local geoip_region = ngx_var.geoip_region - if geoip_region == "00" then - geoip_region = nil - end - - data["request_ip_city"] = geoip_city - data["request_ip_country"] = ngx_var.geoip_city_country_code - data["request_ip_region"] = geoip_region + data["request_ip_city"] = ngx_var.geoip2_data_city_name + data["request_ip_country"] = ngx_var.geoip2_data_country_code + data["request_ip_region"] = ngx_var.geoip2_data_subdivision_code - local geoip_latitude = ngx_var.geoip_latitude + local geoip_latitude = ngx_var.geoip2_data_latitude if geoip_latitude then data["request_ip_lat"] = tonumber(geoip_latitude) - data["request_ip_lon"] = tonumber(ngx_var.geoip_longitude) + data["request_ip_lon"] = tonumber(ngx_var.geoip2_data_longitude) end end diff --git a/tasks/app-deps/lua/iconv b/tasks/app-deps/lua/iconv deleted file mode 100755 index f497cecfe..000000000 --- a/tasks/app-deps/lua/iconv +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -lua_iconv_version="7-3" - -set -e -u -x -source ./tasks/helpers.sh - -luarocks_install "lua-iconv" "$lua_iconv_version" - -stamp diff --git a/tasks/deps/geolitecity b/tasks/deps/geolitecity index c1bd0464e..1acdf2543 100755 --- a/tasks/deps/geolitecity +++ b/tasks/deps/geolitecity @@ -7,15 +7,18 @@ set +x # If the file exists and was created within the past day, then skip doing # anything else. This approach ensures that new files are downloaded, but only # once per day. -if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat" -mmin -1440) ]]; then +if [[ $(find "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb" -mmin -1440) ]]; then exit 0 fi set -x task_working_dir -curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz -gunzip -c GeoLiteCityv6.dat.gz > GeoLiteCityv6.dat -install -D -m 644 ./GeoLiteCityv6.dat "$STAGE_EMBEDDED_DIR/var/db/geoip/city-v6.dat" +curl --location --retry 3 --fail --remote-name https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz +dir="GeoLite2-City" +rm -rf "$dir" +mkdir -p "$dir" +tar -xf GeoLite2-City.tar.gz -C "$dir" --strip-components 1 +install -D -m 644 ./GeoLite2-City/GeoLite2-City.mmdb "$STAGE_EMBEDDED_DIR/var/db/geoip/GeoLite2-City.mmdb" stamp diff --git a/tasks/deps/libgeoip b/tasks/deps/libgeoip deleted file mode 100755 index ad40108d3..000000000 --- a/tasks/deps/libgeoip +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -libgeoip_version="1.6.12" -libgeoip_hash="77d496cc40daa1dbc2b97365807d64d7" - -set -e -u -x -source ./tasks/helpers.sh - -task_working_dir -download "https://github.com/maxmind/geoip-api-c/releases/download/v$libgeoip_version/GeoIP-$libgeoip_version.tar.gz" "md5" "$libgeoip_hash" -extract_download "GeoIP-$libgeoip_version.tar.gz" - -cd "GeoIP-$libgeoip_version" -./configure --prefix="$INSTALL_PREFIX_EMBEDDED" -make -j"$NPROC" -make install DESTDIR="$STAGE_DIR" -find "$STAGE_EMBEDDED_DIR/bin/" -name "geoiplookup*" -exec chrpath -d {} \; - -stamp diff --git a/tasks/deps/libmaxminddb b/tasks/deps/libmaxminddb new file mode 100755 index 000000000..01a2a80f0 --- /dev/null +++ b/tasks/deps/libmaxminddb @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +libmaxminddb_version="1.3.2" +libmaxminddb_hash="67a861965b30d045bf29a2126bcc05ed" + +set -e -u -x +source ./tasks/helpers.sh + +task_working_dir +download "https://github.com/maxmind/libmaxminddb/releases/download/$libmaxminddb_version/libmaxminddb-$libmaxminddb_version.tar.gz" "md5" "$libmaxminddb_hash" +extract_download "libmaxminddb-$libmaxminddb_version.tar.gz" + +cd "libmaxminddb-$libmaxminddb_version" +./configure --prefix="$INSTALL_PREFIX_EMBEDDED" +make -j"$NPROC" +make install DESTDIR="$STAGE_DIR" +find "$STAGE_EMBEDDED_DIR/bin/" -name "mmdblookup" -exec chrpath -d {} \; + +stamp diff --git a/tasks/deps/openresty b/tasks/deps/openresty index d7c5c345d..4244aa068 100755 --- a/tasks/deps/openresty +++ b/tasks/deps/openresty @@ -1,5 +1,7 @@ #!/usr/bin/env bash +ngx_http_geoip2_module_version="3.2" +ngx_http_geoip2_module_hash="2e60d26460d178068f7a4e9a4e64990a" openresty_version="1.13.6.2" openresty_hash="d95bc4bbe15e4b045a0593b4ecc0db38" openssl_version="1.0.2q" @@ -21,6 +23,9 @@ extract_download "openssl-$openssl_version.tar.gz" download "https://openresty.org/download/openresty-$openresty_version.tar.gz" "md5" "$openresty_hash" extract_download "openresty-$openresty_version.tar.gz" +download "https://github.com/leev/ngx_http_geoip2_module/archive/$ngx_http_geoip2_module_version.tar.gz" "md5" "$ngx_http_geoip2_module_hash" +extract_download "$ngx_http_geoip2_module_version.tar.gz" + cd "openresty-$openresty_version" patch -p1 < "$SOURCE_DIR/build/patches/openresty-opm.patch" patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch" @@ -36,12 +41,12 @@ patch -p1 < "$SOURCE_DIR/build/patches/openresty-cli.patch" --with-pcre-opt=-g \ --with-pcre-conf-opt=--enable-unicode-properties \ --with-pcre-jit \ - --with-http_geoip_module \ --with-http_gunzip_module \ --with-http_gzip_static_module \ --with-http_realip_module \ --with-http_ssl_module \ - --with-http_stub_status_module + --with-http_stub_status_module \ + --add-module="../ngx_http_geoip2_module-$ngx_http_geoip2_module_version" make -j"$NPROC" make install DESTDIR="$STAGE_DIR" chrpath -d "$STAGE_EMBEDDED_DIR/openresty/nginx/sbin/nginx" diff --git a/templates/etc/nginx/router.conf.mustache b/templates/etc/nginx/router.conf.mustache index 8c34a1c3f..6ec7e1bfa 100644 --- a/templates/etc/nginx/router.conf.mustache +++ b/templates/etc/nginx/router.conf.mustache @@ -71,7 +71,13 @@ http { resolver_timeout 12s; {{/dns_resolver._nameservers_nginx}} - geoip_city {{db_dir}}/geoip/city-v6.dat; + geoip2 {{db_dir}}/geoip/GeoLite2-City.mmdb { + $geoip2_data_country_code country iso_code; + $geoip2_data_subdivision_code subdivisions 0 iso_code; + $geoip2_data_city_name city names en; + $geoip2_data_latitude location latitude; + $geoip2_data_longitude location longitude; + } geo $banned_ip { default 0; diff --git a/test/processes/test_rpaths.rb b/test/processes/test_rpaths.rb index 88a8bf160..78704f207 100644 --- a/test/processes/test_rpaths.rb +++ b/test/processes/test_rpaths.rb @@ -38,7 +38,7 @@ def test_binary_rpaths "/embedded/openresty/nginx/sbin/nginx", "/embedded/libexec/trafficserver/ssl_cert_loader.so", # LuaRock - "/embedded/apps/core/shared/vendor/lua/lib/lua/5.1/iconv.so", + "/embedded/apps/core/shared/vendor/lua/lib/lua/5.1/yaml.so", # Rubygem "/oj/oj.so", ].each do |expected_path_end| diff --git a/test/proxy/logging/test_ip_geocoding.rb b/test/proxy/logging/test_ip_geocoding.rb index 2cc51255f..efc8e2314 100644 --- a/test/proxy/logging/test_ip_geocoding.rb +++ b/test/proxy/logging/test_ip_geocoding.rb @@ -80,13 +80,32 @@ def test_country_city_no_region :ip => "104.250.168.24", :country => "MC", :region => nil, - :city => "Monte-carlo", + :city => "Monte Carlo", :lat => 43.7333, :lon => 7.4167, }) end def test_country_no_region_city + response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({ + :headers => { + "X-Forwarded-For" => "1.1.1.1", + }, + })) + assert_response_code(200, response) + + record = wait_for_log(response)[:hit_source] + assert_geocode(record, { + :ip => "1.1.1.1", + :country => "AU", + :region => nil, + :city => nil, + :lat => -33.494, + :lon => 143.2104, + }) + end + + def test_no_country_region_city response = Typhoeus.get("http://127.0.0.1:9080/api/hello", log_http_options.deep_merge({ :headers => { "X-Forwarded-For" => "67.43.156.1", @@ -97,11 +116,11 @@ def test_country_no_region_city record = wait_for_log(response)[:hit_source] assert_geocode(record, { :ip => "67.43.156.1", - :country => "A1", + :country => nil, :region => nil, :city => nil, - :lat => 0.0, - :lon => 0.0, + :lat => nil, + :lon => nil, }) end @@ -118,7 +137,7 @@ def test_city_accent_chars :ip => "184.148.224.214", :country => "CA", :region => "QC", - :city => "Trois-rivières", + :city => "Trois-Rivières", :lat => 46.316, :lon => -72.6833, }) @@ -128,12 +147,19 @@ def test_city_accent_chars def assert_geocode(record, options) assert_geocode_log(record, options) - assert_geocode_cache(record, options) + if !options.fetch(:lat).nil? || !options.fetch(:lon).nil? + assert_geocode_cache(record, options) + end end def assert_geocode_log(record, options) assert_equal(options.fetch(:ip), record.fetch("request_ip")) - assert_equal(options.fetch(:country), record.fetch("request_ip_country")) + if(options.fetch(:country).nil?) + assert_nil(record["request_ip_country"]) + refute(record.key?("request_ip_country")) + else + assert_equal(options.fetch(:country), record.fetch("request_ip_country")) + end if(options.fetch(:region).nil?) assert_nil(record["request_ip_region"]) refute(record.key?("request_ip_region"))