Skip to content

Commit

Permalink
Tweaks to deal with merges and get some more tests running.
Browse files Browse the repository at this point in the history
- Restore the old elasticsearch v1 template for running with
  Elasticsearch v2.
- Update some dependencies.
- Restore some things that were accidentally removed in merging.
- Update some of the newer tests from master to work on the updated
  gems.
  • Loading branch information
GUI committed Jun 7, 2018
1 parent a291e9b commit 37825a0
Show file tree
Hide file tree
Showing 14 changed files with 248 additions and 80 deletions.
4 changes: 2 additions & 2 deletions Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ gem "oj", "~> 3.6.2"

# Database libraries
gem "mongoid", "~> 7.0.1"
gem "elasticsearch", "~> 6.0.0"
gem "active_attr", "~> 0.10.2"
gem "elasticsearch", "~> 6.0.2"
gem "active_attr", "~> 0.11.0"

# Factories for test database data
gem "factory_bot", "~> 4.10.0"
Expand Down
20 changes: 10 additions & 10 deletions Gemfile.lock
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
GEM
remote: https://rubygems.org/
specs:
active_attr (0.10.2)
activemodel (>= 3.0.2, < 5.2)
activesupport (>= 3.0.2, < 5.2)
active_attr (0.11.0)
activemodel (>= 3.0.2, < 6.0)
activesupport (>= 3.0.2, < 6.0)
activemodel (5.2.0)
activesupport (= 5.2.0)
activesupport (5.2.0)
Expand Down Expand Up @@ -34,12 +34,12 @@ GEM
cliver (0.3.2)
concurrent-ruby (1.0.5)
database_cleaner (1.7.0)
elasticsearch (6.0.0)
elasticsearch-api (= 6.0.0)
elasticsearch-transport (= 6.0.0)
elasticsearch-api (6.0.0)
elasticsearch (6.0.2)
elasticsearch-api (= 6.0.2)
elasticsearch-transport (= 6.0.2)
elasticsearch-api (6.0.2)
multi_json
elasticsearch-transport (6.0.0)
elasticsearch-transport (6.0.2)
faraday
multi_json
ethon (0.11.0)
Expand Down Expand Up @@ -123,7 +123,7 @@ PLATFORMS
ruby

DEPENDENCIES
active_attr (~> 0.10.2)
active_attr (~> 0.11.0)
activesupport (~> 5.2.0)
addressable (~> 2.5.2)
awesome_print (~> 1.8.0)
Expand All @@ -133,7 +133,7 @@ DEPENDENCIES
childprocess (~> 0.9.0)
concurrent-ruby (~> 1.0.5)
database_cleaner (~> 1.7.0)
elasticsearch (~> 6.0.0)
elasticsearch (~> 6.0.2)
factory_bot (~> 4.10.0)
faker (~> 1.8.7)
lazyhash (~> 0.1.1)
Expand Down
42 changes: 11 additions & 31 deletions config/elasticsearch_templates_v1.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
[
{
"id": "api-umbrella-log-v2-template",
"id": "api-umbrella-log-v1-template",
"template": {
"template": "api-umbrella-logs-v2-*",
"template": "api-umbrella-logs-v1-*",
"settings": {
"index": {
"number_of_shards": 1,
Expand All @@ -12,7 +12,11 @@
"normalizer": {
"lowercase_normalizer": {
"type": "custom",
"filter": ["lowercase", "asciifolding"]
"filter": ["lowercase"]
},
"uppercase_normalizer": {
"type": "custom",
"filter": ["uppercase"]
}
}
},
Expand Down Expand Up @@ -85,15 +89,15 @@
},
"request_ip_country": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
"normalizer": "uppercase_normalizer"
},
"request_ip_region": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
"normalizer": "uppercase_normalizer"
},
"request_method": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
"normalizer": "uppercase_normalizer"
},
"request_origin": {
"type": "keyword",
Expand All @@ -114,31 +118,7 @@
"request_size": {
"type": "integer"
},
"request_url_hierarchy_level0": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level1": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level2": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level3": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level4": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level5": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"request_url_hierarchy_level6": {
"request_url": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
Expand Down
204 changes: 204 additions & 0 deletions config/elasticsearch_templates_v1_es2x.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
[
{
"id": "api-umbrella-log-v1-template",
"template": {
"template": "api-umbrella-logs-v1-*",
"settings": {
"index": {
"number_of_shards": 1,
"codec": "best_compression"
},
"analysis": {
"analyzer": {
"keyword_lowercase": {
"type": "custom",
"tokenizer": "keyword",
"filter": ["lowercase"]
}
}
},
"translog": {
"durability": "async",
"sync_interval": "10s"
}
},
"mappings": {
"log": {
"_all": {
"enabled": false
},
"date_detection": false,
"numeric_detection": false,
"dynamic_templates": [
{
"string_template": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "string",
"index": "analyzed",
"analyzer": "keyword_lowercase"
}
}
}
],
"properties": {
"api_backend_id": {
"type": "string",
"index": "not_analyzed"
},
"api_backend_url_match_id": {
"type": "string",
"index": "not_analyzed"
},
"api_key": {
"type": "string",
"index": "not_analyzed"
},
"gatekeeper_denied_code": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_accept": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_accept_encoding": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_at": {
"type": "date"
},
"request_basic_auth_username": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_connection": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_content_type": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_hierarchy": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_host": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_ip": {
"type": "string",
"index": "not_analyzed"
},
"request_ip_city": {
"type": "string",
"index": "not_analyzed"
},
"request_ip_country": {
"type": "string",
"index": "not_analyzed"
},
"request_ip_region": {
"type": "string",
"index": "not_analyzed"
},
"request_method": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_origin": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_path": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_referer": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_scheme": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_size": {
"type": "integer"
},
"request_url": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_url_query": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_user_agent": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_user_agent_family": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"request_user_agent_type": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"response_age": {
"type": "integer"
},
"response_cache": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"response_content_encoding": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"response_content_length": {
"type": "integer"
},
"response_content_type": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"response_server": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"response_size": {
"type": "integer"
},
"response_status": {
"type": "short"
},
"response_time": {
"type": "integer"
},
"response_transfer_encoding": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"user_id": {
"type": "string",
"index": "not_analyzed"
},
"user_email": {
"type": "string",
"analyzer": "keyword_lowercase"
},
"user_registration_source": {
"type": "string",
"analyzer": "keyword_lowercase"
}
}
}
}
}
}
]
1 change: 1 addition & 0 deletions src/api-umbrella/cli/read_config.lua
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ local function set_computed_config()
["_service_elasticsearch_aws_signing_proxy_enabled?"] = array_includes(config["services"], "elasticsearch_aws_signing_proxy"),
["_service_router_enabled?"] = array_includes(config["services"], "router"),
["_service_web_enabled?"] = array_includes(config["services"], "web"),
["_service_nginx_reloader_enabled?"] = (array_includes(config["services"], "router") and config["nginx"]["_reloader_frequency"]),
router = {
trusted_proxies = trusted_proxies,
},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
local cjson = require "cjson"

local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/elasticsearch_templates_v" .. config["elasticsearch"]["template_version"] .. ".json"
local path = os.getenv("API_UMBRELLA_SRC_ROOT") .. "/config/elasticsearch_templates_v" .. config["elasticsearch"]["template_version"] .. "_es2x.json"
local f, err = io.open(path, "rb")
if err then
ngx.log(ngx.ERR, "failed to open file: ", err)
Expand Down
2 changes: 1 addition & 1 deletion src/api-umbrella/web-app/Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ gem "mongoid_rails_migrations", "~> 1.1.0"
gem "mongoid_store", :git => "https://github.com/GUI/mongoid_store.git"

# Elasticsearch
gem "elasticsearch", "~> 5.0.1"
gem "elasticsearch", "~> 6.0.2"

# OmniAuth-based authentication
gem "devise", "~> 4.4.3"
Expand Down
12 changes: 6 additions & 6 deletions src/api-umbrella/web-app/Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,12 @@ GEM
warden (~> 1.2.3)
devise-i18n (1.6.2)
devise (>= 4.4)
elasticsearch (5.0.1)
elasticsearch-api (= 5.0.1)
elasticsearch-transport (= 5.0.1)
elasticsearch-api (5.0.1)
elasticsearch (6.0.2)
elasticsearch-api (= 6.0.2)
elasticsearch-transport (= 6.0.2)
elasticsearch-api (6.0.2)
multi_json
elasticsearch-transport (5.0.1)
elasticsearch-transport (6.0.2)
faraday
multi_json
erubis (2.7.0)
Expand Down Expand Up @@ -317,7 +317,7 @@ DEPENDENCIES
delayed_job_mongoid (~> 2.3.0)
devise (~> 4.4.3)
devise-i18n (~> 1.6.0)
elasticsearch (~> 5.0.1)
elasticsearch (~> 6.0.2)
font-awesome-rails (~> 4.7.0)
http_accept_language (~> 2.1.1)
i18n-js (~> 3.0.8)
Expand Down
Loading

0 comments on commit 37825a0

Please sign in to comment.