Redirect all web content to HTTPS by default.

This adjusts the existing config options so that API Umbrella redirects
all web-app and website requests to the HTTPS version by default (while
we had configured most of our instances to do this already, it seems
time to make this the default behavior so there's not the weird
page-specific split in behavior).

It also adds a new configuration option so that any "not found"
responses are redirected to HTTPS.

API behavior remains the same (we default to forcing HTTPS, but not via
redirects).

See 18F/api.data.gov#430 for details about the
"not found" redirecting.

config/default.yml

@@ -130,8 +130,9 @@ router:
     ip_connections_log_level: error
   web_app_host: "*"
   web_app_backend_regex: "^/(admin|admins|web-assets)(/|$)"
-  web_app_backend_required_https_regex: "^/(admin|admins)(/|$)"
-  website_backend_required_https_regex_default: "^/(account|signup|contact)(/|$)"
+  web_app_backend_required_https_regex: "^.*"
+  website_backend_required_https_regex_default: "^.*"
+  redirect_not_found_to_https: true
 rsyslog:
   host: 127.0.0.1
   port: 14014

src/api-umbrella/proxy/error_handler.lua

@@ -1,5 +1,6 @@
-local is_hash = require "api-umbrella.utils.is_hash"
 local deep_merge_overwrite_arrays = require "api-umbrella.utils.deep_merge_overwrite_arrays"
+local httpsify_current_url = require "api-umbrella.utils.httpsify_current_url"
+local is_hash = require "api-umbrella.utils.is_hash"
 local lustache = require "lustache"
 local mustache_unescape = require "api-umbrella.utils.mustache_unescape"
 local path = require "pl.path"
@@ -122,6 +123,20 @@ local function render_template(template, data, format, strip_whitespace)
 end
 
 return function(denied_code, settings, extra_data)
+  -- Redirect "not_found" errors to HTTPS.
+  --
+  -- Since these errors aren't subject to an API Backend's HTTPS requirements
+  -- (where we might return the "https_required" error), this helps ensure that
+  -- requests to unknown location (neither API or website backend) are
+  -- redirected to HTTPS like the rest of our non-API content. This ensures
+  -- HTTPS redirects are in place for the root request on custom domains
+  -- without a website or API at the root.
+  if denied_code == "not_found" and config["router"]["redirect_not_found_to_https"] then
+    if ngx.ctx.protocol ~= "https" then
+      return ngx.redirect(httpsify_current_url(), ngx.HTTP_MOVED_PERMANENTLY)
+    end
+  end
+
   -- Store the gatekeeper rejection code for logging.
   ngx.ctx.gatekeeper_denied_code = denied_code
 

test/processes/test_reloads.rb

@@ -171,7 +171,7 @@ def test_file_based_config_changes_updates_templates
   end
 
   def test_file_based_config_changes_updates_apis
-    response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/file-config/info/", http_options)
+    response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/file-config/info/", http_options)
     assert_response_code(404, response)
 
     override_config({
@@ -187,13 +187,13 @@ def test_file_based_config_changes_updates_apis
         },
       ],
     }, "--router") do
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/file-config/info/", http_options)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/file-config/info/", http_options)
       assert_response_code(200, response)
       data = MultiJson.load(response.body)
       assert_equal(data["headers"]["x-test-file-config"], "foo")
     end
 
-    response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/file-config/info/", http_options)
+    response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/file-config/info/", http_options)
     assert_response_code(404, response)
   end
 

test/proxy/logging/test_basics.rb

@@ -510,7 +510,7 @@ def test_case_sensitivity
   end
 
   def test_does_not_website_backend_requests
-    response = Typhoeus.get("http://127.0.0.1:9080/", log_http_options)
+    response = Typhoeus.get("https://127.0.0.1:9081/", log_http_options)
     assert_response_code(200, response)
 
     error = assert_raises Timeout::Error do

test/proxy/routing/test_https_config.rb

@@ -0,0 +1,58 @@
+require_relative "../../test_helper"
+
+class Test::Proxy::Routing::TestHttpsConfig < Minitest::Test
+  include ApiUmbrellaTestHelpers::Setup
+  include ApiUmbrellaTestHelpers::AdminAuth
+  include Minitest::Hooks
+
+  def setup
+    super
+    setup_server
+    once_per_class_setup do
+      override_config_set({
+        "router" => {
+          "web_app_backend_required_https_regex" => "^/admin/web-app-https-test",
+          "website_backend_required_https_regex_default" => "^/website-https-test",
+          "redirect_not_found_to_https" => false,
+          "web_app_host" => "127.0.0.1",
+        },
+      }, "--router")
+    end
+  end
+
+  def after_all
+    super
+    override_config_reset("--router")
+  end
+
+  def test_custom_web_app_regex
+    response = Typhoeus.get("http://127.0.0.1:9080/admin/", keyless_http_options)
+    assert_response_code(200, response)
+    assert_match(%r{<script src="assets/api-umbrella-admin-ui-\w+\.js"}, response.body)
+
+    response = Typhoeus.get("http://127.0.0.1:9080/admin/web-app-https-test", keyless_http_options)
+    assert_response_code(301, response)
+    assert_equal("https://127.0.0.1:9081/admin/web-app-https-test", response.headers["Location"])
+  end
+
+  def test_custom_website_backend_regex
+    response = Typhoeus.get("http://127.0.0.1:9080/", keyless_http_options)
+    assert_response_code(200, response)
+    assert_match("Your API Site Name", response.body)
+
+    response = Typhoeus.get("http://127.0.0.1:9080/website-https-test", keyless_http_options)
+    assert_response_code(301, response)
+    assert_equal("https://127.0.0.1:9081/website-https-test", response.headers["Location"])
+  end
+
+  def test_not_found_https_disabled
+    response = Typhoeus.get("http://127.0.0.1:9080/api-umbrella/v1/state.json", http_options.deep_merge(admin_token).deep_merge({
+      :headers => {
+        "Host" => "#{unique_test_id}-unknown.foo",
+      },
+    }))
+    assert_response_code(404, response)
+    assert_equal("application/json", response.headers["content-type"])
+    assert_match("NOT_FOUND", response.body)
+  end
+end

test/proxy/routing/test_website.rb

@@ -10,7 +10,7 @@ def setup
   end
 
   def test_default_website
-    response = Typhoeus.get("http://127.0.0.1:9080/", keyless_http_options)
+    response = Typhoeus.get("https://127.0.0.1:9081/", keyless_http_options)
     assert_response_code(200, response)
     assert_match("Your API Site Name", response.body)
 
@@ -19,7 +19,11 @@ def test_default_website
     assert_match("API Key Signup", response.body)
   end
 
-  def test_signup_https_redirect
+  def test_https_redirect
+    response = Typhoeus.get("http://127.0.0.1:9080/", keyless_http_options)
+    assert_response_code(301, response)
+    assert_equal("https://127.0.0.1:9081/", response.headers["location"])
+
     response = Typhoeus.get("http://127.0.0.1:9080/signup/", keyless_http_options)
     assert_response_code(301, response)
     assert_equal("https://127.0.0.1:9081/signup/", response.headers["location"])

test/proxy/test_forwarded_port_headers.rb

@@ -46,8 +46,11 @@ def test_default_headers
       when :admin_oauth2_http
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/admins/auth/google_oauth2", response.headers["Location"])
-      when :website_https, :website_http
+      when :website_https
         assert_response_code(200, response)
+      when :website_http
+        assert_response_code(301, response)
+        assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
       when :website_signup_https
         assert_response_code(200, response)
       when :website_signup_http
@@ -75,8 +78,11 @@ def test_forwarded_port
       when :admin_oauth2_http
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/admins/auth/google_oauth2", response.headers["Location"])
-      when :website_https, :website_http
+      when :website_https
         assert_response_code(200, response)
+      when :website_http
+        assert_response_code(301, response)
+        assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
       when :website_signup_https
         assert_response_code(200, response)
       when :website_signup_http
@@ -100,7 +106,8 @@ def test_forwarded_proto_http
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/admins/auth/google_oauth2", response.headers["Location"])
       when :website_https, :website_http
-        assert_response_code(200, response)
+        assert_response_code(301, response)
+        assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
       when :website_signup_https, :website_signup_http
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/signup/", response.headers["Location"])
@@ -145,7 +152,8 @@ def test_forwarded_proto_http_and_port
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/admins/auth/google_oauth2", response.headers["Location"])
       when :website_https, :website_http
-        assert_response_code(200, response)
+        assert_response_code(301, response)
+        assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
       when :website_signup_https, :website_signup_http
         assert_response_code(301, response)
         assert_equal("https://127.0.0.1:9081/signup/", response.headers["Location"])
@@ -194,8 +202,11 @@ def test_override_public_http_port
         when :admin_oauth2_http
           assert_response_code(301, response)
           assert_equal("https://127.0.0.1:9081/admins/auth/google_oauth2", response.headers["Location"])
-        when :website_https, :website_http
+        when :website_https
           assert_response_code(200, response)
+        when :website_http
+          assert_response_code(301, response)
+          assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
         when :website_signup_https
           assert_response_code(200, response)
         when :website_signup_http
@@ -227,8 +238,11 @@ def test_override_public_https_port
         when :admin_oauth2_http
           assert_response_code(301, response)
           assert_equal("https://127.0.0.1:3333/admins/auth/google_oauth2", response.headers["Location"])
-        when :website_https, :website_http
+        when :website_https
           assert_response_code(200, response)
+        when :website_http
+          assert_response_code(301, response)
+          assert_equal("https://127.0.0.1:3333/", response.headers["Location"])
         when :website_signup_https
           assert_response_code(200, response)
         when :website_signup_http
@@ -260,7 +274,10 @@ def test_override_public_http_proto
         when :admin_oauth2_http
           assert_response_code(302, response)
           assert_oauth2_redirect_uri("https://127.0.0.1:9080/admins/auth/google_oauth2/callback", response)
-        when :website_https, :website_http
+        when :website_https
+          assert_response_code(301, response)
+          assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
+        when :website_http
           assert_response_code(200, response)
         when :website_signup_https
           assert_response_code(301, response)
@@ -293,7 +310,10 @@ def test_override_public_https_proto
         when :admin_oauth2_http
           assert_response_code(302, response)
           assert_oauth2_redirect_uri("https://127.0.0.1:9080/admins/auth/google_oauth2/callback", response)
-        when :website_https, :website_http
+        when :website_https
+          assert_response_code(301, response)
+          assert_equal("https://127.0.0.1:9081/", response.headers["Location"])
+        when :website_http
           assert_response_code(200, response)
         when :website_signup_https
           assert_response_code(301, response)
@@ -327,8 +347,11 @@ def test_override_public_ports_defaults
         when :admin_oauth2_http
           assert_response_code(301, response)
           assert_equal("https://127.0.0.1/admins/auth/google_oauth2", response.headers["Location"])
-        when :website_https, :website_http
+        when :website_https
           assert_response_code(200, response)
+        when :website_http
+          assert_response_code(301, response)
+          assert_equal("https://127.0.0.1/", response.headers["Location"])
         when :website_signup_https
           assert_response_code(200, response)
         when :website_signup_http

test/proxy/test_nginx_rewrites.rb

@@ -25,12 +25,12 @@ def test_basic_rewrites
       ],
     }, "--router") do
       # Basic rewrite
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options)
       assert_response_code(301, response)
       assert_equal("https://example.com/something/?foo=bar", response.headers["location"])
 
       # Advanced with replacements rewrite
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite/el/7/file-0.6.0-1.el7.x86_64.rpm?foo=bar", http_options)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite/el/7/file-0.6.0-1.el7.x86_64.rpm?foo=bar", http_options)
       assert_response_code(302, response)
       assert_equal("https://example.com/downloads/v0.6.0/file-0.6.0-1.el7.x86_64.rpm", response.headers["location"])
     end
@@ -52,20 +52,20 @@ def test_default_host
       ],
     }, "--router") do
       # Known host without rewrites
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
         :headers => { "Host" => "default.foo" },
       }))
       assert_response_code(301, response)
       assert_equal("https://example.com/something/?foo=bar", response.headers["location"])
 
       # Known host without rewrites
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
         :headers => { "Host" => "known.foo" },
       }))
       assert_response_code(404, response)
 
       # Unknown host
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
         :headers => { "Host" => "unknown.foo" },
       }))
       assert_response_code(301, response)
@@ -85,14 +85,14 @@ def test_no_default_host
       ],
     }, "--router") do
       # Known host without rewrites
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
         :headers => { "Host" => "default.foo" },
       }))
       assert_response_code(301, response)
       assert_equal("https://example.com/something/?foo=bar", response.headers["location"])
 
       # Unknown host
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/hello/rewrite?foo=bar", http_options.deep_merge({
         :headers => { "Host" => "unknown.foo" },
       }))
       assert_response_code(404, response)
@@ -133,18 +133,18 @@ def test_precedence
       })
 
       # Rewrites match before API Backends.
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/api-example/rewrite_me", http_opts)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/api-example/rewrite_me", http_opts)
       assert_response_code(301, response)
       assert_equal("https://example.com/", response.headers["location"])
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/api-example/rewrite_me_just_kidding", http_opts)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/api-example/rewrite_me_just_kidding", http_opts)
       assert_response_code(200, response)
       assert_match("Hello World", response.body)
 
       # Rewrites match before Website Backends.
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/website-example/rewrite_me", http_opts)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/website-example/rewrite_me", http_opts)
       assert_response_code(301, response)
       assert_equal("https://2.example.com/", response.headers["location"])
-      response = Typhoeus.get("http://127.0.0.1:9080/#{unique_test_id}/website-example/rewrite_me_just_kidding", http_opts)
+      response = Typhoeus.get("https://127.0.0.1:9081/#{unique_test_id}/website-example/rewrite_me_just_kidding", http_opts)
       assert_response_code(404, response)
       assert_match("Test 404 Not Found", response.body)