Tweaks to get separate web-app and admin-ui building during tests.

CMakeLists.txt

@@ -33,6 +33,10 @@ set(VENDOR_DIR ${WORK_DIR}/vendor)
 set(VENDOR_LUA_DIR ${VENDOR_DIR}/share/lua/5.1)
 set(LUAROCKS_CMD env LUA_PATH=${STAGE_EMBEDDED_DIR}/openresty/luajit/share/lua/5.1/?.lua$<SEMICOLON>${STAGE_EMBEDDED_DIR}/openresty/luajit/share/lua/5.1/?/init.lua$<SEMICOLON>$<SEMICOLON> ${STAGE_EMBEDDED_DIR}/bin/luarocks)
 
+# Where to install development-only dependencies.
+set(DEV_INSTALL_PREFIX ${WORK_DIR}/dev-env)
+set(DEV_VENDOR_DIR ${DEV_INSTALL_PREFIX}/vendor)
+
 # Where to install test-only dependencies.
 set(TEST_INSTALL_PREFIX ${WORK_DIR}/test-env)
 set(TEST_VENDOR_DIR ${TEST_INSTALL_PREFIX}/vendor)
@@ -59,6 +63,7 @@ pkg_search_module(LIBFFI REQUIRED libffi)
 require_program(rsync)
 
 include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake)
+include(${CMAKE_SOURCE_DIR}/build/cmake/dev/nodejs.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/elasticsearch.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/libcidr.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/libgeoip.cmake)
@@ -72,8 +77,6 @@ include(${CMAKE_SOURCE_DIR}/build/cmake/ruby.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/rsyslog.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/trafficserver.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/static-site.cmake)
-include(${CMAKE_SOURCE_DIR}/build/cmake/core-lua-deps.cmake)
-include(${CMAKE_SOURCE_DIR}/build/cmake/core-web-app.cmake)
 include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake)
 if(ENABLE_HADOOP_ANALYTICS)
   include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/flume.cmake)

build/cmake/core-admin-ui.cmake

@@ -0,0 +1,48 @@
+file(GLOB_RECURSE admin_ui_files
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.hbs
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.html
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.js
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.scss
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/config/*.js
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/lib/*.js
+)
+add_custom_command(
+  OUTPUT
+    ${STAMP_DIR}/core-admin-ui-build-dir
+    ${CORE_BUILD_DIR}/tmp/admin-ui-build/bower.json
+    ${CORE_BUILD_DIR}/tmp/admin-ui-build/package.json
+  DEPENDS
+    ${admin_ui_files}
+    ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/bower.json
+    ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/package.json
+  COMMAND mkdir -p ${CORE_BUILD_DIR}/tmp/admin-ui-build
+  COMMAND rsync -a -v --delete-after "--filter=:- ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/.gitignore" --exclude=/dist-prod --exclude=/dist-dev ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/ ${CORE_BUILD_DIR}/tmp/admin-ui-build/
+  COMMAND touch ${STAMP_DIR}/core-admin-ui-build-dir
+)
+
+add_custom_command(
+  OUTPUT ${STAMP_DIR}/core-admin-ui-npm-install
+  DEPENDS ${CORE_BUILD_DIR}/tmp/admin-ui-build/package.json
+  COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} npm install && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} npm prune
+  COMMAND touch ${STAMP_DIR}/core-admin-ui-npm-install
+)
+
+add_custom_command(
+  OUTPUT ${STAMP_DIR}/core-admin-ui-bower-install
+  DEPENDS
+    ${CORE_BUILD_DIR}/tmp/admin-ui-build/bower.json
+    ${STAMP_DIR}/core-admin-ui-npm-install
+  COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} ./node_modules/.bin/bower install && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} ./node_modules/.bin/bower prune
+  COMMAND touch ${STAMP_DIR}/core-admin-ui-bower-install
+)
+
+add_custom_command(
+  OUTPUT ${STAMP_DIR}/core-admin-ui-build
+  DEPENDS
+    ${STAMP_DIR}/core-admin-ui-build-dir
+    ${STAMP_DIR}/core-admin-ui-npm-install
+    ${STAMP_DIR}/core-admin-ui-bower-install
+  COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && time env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} ./node_modules/.bin/ember build --environment=development --output-path=./dist-dev
+  COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && time env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} ./node_modules/.bin/ember build --environment=production --output-path=./dist-prod
+  COMMAND touch ${STAMP_DIR}/core-admin-ui-build
+)

build/cmake/core-web-app.cmake

@@ -8,6 +8,23 @@ add_custom_command(
   COMMAND touch ${STAMP_DIR}/core-web-app-bundle
 )
 
+file(GLOB_RECURSE web_asset_files
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.css
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.erb
+  ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.js
+)
+add_custom_command(
+  OUTPUT ${STAMP_DIR}/core-web-app-precompile
+  DEPENDS
+    ${STAMP_DIR}/core-web-app-bundle
+    ${web_asset_files}
+    ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/config/initializers/assets.rb
+    ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile
+    ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile.lock
+  COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/web-app/.bundle RAILS_TMP_PATH=/tmp/web-app-build RAILS_PUBLIC_PATH=${CORE_BUILD_DIR}/tmp/web-app-build bundle exec rake -f ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Rakefile assets:clobber assets:precompile
+  COMMAND touch ${STAMP_DIR}/core-web-app-precompile
+)
+
 # Normally we perform the bundle out-of-source (so the build takes place
 # entirely out of source), but if testing/development is enabled for this
 # build, then also create a local ".bundle/config" item within the source. This

build/cmake/core.cmake

@@ -1,5 +1,9 @@
 set(CORE_BUILD_DIR ${WORK_DIR}/src/api-umbrella-core)
 
+include(${CMAKE_SOURCE_DIR}/build/cmake/core-lua-deps.cmake)
+include(${CMAKE_SOURCE_DIR}/build/cmake/core-web-app.cmake)
+include(${CMAKE_SOURCE_DIR}/build/cmake/core-admin-ui.cmake)
+
 # Copy the vendored libraries into the shared build directory.
 add_custom_command(
   OUTPUT ${CORE_BUILD_DIR}/shared/vendor
@@ -11,48 +15,31 @@ add_custom_command(
   COMMAND touch -c ${CORE_BUILD_DIR}/shared/vendor
 )
 
-# Create the tmp directories in the shared build directory.
-#
-# We create these more specific tmp sub directories so the deb/rpm
-# after-install script can set the necessary permissions on these sub
-# directories to allow for deployments of master on top of a package install.
-add_custom_command(
-  OUTPUT ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/assets
-  COMMAND mkdir -p ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/assets
-  COMMAND touch -c ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/assets
-)
-add_custom_command(
-  OUTPUT ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/sass
-  COMMAND mkdir -p ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/sass
-  COMMAND touch -c ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/sass
-)
-add_custom_command(
-  OUTPUT ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/ember-rails
-  COMMAND mkdir -p ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/ember-rails
-  COMMAND touch -c ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/ember-rails
-)
-
-#
-# Build the shared dir.
-#
-add_custom_command(
-  OUTPUT ${STAMP_DIR}/core-build-shared
-  DEPENDS
-    ${CORE_BUILD_DIR}/shared/vendor
-    ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/assets
-    ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/cache/sass
-    ${CORE_BUILD_DIR}/shared/src/api-umbrella/web-app/tmp/ember-rails
-  COMMAND touch ${STAMP_DIR}/core-build-shared
-)
-
 # Copy the code into a build release directory.
+file(GLOB_RECURSE core_files
+  ${CMAKE_SOURCE_DIR}/bin/*
+  ${CMAKE_SOURCE_DIR}/config/*
+  ${CMAKE_SOURCE_DIR}/templates/*
+)
 add_custom_command(
   OUTPUT ${STAMP_DIR}/core-build-release-dir
+  DEPENDS ${core_files}
   COMMAND mkdir -p ${CORE_BUILD_DIR}/releases/0
   COMMAND rsync -a --delete-after --delete-excluded "--filter=:- ${CMAKE_SOURCE_DIR}/.gitignore" --include=/templates/etc/perp/.boot --exclude=.* --exclude=/templates/etc/test-env* --exclude=/templates/etc/perp/test-env* --exclude=/src/api-umbrella/web-app/spec --exclude=/src/api-umbrella/web-app/app/assets --exclude=/src/api-umbrella/hadoop-analytics --include=/bin/*** --include=/config/*** --include=/LICENSE.txt --include=/templates/*** --include=/src/*** --exclude=* ${CMAKE_SOURCE_DIR}/ ${CORE_BUILD_DIR}/releases/0/
   COMMAND touch ${STAMP_DIR}/core-build-release-dir
 )
 
+add_custom_command(
+  OUTPUT ${STAMP_DIR}/core-build-install-dist
+  DEPENDS ${STAMP_DIR}/core-admin-ui-build
+  DEPENDS ${STAMP_DIR}/core-web-app-precompile
+  COMMAND mkdir -p ${CORE_BUILD_DIR}/releases/0/build/dist
+  COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/tmp/web-app-build/web-assets/ ${CORE_BUILD_DIR}/releases/0/build/dist/web-app-assets/
+  COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/tmp/admin-ui-build/dist-dev/ ${CORE_BUILD_DIR}/releases/0/build/dist/admin-ui-dev/
+  COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/tmp/admin-ui-build/dist-prod/ ${CORE_BUILD_DIR}/releases/0/build/dist/admin-ui/
+  COMMAND touch ${STAMP_DIR}/core-build-install-dist
+)
+
 # Create a symlink to the latest release.
 add_custom_command(
   OUTPUT ${STAMP_DIR}/core-build-current-symlink
@@ -87,17 +74,6 @@ add_custom_command(
   COMMAND touch -c ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app/.bundle/config
 )
 
-# Create a symlink to the shared web-app tmp directory within the release.
-add_custom_command(
-  OUTPUT ${STAMP_DIR}/core-build-release-web-tmp-symlink
-  DEPENDS
-    ${STAMP_DIR}/core-build-release-dir
-		${STAMP_DIR}/core-build-shared
-  WORKING_DIRECTORY ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app
-  COMMAND ln -snf ../../../../../shared/src/api-umbrella/web-app/tmp ./tmp
-  COMMAND touch ${STAMP_DIR}/core-build-release-web-tmp-symlink
-)
-
 #
 # Build the release dir.
 #
@@ -106,7 +82,6 @@ add_custom_command(
   DEPENDS
     ${STAMP_DIR}/core-build-release-dir
     ${STAMP_DIR}/core-build-release-vendor-symlink
-    ${STAMP_DIR}/core-build-release-web-tmp-symlink
     ${STAMP_DIR}/core-build-current-symlink
     ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app/.bundle/config
   COMMAND touch ${STAMP_DIR}/core-build-release
@@ -115,8 +90,9 @@ add_custom_command(
 # Copy the built shared directory to the stage install path.
 add_custom_command(
   OUTPUT ${STAGE_EMBEDDED_DIR}/apps/core
-  DEPENDS ${STAMP_DIR}/core-build-shared
+  DEPENDS ${CORE_BUILD_DIR}/shared/vendor
   DEPENDS ${STAMP_DIR}/core-build-release
+  DEPENDS ${STAMP_DIR}/core-build-install-dist
   COMMAND mkdir -p ${STAGE_EMBEDDED_DIR}/apps/core
   COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/ ${STAGE_EMBEDDED_DIR}/apps/core/
   COMMAND touch -c ${STAGE_EMBEDDED_DIR}/apps/core

build/cmake/dev/nodejs.cmake

@@ -0,0 +1,9 @@
+# NodeJS: For building admin-ui Ember app.
+ExternalProject_Add(
+  nodejs
+  URL https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-linux-x64.tar.xz
+  URL_HASH SHA256=${NODEJS_HASH}
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  INSTALL_COMMAND rsync -a -v <SOURCE_DIR>/ ${DEV_INSTALL_PREFIX}/
+)

build/cmake/versions.cmake

@@ -73,6 +73,8 @@ set(NGX_DYUPS_VERSION d4b3e053dee10e2879882eb4c346ac7d534e2d14)
 set(NGX_DYUPS_HASH bdf4408599602afa38365a426e126d21)
 set(NGX_TXID_VERSION f1c197cb9c42e364a87fbb28d5508e486592ca42)
 set(NGX_TXID_HASH 408ee86eb6e42e27a51514f711c41d6b)
+set(NODEJS_VERSION 4.6.0)
+set(NODEJS_HASH a77ceb75a05984153304ad0f09b11d234ca54a67714ba575b52e4298df0343d1)
 set(OPENRESTY_VERSION 1.11.2.1)
 set(OPENRESTY_HASH f26d152f40c5263b383a5b7c826a6c7e)
 set(OPENSSL_VERSION 1.0.2j)

src/api-umbrella/admin-ui/package.json

@@ -25,19 +25,17 @@
     "ember-buffered-proxy": "0.6.0",
     "ember-busy-blocker": "0.1.0",
     "ember-cli": "2.8.0",
-    "ember-cli-app-version": "^1.0.0",
     "ember-cli-babel": "^5.1.6",
     "ember-cli-dependency-checker": "^1.2.0",
     "ember-cli-eslint": "3.0.0",
     "ember-cli-format-number": "1.0.2",
     "ember-cli-htmlbars": "^1.0.3",
     "ember-cli-htmlbars-inline-precompile": "^0.3.1",
     "ember-cli-qunit": "^1.4.0",
-    "ember-cli-release": "^0.2.9",
-    "ember-cli-sass": "5.5.1",
+    "ember-cli-sass": "5.5.2",
     "ember-cli-sri": "^2.1.0",
     "ember-cli-uglify": "^1.2.0",
-    "ember-cp-validations": "3.0.1",
+    "ember-cp-validations": "3.1.0",
     "ember-data": "2.8.0",
     "ember-export-application-global": "^1.0.5",
     "ember-load-initializers": "^0.5.1",

src/api-umbrella/cli/read_config.lua

@@ -379,6 +379,10 @@ local function set_computed_config()
     },
   })
 
+  if config["app_env"] == "development" then
+    config["_dev_env_install_dir"] = path.join(src_root_dir, "build/work/dev-env")
+  end
+
   if config["app_env"] == "test" then
     config["_test_env_install_dir"] = path.join(src_root_dir, "build/work/test-env")
   end

src/api-umbrella/cli/setup.lua

@@ -101,7 +101,6 @@ local function prepare()
     path.join(config["etc_dir"], "trafficserver/snapshots"),
     path.join(config["log_dir"], "trafficserver"),
     path.join(config["root_dir"], "var/trafficserver"),
-    path.join(config["_src_root_dir"], "src/api-umbrella/web-app/tmp"),
   }
 
   for _, directory in ipairs(dirs) do
@@ -217,12 +216,6 @@ local function set_permissions()
     os.exit(1)
   end
 
-  _, _, err = run_command("chmod 1777 " .. path.join(config["_src_root_dir"], "src/api-umbrella/web-app/tmp"))
-  if err then
-    print("chmod failed: ", err)
-    os.exit(1)
-  end
-
   if config["user"] and config["group"] then
     _, _, err = run_command("chown -R " .. config["user"] .. ":" .. config["group"] .. " " .. path.join(config["etc_dir"], "trafficserver") .. " " .. path.join(config["root_dir"], "var"))
     if err then

src/api-umbrella/web-app/Gemfile

@@ -72,6 +72,10 @@ gem "omniauth-twitter", "~> 1.2.1", :require => false
 # Authorization
 gem "pundit", "~> 1.1.0"
 
+# Generate non-digest assets for i18n content that  the admin-ui component can
+# link to (without knowing the cache busted URLs).
+gem "non-stupid-digest-assets", "~> 1.0.8"
+
 # Views/templates for APIs
 gem "rabl", "~> 0.13.0"
 gem "jbuilder", "~> 2.6.0"

src/api-umbrella/web-app/Gemfile.lock

@@ -249,6 +249,8 @@ GEM
     nokogiri (1.6.8)
       mini_portile2 (~> 2.1.0)
       pkg-config (~> 1.1.7)
+    non-stupid-digest-assets (1.0.8)
+      sprockets (>= 2.0)
     oauth (0.5.1)
     oauth2 (1.2.0)
       faraday (>= 0.8, < 0.10)
@@ -454,6 +456,7 @@ DEPENDENCIES
   mongoid_rails_migrations (~> 1.1.0)
   multi_json (~> 1.12.1)
   nokogiri (~> 1.6.8)
+  non-stupid-digest-assets (~> 1.0.8)
   oj (~> 2.17.4)
   oj_mimic_json (~> 1.0.1)
   omniauth (~> 1.3.1)

src/api-umbrella/web-app/config/application.rb

@@ -131,15 +131,18 @@ class Application < Rails::Application
     # config.i18n.default_locale = :de
 
     if(ENV["RAILS_TMP_PATH"].present?)
-      paths["tmp"] = ENV["RAILS_TMP_PATH"]
-      tmp_assets_cache_path = File.join(ENV["RAILS_TMP_PATH"], "cache/assets")
-      FileUtils.mkdir_p(tmp_assets_cache_path)
-      config.assets.cache_store = [:file_store, tmp_assets_cache_path]
-      config.sass.cache_location = File.join(ENV["RAILS_TMP_PATH"], "cache/sass")
+      config.paths["tmp"] = ENV["RAILS_TMP_PATH"]
+      config.assets.configure do |env|
+        env.cache = Sprockets::Cache::FileStore.new(
+          File.join(ENV["RAILS_TMP_PATH"], "cache/assets"),
+          config.assets.cache_limit,
+          env.logger
+        )
+      end
     end
 
     if(ENV["RAILS_PUBLIC_PATH"].present?)
-      paths["public"] = ENV["RAILS_PUBLIC_PATH"]
+      config.paths["public"] = ENV["RAILS_PUBLIC_PATH"]
     end
 
     # Use a mongo-based cache store (this ensures the cache can be shared

src/api-umbrella/web-app/config/initializers/assets.rb

@@ -10,8 +10,20 @@
 # application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
 Rails.application.config.assets.precompile += [
   "admin/login.css",
+  "admin/server_side_loader.js",
 ]
 
 # Move default assets directory so this project can co-exist with the
 # static-site projectt that delivers most of the web content.
 Rails.application.config.assets.prefix = "/web-assets"
+
+# Generate non-cached busted versions of assets that the admin-ui app needs to
+# link directly to (since it has no knowledge of the cache-busted URLs). We
+# just need to make sure these assets aren't allowed to be cached by the
+# browser.
+#
+# This is used for sharing i18n data between the Rails and Ember app. While not
+# the most optimized solution, it should be fine for sharing this bit of data.
+NonStupidDigestAssets.whitelist += [
+  "admin/server_side_loader.js",
+]