From 830f8b86a7edaffed0400ce04fd845a555ed13ea Mon Sep 17 00:00:00 2001 From: Wolfgang Steiner Date: Thu, 20 Jul 2017 01:51:21 +0200 Subject: [PATCH] add "j2v8optimize" buildstep (performs execstack & strip on linux) - register atexit events before cross-builds are started - switched linux & android docker images to Ubuntu Xenial LTS (see #311) - cleaned up and documented switch-to-winrm-plugin that is used in win32:vagrant builds --- build.py | 30 ++++++--- build_system/config_linux.py | 9 +++ build_system/constants.py | 4 +- build_system/docker_build.py | 21 +++--- build_system/rm.py | 2 +- build_system/vagrant_build.py | 20 +++--- docker/android/Dockerfile | 6 +- docker/linux/Dockerfile | 6 +- docker/shared/install.debian.packages.sh | 4 +- vagrant/win32/switch-to-winrm-plugin.rb | 84 +++--------------------- 10 files changed, 77 insertions(+), 109 deletions(-) diff --git a/build.py b/build.py index 4c83f3171..f2fd09def 100644 --- a/build.py +++ b/build.py @@ -18,6 +18,7 @@ c.build_node_js, c.build_j2v8_cmake, c.build_j2v8_jni, + c.build_j2v8_optimize, c.build_j2v8_java, c.build_j2v8_junit, ] @@ -40,13 +41,6 @@ c.target_win32: win32_config, } -# TODO: shift responsibility to add targets to platform config or no ? -extra_targets = [ - c.target_macos_vagrant, - c.target_win32_docker, - c.target_win32_vagrant, -] - avail_architectures = [ c.arch_x86, c.arch_x64, @@ -55,6 +49,20 @@ avail_build_steps = build_step_sequence + composite_steps +# this goes through all known target platforms, and returns the sub-targets +# that are available for cross-compilation +def get_cross_targets(): + cross_targets = [] + + for tgt in avail_targets.values(): + if (not tgt.cross_compilers): + continue + + for xcomp in tgt.cross_compilers: + cross_targets.append(tgt.name + ":" + xcomp) + + return cross_targets + #----------------------------------------------------------------------- # Command-Line setup #----------------------------------------------------------------------- @@ -65,7 +73,7 @@ help="The build target platform name (must be a valid platform string identifier).", dest="target", required=True, - choices=sorted(avail_targets.keys() + extra_targets)) + choices=sorted(avail_targets.keys() + get_cross_targets())) parser.add_argument("--arch", "-a", help="The build target architecture identifier (the available architectures are also dependent on the selected platform for a build).", @@ -121,6 +129,7 @@ def parse_build_step_option(step): c.build_node_js: lambda: parsed_steps.add(c.build_node_js), c.build_j2v8_cmake: lambda: parsed_steps.add(c.build_j2v8_cmake), c.build_j2v8_jni: lambda: parsed_steps.add(c.build_j2v8_jni), + c.build_j2v8_optimize: lambda: parsed_steps.add(c.build_j2v8_optimize), c.build_j2v8_java: lambda: parsed_steps.add(c.build_j2v8_java), c.build_j2v8_junit: lambda: parsed_steps.add(c.build_j2v8_junit), }.get(step, raise_unhandled_option) @@ -133,6 +142,7 @@ def add_native(): parsed_steps.add(c.build_node_js) parsed_steps.add(c.build_j2v8_cmake) parsed_steps.add(c.build_j2v8_jni) + parsed_steps.add(c.build_j2v8_optimize) def add_managed(): parsed_steps.add(c.build_j2v8_java) @@ -261,6 +271,10 @@ def execute_build_step(compiler_inst, build_step): # execute all requested build steps for step in parsed_steps: + if (not step in target_steps): + sys.exit("Hint: skipping build step \"" + step + "\" (not configured and/or supported for platform \"" + params.target + "\")") + continue + target_step = target_steps[step] # prepare any additional/dynamic parameters for the build and put them into the build-step config diff --git a/build_system/config_linux.py b/build_system/config_linux.py index 6d9aaecfb..7d4cd7407 100644 --- a/build_system/config_linux.py +++ b/build_system/config_linux.py @@ -56,6 +56,15 @@ def build_j2v8_jni(config): linux_config.build_step(c.build_j2v8_jni, build_j2v8_jni) #----------------------------------------------------------------------- +def build_j2v8_optimize(config): + file_abi = config.target.file_abi(config.arch) + return [ + "execstack -c cmake.out/$PLATFORM.$ARCH/libj2v8_linux_" + file_abi + ".so", + "strip --strip-unneeded -R .note -R .comment cmake.out/$PLATFORM.$ARCH/libj2v8_linux_" + file_abi + ".so", + ] + +linux_config.build_step(c.build_j2v8_optimize, build_j2v8_optimize) +#----------------------------------------------------------------------- def build_j2v8_java(config): return \ u.clearNativeLibs(config) + \ diff --git a/build_system/constants.py b/build_system/constants.py index 464560466..115c756bf 100644 --- a/build_system/constants.py +++ b/build_system/constants.py @@ -2,10 +2,7 @@ target_android = 'android' target_linux = 'linux' target_macos = 'macos' -target_macos_vagrant = 'macos:vagrant' target_win32 = 'win32' -target_win32_docker = 'win32:docker' -target_win32_vagrant = 'win32:vagrant' # target architectures arch_x86 = 'x86' @@ -16,6 +13,7 @@ build_node_js = 'nodejs' build_j2v8_cmake = 'j2v8cmake' build_j2v8_jni = 'j2v8jni' +build_j2v8_optimize = 'j2v8optimize' build_j2v8_java = 'j2v8java' build_j2v8_junit = 'j2v8junit' diff --git a/build_system/docker_build.py b/build_system/docker_build.py index 9fb4a02b8..944c1d659 100644 --- a/build_system/docker_build.py +++ b/build_system/docker_build.py @@ -47,10 +47,21 @@ def health_check(self, config): def pre_build(self, config): print ("preparing " + config.platform + "@" + config.arch + " => " + config.name) + docker_stop_str = self.inject_env("docker stop j2v8.$PLATFORM.$ARCH", config) + + def cli_exit_event(): + if (config.no_shutdown): + return + + print "Waiting for docker process to exit..." + self.exec_host_cmd(docker_stop_str, config) + + atexit.register(cli_exit_event) + self.exec_host_cmd("docker build -f $PLATFORM/Dockerfile -t \"j2v8-$PLATFORM\" .", config) def exec_build(self, config): - print ("DOCKER building " + config.platform + "@" + config.arch + " => " + config.name) + print ("DOCKER running " + config.platform + "@" + config.arch + " => " + config.name) is_win32 = utils.is_win32(config.platform) @@ -74,14 +85,6 @@ def exec_build(self, config): print docker_run_str - docker_stop_str = self.inject_env("docker stop j2v8.$PLATFORM.$ARCH", config) - - def cli_exit_event(): - print "Waiting for docker process to exit..." - self.exec_host_cmd(docker_stop_str, config) - - atexit.register(cli_exit_event) - self.exec_host_cmd(docker_run_str, config) def post_build(self, config): diff --git a/build_system/rm.py b/build_system/rm.py index aa383e2a9..ca8ddf65c 100644 --- a/build_system/rm.py +++ b/build_system/rm.py @@ -2,7 +2,7 @@ import sys import shutil -# this is a cross-platform polyfill for "cp" +# this is a cross-platform polyfill for "rm" items = sys.argv[1:] diff --git a/build_system/vagrant_build.py b/build_system/vagrant_build.py index e0754c88d..577dade2f 100644 --- a/build_system/vagrant_build.py +++ b/build_system/vagrant_build.py @@ -20,10 +20,19 @@ def pre_build(self, config): if (config.pre_build_cmd): vagrant_start_cmd = config.pre_build_cmd + utils.host_cmd_sep() + vagrant_start_cmd + def cli_exit_event(): + if (config.no_shutdown): + return + + print "Waiting for vagrant virtual-machine to exit..." + self.exec_host_cmd("vagrant halt", config) + + atexit.register(cli_exit_event) + self.exec_host_cmd(vagrant_start_cmd, config) def exec_build(self, config): - print ("VAGRANT building " + config.platform + "@" + config.arch + " => " + config.name) + print ("VAGRANT running " + config.platform + "@" + config.arch + " => " + config.name) # shell = "powershell -c \"cmd /C " if utils.is_win32(config.platform) else "ssh -c " # cmd_sep = "&& " if utils.is_win32(config.platform) else "; " @@ -63,15 +72,6 @@ def exec_build(self, config): build_cmd = self.inject_env("cd $BUILD_CWD" + cmd_sep + build_cmd, config) vagrant_run_cmd = "vagrant ssh -c '" + build_cmd + "'" - def cli_exit_event(): - if (config.no_shutdown): - return - - print "Waiting for vagrant virtual-machine to exit..." - self.exec_host_cmd("vagrant halt", config) - - atexit.register(cli_exit_event) - self.exec_host_cmd(vagrant_run_cmd, config) def post_build(self, config): diff --git a/docker/android/Dockerfile b/docker/android/Dockerfile index 30437d90f..428a0ae49 100644 --- a/docker/android/Dockerfile +++ b/docker/android/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:zesty +FROM ubuntu:xenial RUN mkdir -p /temp/docker/shared/ WORKDIR /temp/docker/shared/ @@ -31,6 +31,10 @@ COPY ./shared/install.jdk.sh /temp/docker/shared RUN ./install.jdk.sh ENV JAVA_HOME "/opt/jdk/jdk1.8.0_131" +COPY ./shared/install.cmake.sh /temp/docker/shared +RUN ./install.cmake.sh +ENV PATH "$PATH:/opt/cmake/bin" + COPY ./shared/install.gradle.sh /temp/docker/shared RUN ./install.gradle.sh ENV GRADLE_HOME "/opt/gradle-3.5" diff --git a/docker/linux/Dockerfile b/docker/linux/Dockerfile index 1e49d955c..2ba9a37db 100644 --- a/docker/linux/Dockerfile +++ b/docker/linux/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:zesty +FROM ubuntu:xenial RUN mkdir -p /temp/docker/shared/ WORKDIR /temp/docker/shared/ @@ -19,6 +19,10 @@ COPY ./shared/install.maven.sh /temp/docker/shared RUN ./install.maven.sh ENV PATH "$PATH:/opt/apache-maven-3.5.0/bin" +COPY ./shared/install.cmake.sh /temp/docker/shared +RUN ./install.cmake.sh +ENV PATH "$PATH:/opt/cmake/bin" + # download the most critical maven dependencies for the build beforehand COPY ./shared/pom.xml /temp WORKDIR /temp diff --git a/docker/shared/install.debian.packages.sh b/docker/shared/install.debian.packages.sh index f8301df5c..344cdb261 100755 --- a/docker/shared/install.debian.packages.sh +++ b/docker/shared/install.debian.packages.sh @@ -10,6 +10,6 @@ apt-get -qq update && \ file \ python \ make \ - cmake \ wget \ - supervisor + supervisor \ + execstack diff --git a/vagrant/win32/switch-to-winrm-plugin.rb b/vagrant/win32/switch-to-winrm-plugin.rb index 288d378d1..de8f63acd 100644 --- a/vagrant/win32/switch-to-winrm-plugin.rb +++ b/vagrant/win32/switch-to-winrm-plugin.rb @@ -5,7 +5,7 @@ class SwitchToWinRMPlugin < Vagrant.plugin('2') name 'Switch to WinRM Plugin' - # This plugin provides a provisioner called windows_reboot. + # This plugin provides a provisioner called switch_to_winrm. provisioner 'switch_to_winrm' do # Create a provisioner. @@ -27,13 +27,15 @@ def provision env = @machine.instance_variable_get(:@env) if not _provisioned + # stop the VM before we switch the communicator puts "action_halt..." env.action_runner.run(VagrantPlugins::ProviderVirtualBox::Action::action_halt(), { machine: @machine, ui: @machine.ui, }) - # end + # TODO: this is just a copy-paste of the settings from the actual Vagrantfile config + # there should be some practical way to remove this code duplication! puts "switching comm..." @machine.config.vm.communicator = :winrm @machine.config.winrm.username = "IEUser" @@ -44,95 +46,29 @@ def provision @machine.config.vm.synced_folder ".", "/vagrant", disabled: true @machine.config.vm.synced_folder "../../", "C:/j2v8", type: "virtualbox", smb_username: ENV['VAGRANT_SMB_USER'], smb_password: ENV['VAGRANT_SMB_PASSWORD'] + # NOTE: this is copied from https://github.com/mitchellh/vagrant/blob/d1a589c59f75dd2910e47976a742dc6bc99035b0/lib/vagrant/machine.rb#L246 + # it reinstantiates the communicator defined by the vagrant configuration ... requested = @machine.config.vm.communicator requested ||= :ssh klass = Vagrant.plugin("2").manager.communicators[requested] raise Errors::CommunicatorNotFound, comm: requested.to_s if !klass - # puts "inspect: ", Vagrant.plugin("2").manager.inspect # Object - # puts "instance_variables: ", Vagrant.plugin("2").manager.instance_variables # Object - # puts "methods:", Vagrant.plugin("2").manager.methods # Object - comm = klass.new(@machine) + + # ... and then monkey-patches the new instance into the machine @machine.instance_variable_set(:@communicator, comm) puts "patched communicator" + # this applies the changed communicator and also reconfigures the related network settings @machine.config.finalize! - # if not _provisioned + # start the VM now, after we successfully switched the communicator puts "action_boot..." env.action_runner.run(VagrantPlugins::ProviderVirtualBox::Action::action_boot(), { machine: @machine, ui: @machine.ui, }) end - - # puts "A", @machine.class # Object - # puts "B", @machine.instance_variables - # puts "C", @machine.inspect - - # app = Vagrant::Action::Builder.new.tap do |b| - # b.use VagrantPlugins::ProviderVirtualBox::Action::Network - # end - - # puts("running network upadte") - # env.action_runner.run(app, { - # machine: @machine, - # ui: @machine.ui, - # }) - - # comm.wait_for_ready(5000) - # @machine.action('wait_for_communicator') - # Vagrant::Action::Builder.new.tap do |b| - # b.use WaitForCommunicator, [:starting, :running] - # end - # Vagrant::Action::Builtin::WaitForCommunicator.new() - - # app = Vagrant::Action::Builder.new.tap do |b| - # b.use Vagrant::Action::Builtin::WaitForCommunicator, [:starting, :running] - # end - - # Vagrant::Action.run(app) - # runner = Vagrant::Action::Runner.new - # runner.run(app, env) - - # env.action_runner.run(app, { - # machine: @machine, - # ui: @machine.ui, - # }) - - - - # begin - # sleep 5 - # end until @machine.communicate.ready? - - # puts "communicator ready" - - # comm.initialize(@machine) - - # @machine.communicator = klass.new(self) - # @machine.communicator.initialize(@machine) - - # @machine.ui.info("trying action_boot...") - # @machine.action('action_boot') - - # @machine.config.winrm.retry_limit = 1000 - # command = 'shutdown -t 0 -r -f' - # @machine.ui.info("Issuing command: #{command}") - # @machine.communicate.execute(command) do - # if type == :stderr - # @machine.ui.error(data); - # end - # end - - # begin - # sleep 5 - # end until @machine.communicate.ready? - - # # Now the machine is up again, perform the necessary tasks. - # @machine.ui.info("Launching remount_synced_folders action...") - # @machine.action('remount_synced_folders') end # Nothing needs to be done on cleanup.