From fde91dd6149d755154ff76406dce9d2a78d19e49 Mon Sep 17 00:00:00 2001 From: Zijie Wu Date: Sun, 4 Sep 2022 22:26:04 +0000 Subject: [PATCH] Add thread tracking extension to myst-thread --- debugger/Makefile | 6 + debugger/gdb-sgx-plugin/thread.py | 290 ++++++++++++++++++++++++++++++ debugger/myst-gdb | 1 + doc/dev-jumpstart.md | 8 +- kernel/exec.c | 18 ++ kernel/thread.c | 15 +- tests/dotnet-lib-5/Makefile | 2 +- tests/gdb/Makefile | 2 +- tests/hello_world/c/Makefile | 3 + tests/hello_world/java/Makefile | 2 + 10 files changed, 340 insertions(+), 7 deletions(-) create mode 100644 debugger/gdb-sgx-plugin/thread.py diff --git a/debugger/Makefile b/debugger/Makefile index 8ca76afda..16a373855 100644 --- a/debugger/Makefile +++ b/debugger/Makefile @@ -3,6 +3,7 @@ TOP=$(abspath $(CURDIR)/..) BUILDDIR=${TOP}/build all: ${BUILDDIR}/bin/myst-gdb \ + ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py @@ -14,6 +15,10 @@ ${BUILDDIR}/bin/myst-gdb: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin: mkdir -p ${BUILDDIR}/lib/debugger/gdb-sgx-plugin +${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin + rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py + cp ${CURDIR}/gdb-sgx-plugin/thread.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py + ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py cp ${CURDIR}/gdb-sgx-plugin/mprotect.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py @@ -28,6 +33,7 @@ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py: ${BUILDDIR}/lib/debugger/gdb-s clean: rm -f ${BUILDDIR}/bin/myst-gdb + rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py diff --git a/debugger/gdb-sgx-plugin/thread.py b/debugger/gdb-sgx-plugin/thread.py new file mode 100644 index 000000000..9eb8d2a3f --- /dev/null +++ b/debugger/gdb-sgx-plugin/thread.py @@ -0,0 +1,290 @@ +import gdb +import math +import tempfile +from typing import List + +DEBUG = False + +INSTANCE_DEBUG_EXT = None + +class ThreadCreateBreakpoint(gdb.Breakpoint): + + STR_FIRST_ARG = "((struct run_thread_arg*)$rdi)" + + def __init__(self): + """ + Set a breakpoing @ kernel/thread.c:_run_thread, this should be right before myst_setjmp(&thread->jmpbuf) + Choosing this location because this is where the newly created thread about to run + We can get the new thread pid/tid here + """ + super(ThreadCreateBreakpoint, self).__init__('kernel/thread.c:_run_thread', internal=False) + + def stop(self): + # New thread info + ppid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->thread->process->ppid')) + pid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->thread->process->pid')) + tid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->thread->tid')) + tid_target = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->target_tid')) + if DEBUG: + print("Created thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + INSTANCE_DEBUG_EXT.add_thread(pid, tid, ppid, tid_target) + + # Add _main_thread only once as it won't change often + if not INSTANCE_DEBUG_EXT.main_thread_tracked(): + main_ppid = int(gdb.parse_and_eval('_main_thread->process->ppid')) + main_pid = int(gdb.parse_and_eval('_main_thread->process->pid')) + main_tid = int(gdb.parse_and_eval('_main_thread->tid')) + main_tid_target = int(gdb.parse_and_eval('_main_thread->target_tid')) + if DEBUG: + print("Main thread: ppid, pid, tid, tid_target", main_ppid, main_pid, main_tid, main_tid_target) + INSTANCE_DEBUG_EXT.add_thread(main_pid, main_tid, main_ppid, main_tid_target) + + INSTANCE_DEBUG_EXT.copy_parent_process(pid, ppid) + return False + +class ThreadExitBreakpoint(gdb.Breakpoint): + + STR_FIRST_ARG = "((const myst_thread_t*)$rdi)" + + def __init__(self): + """ + Set a breakpoint @ kernel/thread.c:myst_debug_hook_thread_exit, where myst_setjmp returns non-zero + Choosing this location because this is where the thread terminates + """ + super(ThreadExitBreakpoint, self).__init__('kernel/thread.c:myst_debug_hook_thread_exit', internal=False) + + def stop(self): + # Get thread info that is about to terminate + ppid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->process->ppid')) + pid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->process->pid')) + tid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->tid')) + tid_target = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->target_tid')) + if DEBUG: + print("Exit thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + + INSTANCE_DEBUG_EXT.remove_thread(tid) + return False + +class MystExecBreakpoint(gdb.Breakpoint): + + STR_FIRST_ARG = "((const myst_thread_t*)$rdi)" + STR_THIRD_ARG = "((const char**)$rdx)" + + def __init__(self): + """ + Obtain program's name, args and cwd to use in enclave + """ + super(MystExecBreakpoint, self).__init__('kernel/exec.c:myst_debug_hook_myst_exec', internal=False) + + def parse_string_raw(self, str_raw): + """ + Parse the string given the gdb parse_and_eval result + + Example input: + 0x104df31f0 "/bin/usr/some/path" + + Example return: + /bin/usr/some/path + """ + val = str(str_raw) + if ' ' not in val: + return val + + # Don't include " + return val[val.index(' ') + 2:-1] + + def stop(self): + # Same as parse and eval *argv@argc + argc = int(gdb.parse_and_eval('$rsi')) + + # ((const char**)$rdx)[0..1..2..] + argv_raw = [gdb.parse_and_eval('{0}[{1}]'.format(self.STR_THIRD_ARG, i)) for i in range(argc)] + cwd_raw = gdb.parse_and_eval(self.STR_FIRST_ARG + '->process->cwd') + + argv = [self.parse_string_raw(each) for each in argv_raw] + cwd = self.parse_string_raw(cwd_raw) + + # Get pid and ppid + ppid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->process->ppid')) + pid = int(gdb.parse_and_eval(self.STR_FIRST_ARG + '->process->pid')) + + INSTANCE_DEBUG_EXT.set_process_cmd_cwd(pid, ppid, argv, cwd) + + return False + +class DebugExtension: + """ + Manage all breakpoints and program info(process/thread/cmdline/etc) + """ + + # Configure the default pid/tid for the main process + # We only gather _main_thread's information once + tid_main_thread = 101 + + def __init__(self): + self.bp_thread_create = ThreadCreateBreakpoint() + self.bp_thread_exit = ThreadExitBreakpoint() + self.bp_myst_exec = MystExecBreakpoint() + + self.map_commands = { + 'help': lambda: self._command_help(), + 'thread': lambda: self._command_list_thread(), + 'cmdline': lambda: self._command_cmdline(), + 'exe': lambda: self._command_exe(), + 'cwd': lambda: self._command_cwd() + } + + # Store thread info + # Key: tid + # Value: tuple (pid, tid, ppid, tid_target) + self.threads = dict() + + # Store process info + # Key: pid + # Value: tuple (pid, ppid, command line, cwd) + # where command line should be list of string + self.processes = dict() + + self.clear() + + def main_thread_tracked(self): + return self.tid_main_thread in self.threads + + def add_thread(self, pid: int, tid: int, ppid: int, tid_target: int): + """ + ppid: parent pid + tid_target: tid on host + """ + self.threads[tid] = (pid, tid, ppid, tid_target) + + def remove_thread(self, tid: int): + if tid not in self.threads: + print("DebugExtension Error: trying to remove non-exist thread ", tid) + return + del self.threads[tid] + + def copy_parent_process(self, pid, ppid): + """ + When a new process is created, copy the cmdline from its parent + """ + if pid in self.processes: + if DEBUG: + print("Unexpected: pid {0} should not exist in self.processes".format(pid)) + return + + if ppid not in self.processes: + # Parent process might have already exited + return + + _, _, cmdline, cwd = self.processes[ppid] + self.processes[pid] = (pid, ppid, cmdline, cwd) + if DEBUG: + print("Copied", self.processes[pid]) + + def set_process_cmd_cwd(self, pid: int, ppid: int, argv: List[int], cwd: str): + self.processes[pid] = (pid, ppid, argv, cwd) + + def clear(self): + self.threads = dict() + self.processes = dict() + + def dispatch(self, command: str): + if command not in self.map_commands: + print("DebugExtension Error: Invalid command, should be one of ", sorted(list(self.map_commands.keys()))) + return + + self.map_commands[command]() + + def _command_help(self): + msg = """ +myst-thread: Mystikos thread tracker +Commands: +myst-thread help + Print this message + +myst-thread thread + List all tracked threads + +myst-thread cmdline + List the command line for each process + +myst-thread exe + List the executable for each process + +myst-thread cwd + List the current working directory for each process +""" + print(msg) + + def _command_list_thread(self): + if not self.threads: + print("No threads tracked") + return + + print("List of Mystikos controlled threads:") + print("ppid pid tid (target_tid)") + for key in sorted(self.threads.keys()): + pid, tid, ppid, target_tid = self.threads[key] + print("{0} {1} {2} ({3})".format(ppid, pid, tid, target_tid)) + + def _command_cmdline(self): + """ + Similiar to gdb command 'info proc cmdline' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, cmdline, _ = self.processes[pid] + print("Process {0} (Parent {1}) cmdline: {2}".format(pid, ppid, " ".join(cmdline))) + + def _command_exe(self): + """ + Similiar to gdb command 'info proc exe' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, cmdline, _ = self.processes[pid] + print("Process {0} (Parent {1}) exe: {2}".format(pid, ppid, cmdline[0])) + + def _command_cwd(self): + """ + Similiar to gdb command 'info proc cwd' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, _, cwd = self.processes[pid] + print("Process {0} (Parent {1}) cwd: {2}".format(pid, ppid, cwd)) + +command = """ +define myst-thread + if $argc == 0 + python INSTANCE_DEBUG_EXT.dispatch("help") + end + if $argc == 1 + python INSTANCE_DEBUG_EXT.dispatch("$arg0") + end +end +""" + +if __name__ == "__main__": + INSTANCE_DEBUG_EXT = DebugExtension() + + # Register command with gdb. + with tempfile.NamedTemporaryFile('w') as f: + f.write(command) + f.flush() + gdb.execute('source %s' % f.name) + + # Register exit_handler + def exit_handler(event): + global INSTANCE_DEBUG_EXT + INSTANCE_DEBUG_EXT.clear() + gdb.events.exited.connect(exit_handler) diff --git a/debugger/myst-gdb b/debugger/myst-gdb index e5ba6fa3f..b03ed2d0a 100755 --- a/debugger/myst-gdb +++ b/debugger/myst-gdb @@ -29,6 +29,7 @@ LD_PRELOAD=$OE_GDB_PTRACE_PATH gdb \ -iex "directory $MYST_GDB_PLUGIN_DIR" \ -iex "source $OE_GDB_PLUGIN_DIR/gdb_sgx_plugin.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/print.py" \ + -iex "source $MYST_GDB_PLUGIN_DIR/thread.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/mprotect.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/symbol_analyzer.py" \ -iex "set environment LD_PRELOAD" \ diff --git a/doc/dev-jumpstart.md b/doc/dev-jumpstart.md index c0c043e4b..557ce97a8 100644 --- a/doc/dev-jumpstart.md +++ b/doc/dev-jumpstart.md @@ -65,15 +65,15 @@ consists of the following artifacts: * openenclave, including the outputs from building OE SDK. 1. Run a simple application built with gcc ```bash - cd tests/hello + cd tests/hello_world/c make make tests ``` - In the 2nd step `make`, we create a temporary folder `appdir`, compile + During the 2nd step `make`, it creates a temporary folder `appdir`, compile `hello.c` with `gcc`, and place the output executable under `appdir/bin/hello`, finally we create a CPIO archive out of `appdir`. - In the 3rd step `make tests`, we launch `myst`, giving it the CPIO + During the 3rd step `make tests`, it launchs `myst`, giving it the CPIO archive, the command to run (`/bin/hello` in this case), and the command line arguments, e.g., "red", "green", and "blue". With this step, we should see the following outputs: @@ -125,7 +125,7 @@ O Call -> Calls into the host (untrusted environment) 1. For most applications under `tests`, we can launch the debugger with command `make tests GDB=1`. For example: ``` - cd Mystikos/tests/hello + cd tests/hello_world/c make && make tests GDB=1 ``` For applications that are run in [package mode](../solutions/dotnet/Makefile#23), ensure that the field `"Debug":1` is set in the `config.json` file, and the debugger can be launched using the run command: diff --git a/kernel/exec.c b/kernel/exec.c index ae8773e6e..acd6dec0f 100644 --- a/kernel/exec.c +++ b/kernel/exec.c @@ -946,6 +946,22 @@ static int _get_prog_interp(const char* path, char** name_out) return ret; } +// This function is used by debugger/gdb-sgx-plugin/thread.py +// Adding "-O2" so this hook function is not optimized out +#pragma GCC push_options +#pragma GCC optimize "-O2" +void myst_debug_hook_myst_exec( + const myst_thread_t* thread, + size_t argc, + const char* argv[]) +{ + // This function should do nothing + assert(thread != NULL); + assert(argc != 0); + assert(argv != NULL); +} +#pragma GCC pop_options + int myst_exec( myst_thread_t* thread, const void* crt_data_in, @@ -979,6 +995,8 @@ int myst_exec( char* prog_interp = NULL; size_t actual_thread_stack_size = thread_stack_size; + myst_debug_hook_myst_exec(thread, argc, argv); + if (thread_stack_size) _thread_stack_size = thread_stack_size; diff --git a/kernel/thread.c b/kernel/thread.c index 064f05059..583e85221 100644 --- a/kernel/thread.c +++ b/kernel/thread.c @@ -41,7 +41,7 @@ #include #include -//#define TRACE +// #define TRACE myst_spinlock_t myst_process_list_lock = MYST_SPINLOCK_INITIALIZER; @@ -886,6 +886,17 @@ void myst_fork_exec_futex_wake(pid_t pid, pid_t tid) **============================================================================== */ +// This function is used by debugger/gdb-sgx-plugin/thread.py +// Adding "-O2" so this hook function is not optimized out +#pragma GCC push_options +#pragma GCC optimize "-O2" +void myst_debug_hook_thread_exit(const myst_thread_t* thread) +{ + // This function should do nothing + assert(thread != NULL); +} +#pragma GCC pop_options + bool myst_valid_td(const void* td) { return td && ((const myst_td_t*)td)->self == td; @@ -983,6 +994,8 @@ static long _run_thread(void* arg_) { /* ---------- running C-runtime thread descriptor ---------- */ + myst_debug_hook_thread_exit(thread); + assert(myst_gettid() != -1); /* restore the target thread descriptor */ diff --git a/tests/dotnet-lib-5/Makefile b/tests/dotnet-lib-5/Makefile index 3d17f29a3..f760438f7 100644 --- a/tests/dotnet-lib-5/Makefile +++ b/tests/dotnet-lib-5/Makefile @@ -53,7 +53,7 @@ rootfsr: $(RAPPDIR) appdirr: $(MYST-RETRY) $(APPBUILDER) -m -o $(RAPPDIR) Dockerfile.musl.runner -TESTCASE=testcases/pass.txt +TESTCASE=testcases/pass.1 run-runner: $(RUNTEST) $(MYST_EXEC) $(OPTS) $(RROOTFS) --app-config-path config-runner.json $(RUNNER) /testcases/pass.1 /dotnet-lib-release/ diff --git a/tests/gdb/Makefile b/tests/gdb/Makefile index 98a2994f5..837365ae0 100644 --- a/tests/gdb/Makefile +++ b/tests/gdb/Makefile @@ -14,7 +14,7 @@ GDB_CMDS=--batch \ -ex "backtrace" \ --args -GDB_MATCH=Breakpoint 2, .................. in print_hello () +GDB_MATCH=Breakpoint 5, .................. in print_hello () # runtest timeouts cause gdb to hang export NOTIMEOUT=1 diff --git a/tests/hello_world/c/Makefile b/tests/hello_world/c/Makefile index c92968c30..7d6803326 100644 --- a/tests/hello_world/c/Makefile +++ b/tests/hello_world/c/Makefile @@ -22,3 +22,6 @@ myst: clean: rm -rf $(APPDIR) rootfs export ramfs + +gdb: + $(MYST_GDB) --args $(MYST_EXEC) rootfs /bin/hello red green blue $(OPTS) diff --git a/tests/hello_world/java/Makefile b/tests/hello_world/java/Makefile index 319192757..ce2d37742 100644 --- a/tests/hello_world/java/Makefile +++ b/tests/hello_world/java/Makefile @@ -23,3 +23,5 @@ tests: clean: rm -rf ext2rootfs appdir +gdb: + $(MYST_GDB) --args $(MYST_EXEC) $(OPTS) ext2rootfs /opt/openjdk-13/bin/java -ea Helloworld red green blue