From a2e83ec0b34211d78f8210229b6d96e76550584b Mon Sep 17 00:00:00 2001 From: Zijie Wu Date: Sun, 4 Sep 2022 22:26:04 +0000 Subject: [PATCH] Add thread tracking extension to myst-gdb --- debugger/Makefile | 6 + debugger/gdb-sgx-plugin/debug_ext.py | 283 +++++++++++++++++++++++++++ debugger/myst-gdb | 1 + doc/dev-jumpstart.md | 8 +- kernel/thread.c | 2 +- tests/dotnet-lib-5/Makefile | 2 +- tests/hello_world/c/Makefile | 3 + tests/hello_world/java/Makefile | 2 + 8 files changed, 301 insertions(+), 6 deletions(-) create mode 100644 debugger/gdb-sgx-plugin/debug_ext.py diff --git a/debugger/Makefile b/debugger/Makefile index 8ca76afda9..b5ab51a4bb 100644 --- a/debugger/Makefile +++ b/debugger/Makefile @@ -3,6 +3,7 @@ TOP=$(abspath $(CURDIR)/..) BUILDDIR=${TOP}/build all: ${BUILDDIR}/bin/myst-gdb \ + ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/debug_ext.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py \ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py @@ -14,6 +15,10 @@ ${BUILDDIR}/bin/myst-gdb: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin: mkdir -p ${BUILDDIR}/lib/debugger/gdb-sgx-plugin +${BUILDDIR}/lib/debugger/gdb-sgx-plugin/debug_ext.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin + rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/debug_ext.py + cp ${CURDIR}/gdb-sgx-plugin/debug_ext.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/debug_ext.py + ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py cp ${CURDIR}/gdb-sgx-plugin/mprotect.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py @@ -28,6 +33,7 @@ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py: ${BUILDDIR}/lib/debugger/gdb-s clean: rm -f ${BUILDDIR}/bin/myst-gdb + rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/debug_ext.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py diff --git a/debugger/gdb-sgx-plugin/debug_ext.py b/debugger/gdb-sgx-plugin/debug_ext.py new file mode 100644 index 0000000000..ec28e2c0d1 --- /dev/null +++ b/debugger/gdb-sgx-plugin/debug_ext.py @@ -0,0 +1,283 @@ +import gdb +import math +import tempfile +from typing import List + +DEBUG = False + +INSTANCE_DEBUG_EXT = None + +class ThreadCloneBreakpoint(gdb.Breakpoint): + def __init__(self): + """ + Set a breakpoint @ kernel/syscall.c::_SYS_myst_clone + Choosing this location because this is where the parent process/thread creates new process/thread + We can get the parent pid/tid from here + """ + super(ThreadCloneBreakpoint, self).__init__('kernel/syscall.c:4094', internal=False) + + def stop(self): + # Get Parent thread info + ppid = int(gdb.parse_and_eval('myst_thread_self()->process->ppid')) + pid = int(gdb.parse_and_eval('myst_thread_self()->process->pid')) + tid = int(gdb.parse_and_eval('myst_thread_self()->tid')) + tid_target = int(gdb.parse_and_eval('myst_thread_self()->target_tid')) + print("Parent thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + + INSTANCE_DEBUG_EXT.add_thread(pid, tid, ppid, tid_target) + return False + +class ThreadCreateBreakpoint(gdb.Breakpoint): + def __init__(self): + """ + Set a breakpoing @ kernel/thread.c::_run_thread, this should be right before myst_setjmp(&thread->jmpbuf) + Choosing this location because this is where the newly created thread about to run + We can get the new thread pid/tid here + """ + super(ThreadCreateBreakpoint, self).__init__('kernel/thread.c:978', internal=False) + + def stop(self): + # New thread info + ppid = int(gdb.parse_and_eval('thread->process->ppid')) + pid = int(gdb.parse_and_eval('thread->process->pid')) + tid = int(gdb.parse_and_eval('thread->tid')) + tid_target = int(gdb.parse_and_eval('thread->target_tid')) + print("Created thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + INSTANCE_DEBUG_EXT.add_thread(pid, tid, ppid, tid_target) + + # Add _main_thread only once as it won't change often + if not INSTANCE_DEBUG_EXT.main_thread_tracked(): + ppid = int(gdb.parse_and_eval('_main_thread->process->ppid')) + pid = int(gdb.parse_and_eval('_main_thread->process->pid')) + tid = int(gdb.parse_and_eval('_main_thread->tid')) + tid_target = int(gdb.parse_and_eval('_main_thread->target_tid')) + print("Main thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + INSTANCE_DEBUG_EXT.add_thread(pid, tid, ppid, tid_target) + + return False + +class ThreadExitBreakpoint(gdb.Breakpoint): + def __init__(self): + """ + Set a breakpoint @ kernel/thread.c::_run_thread, where myst_setjmp returns non-zero + Choosing this location because this is where the newly created thread terminates + """ + super(ThreadExitBreakpoint, self).__init__('kernel/thread.c:985', internal=False) + + def stop(self): + # Get thread info that is about to terminate + ppid = int(gdb.parse_and_eval('thread->process->ppid')) + pid = int(gdb.parse_and_eval('thread->process->pid')) + tid = int(gdb.parse_and_eval('thread->tid')) + tid_target = int(gdb.parse_and_eval('thread->target_tid')) + print("Exit thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target) + + INSTANCE_DEBUG_EXT.remove_thread(pid, tid) + return False + +class MystExecBreakpoint(gdb.Breakpoint): + def __init__(self): + """ + Set a breakpoint @ kernel/exec.c::myst_exec, right after `process = thread->process` + grab program and args to run in Enclave, and cwd + """ + super(MystExecBreakpoint, self).__init__('kernel/exec.c:1004', internal=False) + + def parse_string_raw(self, str_raw): + """ + Parse the string given the gdb parse_and_eval result + + Example input: + 0x104df31f0 "/bin/usr/some/path" + + Example return: + /bin/usr/some/path + """ + val = str(str_raw) + if ' ' not in val: + return val + + # Don't include " + return val[val.index(' ') + 2:-1] + + def stop(self): + # Same as parse and eval *argv@argc + argc = int(gdb.parse_and_eval('argc')) + + argv_raw = [gdb.parse_and_eval('argv[{0}]'.format(i)) for i in range(argc)] + cwd_raw = gdb.parse_and_eval('process->cwd') + + argv = [self.parse_string_raw(each) for each in argv_raw] + cwd = self.parse_string_raw(cwd_raw) + + # Get pid and ppid + ppid = int(gdb.parse_and_eval('process->ppid')) + pid = int(gdb.parse_and_eval('process->pid')) + + INSTANCE_DEBUG_EXT.set_process_cmd_cwd(pid, ppid, argv, cwd) + + return False + +class DebugExtension: + """ + Manage all breakpoints and program info(process/thread/cmdline/etc) + """ + + # Configure the default pid/tid for the main process + # We only gather _main_thread's information once + pid_main_process = 101 + tid_main_thread = 101 + + def __init__(self): + # Disabled as calling 'myst_thread_self()' from gdb seems to cause issues + # self.bThreadBeforeCreate = ThreadCloneBreakpoint() + + self.bp_thread_create = ThreadCreateBreakpoint() + self.bp_thread_exit = ThreadExitBreakpoint() + self.bp_myst_exec = MystExecBreakpoint() + + self.map_commands = { + 'help': lambda: self._command_help(), + 'thread': lambda: self._command_list_thread(), + 'cmdline': lambda: self._command_cmdline(), + 'exe': lambda: self._command_exe(), + 'cwd': lambda: self._command_cwd() + } + + # Store thread info + # Key: tuple (pid, tid) + # Value: tuple (pid, tid, ppid, tid_target) + self.threads = dict() + + # Store process info + # Key: pid + # Value: tuple (pid, ppid, command line, cwd) + # where command line should be list of string + self.processes = dict() + + self.clear() + + def main_thread_tracked(self): + return (self.pid_main_process, self.tid_main_thread) in self.threads + + def add_thread(self, pid: int, tid: int, ppid: int, tid_target: int): + """ + ppid: parent pid + tid_target: tid on host + """ + self.threads[(pid, tid)] = (pid, tid, ppid, tid_target) + + def remove_thread(self, pid: int, tid: int): + _key = (pid, tid) + if _key not in self.threads: + print("DebugExtension Error: trying to remove non-exist thread ", _key) + return + del self.threads[_key] + + def set_process_cmd_cwd(self, pid: int, ppid: int, argv: List[int], cwd: str): + self.processes[pid] = (pid, ppid, argv, cwd) + + def clear(self): + self.threads = dict() + self.processes = dict() + + def dispatch(self, command: str): + if command not in self.map_commands: + print("DebugExtension Error: Invalid command, should be one of ", sorted(list(self.map_commands.keys()))) + return + + self.map_commands[command]() + + def _command_help(self): + msg = """ +myst-debug: Mystikos thread tracker +Commands: +myst-debug help + Print this message + +myst-debug thread + List all tracked threads + +myst-debug cmdline + List the command line for each process + +myst-debug exe + List the executable for each process + +myst-debug cwd + List the current working directory for each process +""" + print(msg) + + def _command_list_thread(self): + if not self.threads: + print("No threads tracked") + return + + print("List of Mystikos controlled threads:") + print("ppid pid tid (target_tid)") + for key in sorted(self.threads.keys()): + pid, tid, ppid, target_tid = self.threads[key] + print("{0} {1} {2} ({3})".format(ppid, pid, tid, target_tid)) + + def _command_cmdline(self): + """ + Similiar to gdb command 'info proc cmdline' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, cmdline, _ = self.processes[pid] + print("Process {0} (Parent {1}) cmdline: {2}".format(pid, ppid, " ".join(cmdline))) + + def _command_exe(self): + """ + Similiar to gdb command 'info proc exe' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, cmdline, _ = self.processes[pid] + print("Process {0} (Parent {1}) exe: {2}".format(pid, ppid, cmdline[0])) + + def _command_cwd(self): + """ + Similiar to gdb command 'info proc cwd' + """ + if not self.processes: + print("No processes tracked") + return + + for pid in sorted(self.processes.keys()): + pid, ppid, _, cwd = self.processes[pid] + print("Process {0} (Parent {1}) cwd: {2}".format(pid, ppid, cwd)) + +command = """ +define myst-debug + if $argc == 0 + python INSTANCE_DEBUG_EXT.dispatch("help") + end + if $argc == 1 + python INSTANCE_DEBUG_EXT.dispatch("$arg0") + end +end +""" + +if __name__ == "__main__": + INSTANCE_DEBUG_EXT = DebugExtension() + + # Register command with gdb. + with tempfile.NamedTemporaryFile('w') as f: + f.write(command) + f.flush() + gdb.execute('source %s' % f.name) + + # Register exit_handler + def exit_handler(event): + global INSTANCE_DEBUG_EXT + INSTANCE_DEBUG_EXT.clear() + gdb.events.exited.connect(exit_handler) diff --git a/debugger/myst-gdb b/debugger/myst-gdb index e5ba6fa3f6..69f80b5146 100755 --- a/debugger/myst-gdb +++ b/debugger/myst-gdb @@ -29,6 +29,7 @@ LD_PRELOAD=$OE_GDB_PTRACE_PATH gdb \ -iex "directory $MYST_GDB_PLUGIN_DIR" \ -iex "source $OE_GDB_PLUGIN_DIR/gdb_sgx_plugin.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/print.py" \ + -iex "source $MYST_GDB_PLUGIN_DIR/debug_ext.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/mprotect.py" \ -iex "source $MYST_GDB_PLUGIN_DIR/symbol_analyzer.py" \ -iex "set environment LD_PRELOAD" \ diff --git a/doc/dev-jumpstart.md b/doc/dev-jumpstart.md index c0c043e4b8..557ce97a82 100644 --- a/doc/dev-jumpstart.md +++ b/doc/dev-jumpstart.md @@ -65,15 +65,15 @@ consists of the following artifacts: * openenclave, including the outputs from building OE SDK. 1. Run a simple application built with gcc ```bash - cd tests/hello + cd tests/hello_world/c make make tests ``` - In the 2nd step `make`, we create a temporary folder `appdir`, compile + During the 2nd step `make`, it creates a temporary folder `appdir`, compile `hello.c` with `gcc`, and place the output executable under `appdir/bin/hello`, finally we create a CPIO archive out of `appdir`. - In the 3rd step `make tests`, we launch `myst`, giving it the CPIO + During the 3rd step `make tests`, it launchs `myst`, giving it the CPIO archive, the command to run (`/bin/hello` in this case), and the command line arguments, e.g., "red", "green", and "blue". With this step, we should see the following outputs: @@ -125,7 +125,7 @@ O Call -> Calls into the host (untrusted environment) 1. For most applications under `tests`, we can launch the debugger with command `make tests GDB=1`. For example: ``` - cd Mystikos/tests/hello + cd tests/hello_world/c make && make tests GDB=1 ``` For applications that are run in [package mode](../solutions/dotnet/Makefile#23), ensure that the field `"Debug":1` is set in the `config.json` file, and the debugger can be launched using the run command: diff --git a/kernel/thread.c b/kernel/thread.c index 8d2c9ff8ea..d9076b486a 100644 --- a/kernel/thread.c +++ b/kernel/thread.c @@ -40,7 +40,7 @@ #include #include -//#define TRACE +// #define TRACE myst_spinlock_t myst_process_list_lock = MYST_SPINLOCK_INITIALIZER; diff --git a/tests/dotnet-lib-5/Makefile b/tests/dotnet-lib-5/Makefile index 3d17f29a3a..f760438f75 100644 --- a/tests/dotnet-lib-5/Makefile +++ b/tests/dotnet-lib-5/Makefile @@ -53,7 +53,7 @@ rootfsr: $(RAPPDIR) appdirr: $(MYST-RETRY) $(APPBUILDER) -m -o $(RAPPDIR) Dockerfile.musl.runner -TESTCASE=testcases/pass.txt +TESTCASE=testcases/pass.1 run-runner: $(RUNTEST) $(MYST_EXEC) $(OPTS) $(RROOTFS) --app-config-path config-runner.json $(RUNNER) /testcases/pass.1 /dotnet-lib-release/ diff --git a/tests/hello_world/c/Makefile b/tests/hello_world/c/Makefile index c92968c30e..7d6803326d 100644 --- a/tests/hello_world/c/Makefile +++ b/tests/hello_world/c/Makefile @@ -22,3 +22,6 @@ myst: clean: rm -rf $(APPDIR) rootfs export ramfs + +gdb: + $(MYST_GDB) --args $(MYST_EXEC) rootfs /bin/hello red green blue $(OPTS) diff --git a/tests/hello_world/java/Makefile b/tests/hello_world/java/Makefile index 3191927570..ce2d37742c 100644 --- a/tests/hello_world/java/Makefile +++ b/tests/hello_world/java/Makefile @@ -23,3 +23,5 @@ tests: clean: rm -rf ext2rootfs appdir +gdb: + $(MYST_GDB) --args $(MYST_EXEC) $(OPTS) ext2rootfs /opt/openjdk-13/bin/java -ea Helloworld red green blue