Skip to content

Commit

Permalink
Add thread tracking extension to myst-thread
Browse files Browse the repository at this point in the history
  • Loading branch information
Zijie Wu committed Sep 21, 2022
1 parent 0fbb661 commit 5041968
Show file tree
Hide file tree
Showing 9 changed files with 308 additions and 7 deletions.
6 changes: 6 additions & 0 deletions debugger/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ TOP=$(abspath $(CURDIR)/..)
BUILDDIR=${TOP}/build

all: ${BUILDDIR}/bin/myst-gdb \
${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py \
${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py \
${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py \
${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py
Expand All @@ -14,6 +15,10 @@ ${BUILDDIR}/bin/myst-gdb:
${BUILDDIR}/lib/debugger/gdb-sgx-plugin:
mkdir -p ${BUILDDIR}/lib/debugger/gdb-sgx-plugin

${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py
cp ${CURDIR}/gdb-sgx-plugin/thread.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py

${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py: ${BUILDDIR}/lib/debugger/gdb-sgx-plugin
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py
cp ${CURDIR}/gdb-sgx-plugin/mprotect.py ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py
Expand All @@ -28,6 +33,7 @@ ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py: ${BUILDDIR}/lib/debugger/gdb-s

clean:
rm -f ${BUILDDIR}/bin/myst-gdb
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/thread.py
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/mprotect.py
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/symbol_analyzer.py
rm -f ${BUILDDIR}/lib/debugger/gdb-sgx-plugin/print.py
Expand Down
280 changes: 280 additions & 0 deletions debugger/gdb-sgx-plugin/thread.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,280 @@
import gdb
import math
import tempfile
from typing import List

DEBUG = False

INSTANCE_DEBUG_EXT = None

class ThreadCreateBreakpoint(gdb.Breakpoint):
def __init__(self):
"""
Set a breakpoing @ kernel/thread.c:_run_thread, this should be right before myst_setjmp(&thread->jmpbuf)
Choosing this location because this is where the newly created thread about to run
We can get the new thread pid/tid here
"""
super(ThreadCreateBreakpoint, self).__init__('kernel/thread.c:_run_thread', internal=False)

def stop(self):
# New thread info
ppid = int(gdb.parse_and_eval('((struct run_thread_arg*)arg_)->thread->process->ppid'))
pid = int(gdb.parse_and_eval('((struct run_thread_arg*)arg_)->thread->process->pid'))
tid = int(gdb.parse_and_eval('((struct run_thread_arg*)arg_)->thread->tid'))
tid_target = int(gdb.parse_and_eval('((struct run_thread_arg*)arg_)->target_tid'))
if DEBUG:
print("Created thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target)
INSTANCE_DEBUG_EXT.add_thread(pid, tid, ppid, tid_target)

# Add _main_thread only once as it won't change often
if not INSTANCE_DEBUG_EXT.main_thread_tracked():
main_ppid = int(gdb.parse_and_eval('_main_thread->process->ppid'))
main_pid = int(gdb.parse_and_eval('_main_thread->process->pid'))
main_tid = int(gdb.parse_and_eval('_main_thread->tid'))
main_tid_target = int(gdb.parse_and_eval('_main_thread->target_tid'))
if DEBUG:
print("Main thread: ppid, pid, tid, tid_target", main_ppid, main_pid, main_tid, main_tid_target)
INSTANCE_DEBUG_EXT.add_thread(main_pid, main_tid, main_ppid, main_tid_target)

INSTANCE_DEBUG_EXT.copy_parent_process(pid, ppid)
return False

class ThreadExitBreakpoint(gdb.Breakpoint):
def __init__(self):
"""
Set a breakpoint @ kernel/thread.c:myst_debug_hook_thread_exit, where myst_setjmp returns non-zero
Choosing this location because this is where the thread terminates
"""
super(ThreadExitBreakpoint, self).__init__('kernel/thread.c:myst_debug_hook_thread_exit', internal=False)

def stop(self):
# Get thread info that is about to terminate
ppid = int(gdb.parse_and_eval('thread->process->ppid'))
pid = int(gdb.parse_and_eval('thread->process->pid'))
tid = int(gdb.parse_and_eval('thread->tid'))
tid_target = int(gdb.parse_and_eval('thread->target_tid'))
if DEBUG:
print("Exit thread: ppid, pid, tid, tid_target", ppid, pid, tid, tid_target)

INSTANCE_DEBUG_EXT.remove_thread(tid)
return False

class MystExecBreakpoint(gdb.Breakpoint):
def __init__(self):
"""
Set a breakpoint @ kernel/exec.c:myst_exec, right after `process = thread->process`
grab program and args to run in Enclave, and cwd
"""
super(MystExecBreakpoint, self).__init__('kernel/exec.c:myst_exec', internal=False)

def parse_string_raw(self, str_raw):
"""
Parse the string given the gdb parse_and_eval result
Example input:
0x104df31f0 "/bin/usr/some/path"
Example return:
/bin/usr/some/path
"""
val = str(str_raw)
if ' ' not in val:
return val

# Don't include "
return val[val.index(' ') + 2:-1]

def stop(self):
# Same as parse and eval *argv@argc
argc = int(gdb.parse_and_eval('argc'))

argv_raw = [gdb.parse_and_eval('argv[{0}]'.format(i)) for i in range(argc)]
cwd_raw = gdb.parse_and_eval('thread->process->cwd')

argv = [self.parse_string_raw(each) for each in argv_raw]
cwd = self.parse_string_raw(cwd_raw)

# Get pid and ppid
ppid = int(gdb.parse_and_eval('thread->process->ppid'))
pid = int(gdb.parse_and_eval('thread->process->pid'))

INSTANCE_DEBUG_EXT.set_process_cmd_cwd(pid, ppid, argv, cwd)

return False

class DebugExtension:
"""
Manage all breakpoints and program info(process/thread/cmdline/etc)
"""

# Configure the default pid/tid for the main process
# We only gather _main_thread's information once
tid_main_thread = 101

def __init__(self):
self.bp_thread_create = ThreadCreateBreakpoint()
self.bp_thread_exit = ThreadExitBreakpoint()
self.bp_myst_exec = MystExecBreakpoint()

self.map_commands = {
'help': lambda: self._command_help(),
'thread': lambda: self._command_list_thread(),
'cmdline': lambda: self._command_cmdline(),
'exe': lambda: self._command_exe(),
'cwd': lambda: self._command_cwd()
}

# Store thread info
# Key: tid
# Value: tuple (pid, tid, ppid, tid_target)
self.threads = dict()

# Store process info
# Key: pid
# Value: tuple (pid, ppid, command line, cwd)
# where command line should be list of string
self.processes = dict()

self.clear()

def main_thread_tracked(self):
return self.tid_main_thread in self.threads

def add_thread(self, pid: int, tid: int, ppid: int, tid_target: int):
"""
ppid: parent pid
tid_target: tid on host
"""
self.threads[tid] = (pid, tid, ppid, tid_target)

def remove_thread(self, tid: int):
if tid not in self.threads:
print("DebugExtension Error: trying to remove non-exist thread ", tid)
return
del self.threads[tid]

def copy_parent_process(self, pid, ppid):
"""
When a new process is created, copy the cmdline from its parent
"""
if pid in self.processes:
if DEBUG:
print("Unexpected: pid {0} should not exist in self.processes".format(pid))
return

if ppid not in self.processes:
# Parent process might have already exited
return

_, _, cmdline, cwd = self.processes[ppid]
self.processes[pid] = (pid, ppid, cmdline, cwd)
if DEBUG:
print("Copied", self.processes[pid])

def set_process_cmd_cwd(self, pid: int, ppid: int, argv: List[int], cwd: str):
self.processes[pid] = (pid, ppid, argv, cwd)

def clear(self):
self.threads = dict()
self.processes = dict()

def dispatch(self, command: str):
if command not in self.map_commands:
print("DebugExtension Error: Invalid command, should be one of ", sorted(list(self.map_commands.keys())))
return

self.map_commands[command]()

def _command_help(self):
msg = """
myst-thread: Mystikos thread tracker
Commands:
myst-thread help
Print this message
myst-thread thread
List all tracked threads
myst-thread cmdline
List the command line for each process
myst-thread exe
List the executable for each process
myst-thread cwd
List the current working directory for each process
"""
print(msg)

def _command_list_thread(self):
if not self.threads:
print("No threads tracked")
return

print("List of Mystikos controlled threads:")
print("ppid pid tid (target_tid)")
for key in sorted(self.threads.keys()):
pid, tid, ppid, target_tid = self.threads[key]
print("{0} {1} {2} ({3})".format(ppid, pid, tid, target_tid))

def _command_cmdline(self):
"""
Similiar to gdb command 'info proc cmdline'
"""
if not self.processes:
print("No processes tracked")
return

for pid in sorted(self.processes.keys()):
pid, ppid, cmdline, _ = self.processes[pid]
print("Process {0} (Parent {1}) cmdline: {2}".format(pid, ppid, " ".join(cmdline)))

def _command_exe(self):
"""
Similiar to gdb command 'info proc exe'
"""
if not self.processes:
print("No processes tracked")
return

for pid in sorted(self.processes.keys()):
pid, ppid, cmdline, _ = self.processes[pid]
print("Process {0} (Parent {1}) exe: {2}".format(pid, ppid, cmdline[0]))

def _command_cwd(self):
"""
Similiar to gdb command 'info proc cwd'
"""
if not self.processes:
print("No processes tracked")
return

for pid in sorted(self.processes.keys()):
pid, ppid, _, cwd = self.processes[pid]
print("Process {0} (Parent {1}) cwd: {2}".format(pid, ppid, cwd))

command = """
define myst-thread
if $argc == 0
python INSTANCE_DEBUG_EXT.dispatch("help")
end
if $argc == 1
python INSTANCE_DEBUG_EXT.dispatch("$arg0")
end
end
"""

if __name__ == "__main__":
INSTANCE_DEBUG_EXT = DebugExtension()

# Register command with gdb.
with tempfile.NamedTemporaryFile('w') as f:
f.write(command)
f.flush()
gdb.execute('source %s' % f.name)

# Register exit_handler
def exit_handler(event):
global INSTANCE_DEBUG_EXT
INSTANCE_DEBUG_EXT.clear()
gdb.events.exited.connect(exit_handler)
1 change: 1 addition & 0 deletions debugger/myst-gdb
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ LD_PRELOAD=$OE_GDB_PTRACE_PATH gdb \
-iex "directory $MYST_GDB_PLUGIN_DIR" \
-iex "source $OE_GDB_PLUGIN_DIR/gdb_sgx_plugin.py" \
-iex "source $MYST_GDB_PLUGIN_DIR/print.py" \
-iex "source $MYST_GDB_PLUGIN_DIR/thread.py" \
-iex "source $MYST_GDB_PLUGIN_DIR/mprotect.py" \
-iex "source $MYST_GDB_PLUGIN_DIR/symbol_analyzer.py" \
-iex "set environment LD_PRELOAD" \
Expand Down
8 changes: 4 additions & 4 deletions doc/dev-jumpstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,15 @@ consists of the following artifacts:
* openenclave, including the outputs from building OE SDK.
1. Run a simple application built with gcc
```bash
cd tests/hello
cd tests/hello_world/c
make
make tests
```
In the 2nd step `make`, we create a temporary folder `appdir`, compile
During the 2nd step `make`, it creates a temporary folder `appdir`, compile
`hello.c` with `gcc`, and place the output executable under
`appdir/bin/hello`, finally we create a CPIO archive out of `appdir`.

In the 3rd step `make tests`, we launch `myst`, giving it the CPIO
During the 3rd step `make tests`, it launchs `myst`, giving it the CPIO
archive, the command to run (`/bin/hello` in this case), and
the command line arguments, e.g., "red", "green", and "blue".
With this step, we should see the following outputs:
Expand Down Expand Up @@ -125,7 +125,7 @@ O Call -> Calls into the host (untrusted environment)
1. For most applications under `tests`, we can launch the debugger with
command `make tests GDB=1`. For example:
```
cd Mystikos/tests/hello
cd tests/hello_world/c
make && make tests GDB=1
```
For applications that are run in [package mode](../solutions/dotnet/Makefile#23), ensure that the field `"Debug":1` is set in the `config.json` file, and the debugger can be launched using the run command:
Expand Down
11 changes: 10 additions & 1 deletion kernel/thread.c
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
#include <myst/times.h>
#include <myst/trace.h>

//#define TRACE
// #define TRACE

myst_spinlock_t myst_process_list_lock = MYST_SPINLOCK_INITIALIZER;

Expand Down Expand Up @@ -885,6 +885,13 @@ void myst_fork_exec_futex_wake(pid_t pid, pid_t tid)
**==============================================================================
*/

// This function is used by debugger/gdb-sgx-plugin/thread.py
void myst_debug_hook_thread_exit(const myst_thread_t* thread)
{
// This function should do nothing
assert(thread != NULL);
}

bool myst_valid_td(const void* td)
{
return td && ((const myst_td_t*)td)->self == td;
Expand Down Expand Up @@ -982,6 +989,8 @@ static long _run_thread(void* arg_)
{
/* ---------- running C-runtime thread descriptor ---------- */

myst_debug_hook_thread_exit(thread);

assert(myst_gettid() != -1);

/* restore the target thread descriptor */
Expand Down
2 changes: 1 addition & 1 deletion tests/dotnet-lib-5/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ rootfsr: $(RAPPDIR)
appdirr:
$(MYST-RETRY) $(APPBUILDER) -m -o $(RAPPDIR) Dockerfile.musl.runner

TESTCASE=testcases/pass.txt
TESTCASE=testcases/pass.1

run-runner:
$(RUNTEST) $(MYST_EXEC) $(OPTS) $(RROOTFS) --app-config-path config-runner.json $(RUNNER) /testcases/pass.1 /dotnet-lib-release/
Expand Down
2 changes: 1 addition & 1 deletion tests/gdb/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ GDB_CMDS=--batch \
-ex "backtrace" \
--args

GDB_MATCH=Breakpoint 2, .................. in print_hello ()
GDB_MATCH=Breakpoint 5, .................. in print_hello ()

# runtest timeouts cause gdb to hang
export NOTIMEOUT=1
Expand Down
Loading

0 comments on commit 5041968

Please sign in to comment.