From 62c0a02cdc075d674ea65ef0b4f73abc2f2a1808 Mon Sep 17 00:00:00 2001 From: Mike Brasher Date: Thu, 20 Aug 2020 21:54:38 +0000 Subject: [PATCH 1/6] added userspace isolation --- Makefile | 8 ++- config.mak | 1 + openenclave | 2 +- src/enclave/enclave_init.c | 96 +++++++++++++++++++++++-- src/main-oe/sgxlkl_run_oe.c | 49 +++++++++++-- user/Makefile | 50 +++++++++++++ user/README.md | 42 +++++++++++ user/enter.c | 77 ++++++++++++++++++++ user/stubs.c | 138 ++++++++++++++++++++++++++++++++++++ user/userargs.h | 52 ++++++++++++++ 10 files changed, 501 insertions(+), 14 deletions(-) create mode 100644 user/Makefile create mode 100644 user/README.md create mode 100644 user/enter.c create mode 100644 user/stubs.c create mode 100644 user/userargs.h diff --git a/Makefile b/Makefile index d09389b9c..d68af1c73 100755 --- a/Makefile +++ b/Makefile @@ -120,11 +120,15 @@ sgx-lkl: ${THIRD_PARTY_LIB_DEVICE_MAPPER} ${THIRD_PARTY_LIB_EXT2FS} ${THIRD_PART $(SGXLKL_LIB_TARGET): $(SGXLKL_BUILD_VARIANT) # Generate the RSA key and sign the libsgxlkl.so -$(BUILD_DIR)/$(SGXLKL_LIB_TARGET_SIGNED): $(SGXLKL_LIB_TARGET) +$(BUILD_DIR)/$(SGXLKL_LIB_TARGET_SIGNED): $(SGXLKL_LIB_TARGET) $(SGXLKL_USER_LIB_TARGET) + $(MAKE) -C user @echo "openssl genrsa -out private.pem -3 3072" @openssl genrsa -out $(BUILD_DIR)/private.pem -3 3072 @echo "oesign sign -e $(SGXLKL_LIB_TARGET) -c config/eeid-params.conf -k private.pem" - @$(OE_OESIGN_TOOL_PATH)/oesign sign -e $(BUILD_DIR)/$(SGXLKL_LIB_TARGET) -c $(OESIGN_CONFIG_PATH)/eeid-params.conf -k $(BUILD_DIR)/private.pem + $(OE_OESIGN_TOOL_PATH)/oesign sign -e "$(BUILD_DIR)/$(SGXLKL_LIB_TARGET):$(BUILD_DIR)/$(SGXLKL_USER_LIB_TARGET)" -c $(OESIGN_CONFIG_PATH)/eeid-params.conf -k $(BUILD_DIR)/private.pem + +$(SGXLKL_USER_LIB_TARGET): + $(MAKE) -C user # Create a link named build to appropiate build directory. create-build-link: diff --git a/config.mak b/config.mak index 04b4428d3..ab9481853 100755 --- a/config.mak +++ b/config.mak @@ -14,6 +14,7 @@ $(info $$SGXLKL_ROOT = [${SGXLKL_ROOT}]) SGXLKL_RUN_TARGET ?= sgx-lkl-run-oe SGXLKL_LIB_TARGET ?= libsgxlkl.so +SGXLKL_USER_LIB_TARGET ?= libsgxlkl-user.so SGXLKL_LIB_TARGET_SIGNED ?= libsgxlkl.so.signed SGXLKL_STATIC_LIB ?= libsgxlkl.a diff --git a/openenclave b/openenclave index fff725099..85971a67f 160000 --- a/openenclave +++ b/openenclave @@ -1 +1 @@ -Subproject commit fff7250998c47a36be6e108160b154e45d8a1d96 +Subproject commit 85971a67f89efcdc37446bd96bb10fb75dede0b6 diff --git a/src/enclave/enclave_init.c b/src/enclave/enclave_init.c index 6ef0d69e8..1d485ea6b 100644 --- a/src/enclave/enclave_init.c +++ b/src/enclave/enclave_init.c @@ -14,6 +14,8 @@ #include "enclave/wireguard_util.h" #include "shared/env.h" +#include "../../user/userargs.h" + extern struct mpmcq __scheduler_queue; _Noreturn void __dls3(elf64_stack_t* conf, void* tos); @@ -89,8 +91,74 @@ static void init_wireguard() wgu_list_devices(); } -static int startmain(void* args) +static int _sgxlkl_host_syscall_mprotect( + int* retval, void* addr, size_t len, int prot) +{ + return (int)sgxlkl_host_syscall_mprotect(retval, addr, len, prot); +} + +void _barrier(void) +{ + a_barrier(); +} + +static long _lkl_syscall_wrapper(long no, long* params) +{ + //sgxlkl_warn("syscall begin: no=%u\n", no); + long ret = lkl_syscall(no, params); + //sgxlkl_warn("syscall end: ret=%u\n", ret); + return ret; +} + +static void _enter_user_space( + int argc, + char** argv, + void* stack, + size_t num_ethreads, + struct timespec clock_res[4]) { + extern void* __oe_get_isolated_image_entry_point(void); + extern const void* __oe_get_isolated_image_base(); + typedef int (*sgxlkl_user_enter_proc_t)(void* userargs); + sgxlkl_user_enter_proc_t proc = __oe_get_isolated_image_entry_point(); + static sgxlkl_userargs_t _userargs = + { + /* ATTN:MEB: eliminate all of these bypasses except lkl_syscall */ + _lkl_syscall_wrapper, + sgxlkl_warn, + sgxlkl_error, + sgxlkl_fail, + lthread_current, + enclave_mmap, + _sgxlkl_host_syscall_mprotect, + }; + + if (!proc) + sgxlkl_fail("failed to obtain user space entry point"); + + _userargs.argc = argc; + _userargs.argv = argv; + _userargs.stack = stack; + _userargs.elf64_hdr = (const void*)__oe_get_isolated_image_base(); + _userargs.num_ethreads = num_ethreads; + _userargs.sw_debug_mode = sgxlkl_in_sw_debug_mode(); + memcpy(_userargs.clock_res, clock_res, sizeof(_userargs.clock_res)); + + (*proc)(&_userargs); +} + +typedef struct startmain_args +{ + int argc; + char** argv; + struct timespec clock_res[8]; +} +startmain_args_t; + +static int startmain(void* args_) +{ + startmain_args_t* args = args_; + __init_libc(sgxlkl_enclave_state.elf64_stack.envp, sgxlkl_enclave_state.elf64_stack.argv[0]); __libc_start_init(); @@ -114,9 +182,23 @@ static int startmain(void* args) init_wireguard(); find_and_mount_disks(); +/* Change to 0 to run application within the kernel image */ +#if 1 + /* Enter the isolated image */ + _enter_user_space( + args->argc, + args->argv, + &sgxlkl_enclave_state.elf64_stack, + sgxlkl_enclave_state.config->ethreads, + args->clock_res); +#else /* Launch stage 3 dynamic linker, passing in top of stack to overwrite. * The dynamic linker will then load the application proper; here goes! */ __dls3(&sgxlkl_enclave_state.elf64_stack, __builtin_frame_address(0)); + (void)_enter_user_space; + (void)args; +#endif + return 0; } int __libc_init_enclave(int argc, char** argv) @@ -156,7 +238,7 @@ int __libc_init_enclave(int argc, char** argv) max_lthreads = next_power_of_2(max_lthreads); newmpmcq(&__scheduler_queue, max_lthreads, 0); - + init_ethread_tp(); size_t espins = cfg->espins; @@ -166,9 +248,15 @@ int __libc_init_enclave(int argc, char** argv) SGXLKL_VERBOSE("calling _lthread_sched_init()\n"); _lthread_sched_init(cfg->stacksize); - if (lthread_create(<, NULL, startmain, NULL) != 0) + /* Run startmain() in a new lthread */ { - sgxlkl_fail("Failed to create lthread for startmain()\n"); + static startmain_args_t args; + args.argc = argc; + args.argv = argv; + memcpy(args.clock_res, tmp, sizeof(args.clock_res)); + + if (lthread_create(<, NULL, startmain, &args) != 0) + sgxlkl_fail("Failed to create lthread for startmain()\n"); } lthread_run(); diff --git a/src/main-oe/sgxlkl_run_oe.c b/src/main-oe/sgxlkl_run_oe.c index d38414fad..bfc73cff0 100644 --- a/src/main-oe/sgxlkl_run_oe.c +++ b/src/main-oe/sgxlkl_run_oe.c @@ -704,10 +704,10 @@ int getopt_sgxlkl(int argc, char* argv[], struct option long_options[]) return -1; } -/* Determines path of libsgxlkl.so.signed */ -void get_signed_libsgxlkl_path(char* path_buf, size_t len) +/* Determines path of the given library */ +void find_lib(const char* libname, char* path_buf, size_t len) { - /* Look for libsgxlkl.so.signed in: + /* Look for library in: * 1. . * 2. ../lib * 3. /lib @@ -756,7 +756,7 @@ void get_signed_libsgxlkl_path(char* path_buf, size_t len) "%.*s/%s", (int)base_len, base, - "libsgxlkl.so.signed") < max_len) + libname) < max_len) { // If accessible, path found. if (!access(path_buf, R_OK)) @@ -767,7 +767,19 @@ void get_signed_libsgxlkl_path(char* path_buf, size_t len) base += strspn(base, ":"); } - sgxlkl_host_fail("Unable to locate libsgxlkl.so.signed\n"); + sgxlkl_host_fail("Unable to locate %s\n", libname); +} + +/* Determines path of libsgxlkl.so.signed */ +void get_signed_libsgxlkl_path(char* path_buf, size_t len) +{ + find_lib("libsgxlkl.so.signed", path_buf, len); +} + +/* Determines path of libsgxlkl-user */ +void get_libsgxlkl_user_path(char* path_buf, size_t len) +{ + find_lib("libsgxlkl-user.so", path_buf, len); } void mk_clock_res_string(int clock) @@ -1358,6 +1370,7 @@ void* enclave_init(ethread_args_t* args) /* Creates an SGX-LKL enclave with enclave configuration in the EEID. */ void _create_enclave( char* libsgxlkl, + char* libsgxlkl_user, uint32_t oe_flags, oe_enclave_t** oe_enclave) { @@ -1367,6 +1380,7 @@ void _create_enclave( char* buffer = NULL; size_t buffer_size = 0; + char path[PATH_MAX]; serialize_enclave_config( &sgxlkl_host_state.enclave_config, &buffer, &buffer_size); @@ -1391,8 +1405,15 @@ void _create_enclave( setting.u.eeid = eeid; + // Format the follwing path : + if (snprintf(path, sizeof(path), "%s:%s", libsgxlkl, libsgxlkl_user) + >= sizeof(path)) + { + sgxlkl_host_fail("path overflow: %s:%s\n", libsgxlkl, libsgxlkl_user); + } + result = oe_create_sgxlkl_enclave( - libsgxlkl, OE_ENCLAVE_TYPE_SGX, oe_flags, &setting, 1, oe_enclave); + path, OE_ENCLAVE_TYPE_SGX, oe_flags, &setting, 1, oe_enclave); free(eeid); @@ -1682,6 +1703,7 @@ int main(int argc, char* argv[], char* envp[]) char* host_config_path = NULL; char* enclave_config_path = NULL; char libsgxlkl[PATH_MAX]; + char libsgxlkl_user[PATH_MAX]; // const sgxlkl_host_config_t* hconf = &host_state.config; const sgxlkl_enclave_config_t* econf = &sgxlkl_host_state.enclave_config; char* root_hd = NULL; @@ -1696,6 +1718,7 @@ int main(int argc, char* argv[], char* envp[]) cpu_set_t set; void* return_value; bool enclave_image_provided = false; + bool isolated_image_provided = false; oe_enclave_t* oe_enclave = NULL; uint32_t oe_flags = 0; @@ -1722,6 +1745,7 @@ int main(int argc, char* argv[], char* envp[]) {"help-tls", no_argument, 0, 't'}, {"help", no_argument, 0, 'h'}, {"enclave-image", required_argument, 0, 'e'}, + {"isolated-image", required_argument, 0, 'i'}, {"host-config", required_argument, 0, 'H'}, {"enclave-config", required_argument, 0, 'c'}, {0, 0, 0, 0}}; @@ -1746,6 +1770,10 @@ int main(int argc, char* argv[], char* envp[]) enclave_image_provided = true; strcpy(libsgxlkl, optarg); break; + case 'i': + enclave_image_provided = true; + strcpy(libsgxlkl_user, optarg); + break; case 'v': version(); exit(EXIT_SUCCESS); @@ -1863,6 +1891,13 @@ int main(int argc, char* argv[], char* envp[]) } sgxlkl_host_verbose_raw("result=%s\n", libsgxlkl); + sgxlkl_host_verbose("get_libsgxlkl_user_path... "); + if (!isolated_image_provided) + { + get_libsgxlkl_user_path(libsgxlkl_user, PATH_MAX); + } + sgxlkl_host_verbose_raw("result=%s\n", libsgxlkl_user); + parse_cpu_affinity_params( sgxlkl_host_state.config.ethreads_affinity, ðreads_cores, @@ -1903,7 +1938,7 @@ int main(int argc, char* argv[], char* envp[]) /* Enclave creation */ sgxlkl_host_verbose("oe_create_enclave...\n"); - _create_enclave(libsgxlkl, oe_flags, &oe_enclave); + _create_enclave(libsgxlkl, libsgxlkl_user, oe_flags, &oe_enclave); /* Perform host interface initialization */ sgxlkl_host_interface_initialization(); diff --git a/user/Makefile b/user/Makefile new file mode 100644 index 000000000..bc79c9046 --- /dev/null +++ b/user/Makefile @@ -0,0 +1,50 @@ +TOP = $(abspath ..) + +-include $(TOP)/sgx-lkl-musl/muslobjs.mak + +TARGET = $(TOP)/build_musl/libsgxlkl-user.so + +CFLAGS = -m64 -g -O3 -g -fPIC -Werror -std=c99 -nostdinc -ffreestanding -fexcess-precision=standard -frounding-math -Wa,--noexecstack -D_XOPEN_SOURCE=700 -pipe -fomit-frame-pointer -fno-unwind-tables -fno-asynchronous-unwind-tables -ffunction-sections -fdata-sections -Werror=implicit-function-declaration -Werror=implicit-int -Werror=pointer-sign -D__USE_GNU -DDL_NOMMU_SUPPORT=1 + +INCLUDES = +INCLUDES += -I$(TOP)/build_musl/sgx-lkl-musl/include +INCLUDES += -I$(TOP)/build_musl/openenclave/include +INCLUDES += -I$(TOP)/build_musl/config +INCLUDES += -I$(TOP)/sgx-lkl-musl/src/internal +INCLUDES += -I$(TOP)/sgx-lkl-musl/arch/x86_64 +INCLUDES += -I$(TOP)/src/include + +DYNAMIC_LIST = $(TOP)/sgx-lkl-musl/dynamic.list + +LDFLAGS1 = -Wl,--sort-section,alignment -Wl,--sort-common -Wl,--gc-sections -Wl,--hash-style=both -Wl,--no-undefined -Wl,--exclude-libs=ALL -Wl,--dynamic-list=$(DYNAMIC_LIST) -nostdlib -nodefaultlibs -nostartfiles +LDFLAGS1 += -Wl,-esgxlkl_user_enter + +LDFLAGS2 = -Wl,-Bstatic -Wl,-Bsymbolic -Wl,--export-dynamic -Wl,-pie -Wl,--build-id -Wl,-z,noexecstack -Wl,-z,now + +LDFLAGS = $(LDFLAGS1) $(LDFLAGS2) + +LDFLAGS += -lgcc + +SOURCES = $(wildcard *.c) + +ifndef MUSL_OBJECTS +$(error "please run $(TOP)/sgx-lkl-musl/Makefile first") +endif + +LOCAL_OBJECTS = $(SOURCES:.c=.o) +OBJECTS = $(LOCAL_OBJECTS) $(MUSL_OBJECTS) + +all: $(OBJECTS) + $(CC) -o $(TARGET) $(OBJECTS) $(LDFLAGS) + +%.o: %.c + $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $< + +clean: + rm -f $(TARGET) $(LOCAL_OBJECTS) + +muslobjs: + echo $(MUSL_OBJECTS) + +nm: + nm $(TARGET) | grep dls diff --git a/user/README.md b/user/README.md new file mode 100644 index 000000000..8bfb598e7 --- /dev/null +++ b/user/README.md @@ -0,0 +1,42 @@ +libsgxlkl-user.so +================= + +This directory builds **libsgxlkl-user.so**: the user-space image that hosts +the C runtime (crt), which contains the dynamic program loader (ldso) and the +the C library (libc). + +SGX-LKL builds two ELF images that are loaded into the enclave. + + - libsgxlkl.so (the kernel-space image) + - libsgxlkl-user.so (the user-space image) + +Both are passed to the Open Enclave **create_enclave()** function in the +**path** argument, which has the form: + + ":" + +For example: + + "libsgxlkl.so:libsgxlkl-user.so" + +Open Enclave loads the two images into distinct ELF memory regions. This +effectively isolates the symbols of the two images. The kernel enters the +user-space image through its entry point given by its ELF header, passing +state information and callbacks (through a C structure). The user-space +image calls back into the kernel via these callbacks. + +The main functions of the user-space image are to + + - Initialize the C library + - Load the application program and any shared libraries + - Start executing the application program + +During its execution, the program calls C library functions that may initiate +syscalls. These syscalls are forwarded to the kernel-space image for handling. + +This directory provides the following sources: + + - userargs.h - defines the struct passed by the kernel to the entry point. + - enter.c - contains the entry point (**sgxlkl_user_enter()**). + - stubs.c - contains stubs that invoke callback functions. + diff --git a/user/enter.c b/user/enter.c new file mode 100644 index 000000000..d7d37f716 --- /dev/null +++ b/user/enter.c @@ -0,0 +1,77 @@ +#include +#include + +#ifndef hidden +#define hidden __attribute__((__visibility__("hidden"))) +#endif + +#include "pthread_impl.h" +#include "userargs.h" + +_Noreturn void __dls3(void* conf, void* tos); +void __libc_start_init(void); +void sgxlkl_warn(const char* fmt, ...); +void __init_libc(char **envp, char *pn); +void* _dlstart_c(size_t base); + +_Noreturn void __dls3(void* conf, void* tos); +void __libc_start_init(void); +void sgxlkl_warn(const char* fmt, ...); +void __init_libc(char **envp, char *pn); +void* _dlstart_c(size_t base); +void init_sysconf(long nproc_conf, long nproc_onln); + +static inline void _barrier() +{ + __asm__ __volatile__( "" : : : "memory" ); +} + +/* forward declaration */ +struct dso; + +void __attribute__ ((noinline)) +__gdb_hook_load_debug_symbols(struct dso *dso, void *symmem, ssize_t symsz) +{ + __asm__ volatile ("" : : "m" (dso), "m" (symmem), "m" (symsz)); +} + +void __attribute__ ((noinline)) +__gdb_hook_load_debug_symbols_from_file(struct dso *dso, char *libpath) +{ + __asm__ volatile ("" : : "m" (dso), "m" (libpath)); +} + +void __attribute__ ((noinline)) +__gdb_hook_load_debug_symbols_wrap(struct dso *dso, void *symmem, ssize_t symsz) +{ + return __gdb_hook_load_debug_symbols(dso, symmem, symsz); +} + +void __attribute__ ((noinline)) +__gdb_hook_load_debug_symbols_from_file_wrap(struct dso *dso, char *libpath) +{ + return __gdb_hook_load_debug_symbols_from_file(dso, libpath); +} + +void sgxlkl_user_enter(sgxlkl_userargs_t* args) +{ + __sgxlkl_userargs = args; + + _dlstart_c((size_t)args->elf64_hdr); + + libc.user_tls_enabled = 1; + + init_sysconf(args->num_ethreads, args->num_ethreads); + + init_clock_res((struct timespec*)args->clock_res); + + __init_libc(args->argv + args->argc + 1, args->argv[0]); + + __libc_start_init(); + _barrier(); + + pthread_t self = __pthread_self(); + self->locale = &libc.global_locale; + + __dls3(args->stack, __builtin_frame_address(0)); +} diff --git a/user/stubs.c b/user/stubs.c new file mode 100644 index 000000000..f17988ff2 --- /dev/null +++ b/user/stubs.c @@ -0,0 +1,138 @@ +#include "userargs.h" +#include + +sgxlkl_userargs_t* __sgxlkl_userargs; + +void sgxlkl_warn(const char* fmt, ...); + +int snprintf(char *str, size_t size, const char *format, ...); + +/* +**============================================================================== +** +** syscall: +** +**============================================================================== +*/ + +long lkl_syscall(long no, long* params) +{ + long ret = __sgxlkl_userargs->ua_lkl_syscall(no, params); + + return ret; +} + +long __sgxlkl_log_syscall( + // sgxlkl_syscall_kind type, + uint32_t type, + long n, + long res, + int params_len, + ...) +{ + sgxlkl_warn("__sgxlkl_log_syscall() unimplemented in user space"); + return 0; +} + +/* +**============================================================================== +** +** bypasses: +** +**============================================================================== +*/ + +void sgxlkl_warn(const char* msg, ...) +{ + /* ATTN: ignore variadic arguments */ + return __sgxlkl_userargs->ua_sgxlkl_warn(msg); +} + +void sgxlkl_error(const char* msg, ...) +{ + /* ATTN: ignore variadic arguments */ + return __sgxlkl_userargs->ua_sgxlkl_error(msg); +} + +void sgxlkl_fail(const char* msg, ...) +{ + /* ATTN: ignore variadic arguments */ + return __sgxlkl_userargs->ua_sgxlkl_fail(msg); +} + +bool sgxlkl_in_sw_debug_mode() +{ + return __sgxlkl_userargs->sw_debug_mode; +} + +struct lthread* lthread_current() +{ + return __sgxlkl_userargs->ua_lthread_current(); +} + +#if 0 +int enclave_mmap_flags_supported(int flags, int fd) +{ + return __sgxlkl_userargs->ua_enclave_mmap_flags_supported(flags, fd); +} +#endif + +void* enclave_mmap( + void* addr, + size_t length, + int mmap_fixed, + int prot, + int zero_pages) +{ + return __sgxlkl_userargs->ua_enclave_mmap(addr, length, mmap_fixed, + prot, zero_pages); +} + +typedef enum +{ + OE_OK, + OE_FAILURE, +} +oe_result_t; + +oe_result_t sgxlkl_host_syscall_mprotect( + int* retval, void* addr, size_t len, int prot) +{ + return __sgxlkl_userargs->ua_sgxlkl_host_syscall_mprotect( + retval, addr, len, prot); +} + +/* +**============================================================================== +** +** weak form of main (will be overriden by app main) +** +**============================================================================== +*/ + +__attribute__((weak)) +void main() +{ +} + +/* +**============================================================================== +** +** undefined builtins: +** +**============================================================================== +*/ + +#pragma GCC diagnostic ignored "-Wbuiltin-declaration-mismatch" + +void __muldc3() +{ +} + +void __mulsc3() +{ +} + +void __mulxc3() +{ +} diff --git a/user/userargs.h b/user/userargs.h new file mode 100644 index 000000000..4e70ea633 --- /dev/null +++ b/user/userargs.h @@ -0,0 +1,52 @@ +#ifndef _SGXLKL_USER_FUNCTBL_H +#define _SGXLKL_USER_FUNCTBL_H + +#include +#include +#include + +typedef long time_t; + +struct sgxlkl_user_timespec +{ + time_t tv_sec; + long tv_nsec; +}; + +typedef int64_t off_t; + +typedef struct sgxlkl_userargs +{ + /* Functions: ATTN: remove all but lkl_syscall() */ + long (*ua_lkl_syscall)(long no, long* params); + void (*ua_sgxlkl_warn)(const char* msg, ...); + void (*ua_sgxlkl_error)(const char* msg, ...); + void (*ua_sgxlkl_fail)(const char* msg, ...); + struct lthread* (*ua_lthread_current)(void); + void* (*ua_enclave_mmap)( + void* addr, + size_t length, + int mmap_fixed, + int prot, + int zero_pages); + int (*ua_sgxlkl_host_syscall_mprotect)( + int* retval, void* addr, size_t len, int prot); + + /* Arguments */ + int argc; + char** argv; + void* stack; + const void* elf64_hdr; + size_t num_ethreads; + + /* to be passed to init_clock_res() */ + struct sgxlkl_user_timespec clock_res[8]; + + /* where in debug mode or not */ + bool sw_debug_mode; +} +sgxlkl_userargs_t; + +extern sgxlkl_userargs_t* __sgxlkl_userargs; + +#endif /* _SGXLKL_USER_FUNCTBL_H */ From ff2db6f62ae241497fe3dfd2808352566df45df1 Mon Sep 17 00:00:00 2001 From: Mike Brasher Date: Thu, 20 Aug 2020 21:58:38 +0000 Subject: [PATCH 2/6] sgx-lkl-musl userspace changes --- sgx-lkl-musl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sgx-lkl-musl b/sgx-lkl-musl index a6360f883..7f806f93d 160000 --- a/sgx-lkl-musl +++ b/sgx-lkl-musl @@ -1 +1 @@ -Subproject commit a6360f883fd906b18c1878254547ad72da28d7e6 +Subproject commit 7f806f93d86f8cb95d025bb8680c3ecda2913ef6 From 90ae96624392d67dcd0e38b247c76899d385da4d Mon Sep 17 00:00:00 2001 From: Mike Brasher Date: Thu, 20 Aug 2020 23:42:28 +0000 Subject: [PATCH 3/6] remove two syscall bypasses --- Makefile | 1 - src/enclave/enclave_init.c | 56 ++++++++++++++------------------------ user/Makefile | 3 +- user/enter.c | 8 +++++- user/stubs.c | 19 ------------- user/userargs.h | 3 -- 6 files changed, 30 insertions(+), 60 deletions(-) diff --git a/Makefile b/Makefile index d68af1c73..01bbdd03e 100755 --- a/Makefile +++ b/Makefile @@ -121,7 +121,6 @@ $(SGXLKL_LIB_TARGET): $(SGXLKL_BUILD_VARIANT) # Generate the RSA key and sign the libsgxlkl.so $(BUILD_DIR)/$(SGXLKL_LIB_TARGET_SIGNED): $(SGXLKL_LIB_TARGET) $(SGXLKL_USER_LIB_TARGET) - $(MAKE) -C user @echo "openssl genrsa -out private.pem -3 3072" @openssl genrsa -out $(BUILD_DIR)/private.pem -3 3072 @echo "oesign sign -e $(SGXLKL_LIB_TARGET) -c config/eeid-params.conf -k private.pem" diff --git a/src/enclave/enclave_init.c b/src/enclave/enclave_init.c index 1d485ea6b..89518b4d8 100644 --- a/src/enclave/enclave_init.c +++ b/src/enclave/enclave_init.c @@ -91,17 +91,6 @@ static void init_wireguard() wgu_list_devices(); } -static int _sgxlkl_host_syscall_mprotect( - int* retval, void* addr, size_t len, int prot) -{ - return (int)sgxlkl_host_syscall_mprotect(retval, addr, len, prot); -} - -void _barrier(void) -{ - a_barrier(); -} - static long _lkl_syscall_wrapper(long no, long* params) { //sgxlkl_warn("syscall begin: no=%u\n", no); @@ -119,32 +108,29 @@ static void _enter_user_space( { extern void* __oe_get_isolated_image_entry_point(void); extern const void* __oe_get_isolated_image_base(); - typedef int (*sgxlkl_user_enter_proc_t)(void* userargs); - sgxlkl_user_enter_proc_t proc = __oe_get_isolated_image_entry_point(); - static sgxlkl_userargs_t _userargs = - { - /* ATTN:MEB: eliminate all of these bypasses except lkl_syscall */ - _lkl_syscall_wrapper, - sgxlkl_warn, - sgxlkl_error, - sgxlkl_fail, - lthread_current, - enclave_mmap, - _sgxlkl_host_syscall_mprotect, - }; - - if (!proc) - sgxlkl_fail("failed to obtain user space entry point"); + typedef int (*sgxlkl_user_enter_proc_t)(void* args, size_t size); + sgxlkl_userargs_t args; + sgxlkl_user_enter_proc_t proc; - _userargs.argc = argc; - _userargs.argv = argv; - _userargs.stack = stack; - _userargs.elf64_hdr = (const void*)__oe_get_isolated_image_base(); - _userargs.num_ethreads = num_ethreads; - _userargs.sw_debug_mode = sgxlkl_in_sw_debug_mode(); - memcpy(_userargs.clock_res, clock_res, sizeof(_userargs.clock_res)); + memset(&args, 0, sizeof(args)); + + if (!(proc = __oe_get_isolated_image_entry_point())) + sgxlkl_fail("failed to obtain user space entry point"); - (*proc)(&_userargs); + args.ua_lkl_syscall = _lkl_syscall_wrapper; + args.ua_sgxlkl_warn = sgxlkl_warn; + args.ua_sgxlkl_error = sgxlkl_error; + args.ua_sgxlkl_fail = sgxlkl_fail; + args.ua_enclave_mmap = enclave_mmap; + args.argc = argc; + args.argv = argv; + args.stack = stack; + args.elf64_hdr = (const void*)__oe_get_isolated_image_base(); + args.num_ethreads = num_ethreads; + args.sw_debug_mode = sgxlkl_in_sw_debug_mode(); + memcpy(args.clock_res, clock_res, sizeof(args.clock_res)); + + (*proc)(&args, sizeof(args)); } typedef struct startmain_args diff --git a/user/Makefile b/user/Makefile index bc79c9046..6f7f83bca 100644 --- a/user/Makefile +++ b/user/Makefile @@ -35,7 +35,8 @@ LOCAL_OBJECTS = $(SOURCES:.c=.o) OBJECTS = $(LOCAL_OBJECTS) $(MUSL_OBJECTS) all: $(OBJECTS) - $(CC) -o $(TARGET) $(OBJECTS) $(LDFLAGS) + @ $(CC) -o $(TARGET) $(OBJECTS) $(LDFLAGS) + @ echo "########## Created $(TARGET)" %.o: %.c $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $< diff --git a/user/enter.c b/user/enter.c index d7d37f716..32ad1f119 100644 --- a/user/enter.c +++ b/user/enter.c @@ -53,10 +53,16 @@ __gdb_hook_load_debug_symbols_from_file_wrap(struct dso *dso, char *libpath) return __gdb_hook_load_debug_symbols_from_file(dso, libpath); } -void sgxlkl_user_enter(sgxlkl_userargs_t* args) +void sgxlkl_user_enter(sgxlkl_userargs_t* args, size_t args_size) { __sgxlkl_userargs = args; + if (sizeof(sgxlkl_userargs_t) != args_size) + { + a_crash(); + *((int*)0) = 0; + } + _dlstart_c((size_t)args->elf64_hdr); libc.user_tls_enabled = 1; diff --git a/user/stubs.c b/user/stubs.c index f17988ff2..2709aec5d 100644 --- a/user/stubs.c +++ b/user/stubs.c @@ -65,18 +65,6 @@ bool sgxlkl_in_sw_debug_mode() return __sgxlkl_userargs->sw_debug_mode; } -struct lthread* lthread_current() -{ - return __sgxlkl_userargs->ua_lthread_current(); -} - -#if 0 -int enclave_mmap_flags_supported(int flags, int fd) -{ - return __sgxlkl_userargs->ua_enclave_mmap_flags_supported(flags, fd); -} -#endif - void* enclave_mmap( void* addr, size_t length, @@ -95,13 +83,6 @@ typedef enum } oe_result_t; -oe_result_t sgxlkl_host_syscall_mprotect( - int* retval, void* addr, size_t len, int prot) -{ - return __sgxlkl_userargs->ua_sgxlkl_host_syscall_mprotect( - retval, addr, len, prot); -} - /* **============================================================================== ** diff --git a/user/userargs.h b/user/userargs.h index 4e70ea633..aba68b8aa 100644 --- a/user/userargs.h +++ b/user/userargs.h @@ -22,15 +22,12 @@ typedef struct sgxlkl_userargs void (*ua_sgxlkl_warn)(const char* msg, ...); void (*ua_sgxlkl_error)(const char* msg, ...); void (*ua_sgxlkl_fail)(const char* msg, ...); - struct lthread* (*ua_lthread_current)(void); void* (*ua_enclave_mmap)( void* addr, size_t length, int mmap_fixed, int prot, int zero_pages); - int (*ua_sgxlkl_host_syscall_mprotect)( - int* retval, void* addr, size_t len, int prot); /* Arguments */ int argc; From cec8b0b39a0a43dec259ab20fab7f7289ed00c63 Mon Sep 17 00:00:00 2001 From: Mike Brasher Date: Thu, 20 Aug 2020 23:48:30 +0000 Subject: [PATCH 4/6] cleanup --- sgx-lkl-musl | 2 +- user/Makefile | 2 +- user/enter.c | 9 --------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/sgx-lkl-musl b/sgx-lkl-musl index 7f806f93d..447f8a85e 160000 --- a/sgx-lkl-musl +++ b/sgx-lkl-musl @@ -1 +1 @@ -Subproject commit 7f806f93d86f8cb95d025bb8680c3ecda2913ef6 +Subproject commit 447f8a85ee09562bb6d38e6dcca31314bfab6f59 diff --git a/user/Makefile b/user/Makefile index 6f7f83bca..47ac0ea2e 100644 --- a/user/Makefile +++ b/user/Makefile @@ -36,7 +36,7 @@ OBJECTS = $(LOCAL_OBJECTS) $(MUSL_OBJECTS) all: $(OBJECTS) @ $(CC) -o $(TARGET) $(OBJECTS) $(LDFLAGS) - @ echo "########## Created $(TARGET)" + @ echo "Created $(TARGET)" %.o: %.c $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $< diff --git a/user/enter.c b/user/enter.c index 32ad1f119..63ed8742b 100644 --- a/user/enter.c +++ b/user/enter.c @@ -8,12 +8,6 @@ #include "pthread_impl.h" #include "userargs.h" -_Noreturn void __dls3(void* conf, void* tos); -void __libc_start_init(void); -void sgxlkl_warn(const char* fmt, ...); -void __init_libc(char **envp, char *pn); -void* _dlstart_c(size_t base); - _Noreturn void __dls3(void* conf, void* tos); void __libc_start_init(void); void sgxlkl_warn(const char* fmt, ...); @@ -58,10 +52,7 @@ void sgxlkl_user_enter(sgxlkl_userargs_t* args, size_t args_size) __sgxlkl_userargs = args; if (sizeof(sgxlkl_userargs_t) != args_size) - { a_crash(); - *((int*)0) = 0; - } _dlstart_c((size_t)args->elf64_hdr); From 698355e09307be383df454e9e9bf677ece5d445b Mon Sep 17 00:00:00 2001 From: Mike Brasher Date: Mon, 24 Aug 2020 15:41:40 +0000 Subject: [PATCH 5/6] added libsgxlkl-user.so install rule --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 01bbdd03e..520a3f7f4 100755 --- a/Makefile +++ b/Makefile @@ -153,6 +153,7 @@ install-git-pre-commit-hook: scripts/pre-commit install: mkdir -p ${PREFIX}/bin ${PREFIX}/lib ${PREFIX}/lib/gdb $(PREFIX)/lib/gdb/openenclave ${PREFIX}/share ${PREFIX}/share/schemas ${PREFIX}/tools + cp $(BUILD_DIR)/$(SGXLKL_USER_LIB_TARGET) $(PREFIX)/lib cp $(BUILD_DIR)/$(SGXLKL_LIB_TARGET_SIGNED) $(PREFIX)/lib cp $(BUILD_DIR)/$(SGXLKL_RUN_TARGET) $(PREFIX)/bin cp $(TOOLS)/sgx-lkl-java $(PREFIX)/bin From d96391eb3c7145f559fd85172ac5f78400d56a71 Mon Sep 17 00:00:00 2001 From: Vikas Tikoo Date: Fri, 28 Aug 2020 20:04:29 +0000 Subject: [PATCH 6/6] Add weak definition for __sgx_init_enclave --- user/stubs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/user/stubs.c b/user/stubs.c index 2709aec5d..e1a1bca52 100644 --- a/user/stubs.c +++ b/user/stubs.c @@ -96,6 +96,11 @@ void main() { } +__attribute__((weak)) +void __sgx_init_enclave() +{ +} + /* **============================================================================== **