[LibOS] Support executable scripts (shebangs) as entrypoint

Signed-off-by: aneessahib <[email protected]>

libos/include/libos_handle.h

@@ -277,7 +277,8 @@ int walk_handle_map(int (*callback)(struct libos_fd_handle*, struct libos_handle
                     struct libos_handle_map* map);
 
 int init_handle(void);
-int init_important_handles(void);
+int init_std_handles(void);
+int init_exec_handle(const char* const* argv, char*** out_new_argv);
 
 int open_executable(struct libos_handle* hdl, const char* path);
 

libos/include/libos_internal.h

@@ -17,7 +17,7 @@
 #include "pal.h"
 #include "pal_error.h"
 
-noreturn void libos_init(int argc, const char* const* argv, const char* const* envp);
+noreturn void libos_init(const char* const* argv, const char* const* envp);
 
 /* important macros and static inline functions */
 

libos/include/libos_process.h

@@ -69,7 +69,8 @@ struct libos_process {
 
 extern struct libos_process g_process;
 
-int init_process(int argc, const char* const* argv);
+int init_process(void);
+int init_process_cmdline(const char* const* argv);
 
 /* Allocates a new child process structure, initializing all fields. */
 struct libos_child_process* create_child_process(void);

libos/src/arch/x86_64/start.S

@@ -13,9 +13,9 @@ libos_start:
     xorq %rbp, %rbp
 
     # Arguments for libos_init:
-    movq 0(%rsp), %rdi         # argc
-    leaq 8(%rsp), %rsi         # argv
-    leaq 8(%rsi,%rdi,8), %rdx  # envp, after all args (including argv[argc] = NULL)
+    movq 0(%rsp), %r8          # argc, unused by libos_init, used to calculate location of envp
+    leaq 8(%rsp), %rdi         # argv
+    leaq 8(%rdi,%r8,8), %rsi   # envp, after all args (including argv[argc] = NULL)
 
     # Required by System V AMD64 ABI.
     andq  $~0xF, %rsp

libos/src/bookkeep/libos_handle.c

@@ -69,20 +69,21 @@ int open_executable(struct libos_handle* hdl, const char* path) {
     return ret;
 }
 
-static int init_exec_handle(void) {
+int init_exec_handle(const char* const* argv, char*** out_new_argv) {
     lock(&g_process.fs_lock);
     if (g_process.exec) {
         /* `g_process.exec` handle is already initialized if we did execve. See
          * `libos_syscall_execve_rtld`. */
         unlock(&g_process.fs_lock);
+        *out_new_argv = NULL;
         return 0;
     }
     unlock(&g_process.fs_lock);
 
     /* Initialize `g_process.exec` based on `libos.entrypoint` manifest key. */
     char* entrypoint = NULL;
     const char* exec_path;
-    struct libos_handle* hdl = NULL;
+    struct libos_handle* exec_handle = NULL;
     int ret;
 
     /* Initialize `g_process.exec` based on `libos.entrypoint` manifest key. */
@@ -109,28 +110,23 @@ static int init_exec_handle(void) {
         goto out;
     }
 
-    hdl = get_new_handle();
-    if (!hdl) {
-        ret = -ENOMEM;
-        goto out;
-    }
-
-    ret = open_executable(hdl, exec_path);
+    char** new_argv = NULL;
+    ret = load_and_check_exec(exec_path, argv, &exec_handle, &new_argv);
     if (ret < 0) {
-        log_error("Error opening executable %s: %d", exec_path, ret);
         goto out;
     }
 
     lock(&g_process.fs_lock);
-    g_process.exec = hdl;
-    get_handle(hdl);
+    g_process.exec = exec_handle;
+    get_handle(exec_handle);
     unlock(&g_process.fs_lock);
 
+    *out_new_argv = new_argv;
     ret = 0;
 out:
     free(entrypoint);
-    if (hdl)
-        put_handle(hdl);
+    if (exec_handle)
+        put_handle(exec_handle);
     return ret;
 }
 
@@ -152,12 +148,12 @@ int init_handle(void) {
     return 0;
 }
 
-int init_important_handles(void) {
+int init_std_handles(void) {
     int ret;
     struct libos_thread* thread = get_cur_thread();
 
     if (thread->handle_map)
-        goto done;
+        return 0;
 
     struct libos_handle_map* handle_map = get_thread_handle_map(thread);
 
@@ -228,9 +224,7 @@ int init_important_handles(void) {
         handle_map->fd_top = 2;
 
     unlock(&handle_map->lock);
-
-done:
-    return init_exec_handle();
+    return 0;
 }
 
 struct libos_handle* __get_fd_handle(uint32_t fd, int* fd_flags, struct libos_handle_map* map) {

libos/src/bookkeep/libos_process.c

@@ -17,7 +17,7 @@ typedef bool (*child_cmp_t)(const struct libos_child_process*, unsigned long);
 
 struct libos_process g_process = { .pid = 0 };
 
-int init_process(int argc, const char* const* argv) {
+int init_process(void) {
     if (g_process.pid) {
         /* `g_process` is already initialized, e.g. via checkpointing code. */
         return 0;
@@ -56,24 +56,26 @@ int init_process(int argc, const char* const* argv) {
     /* `g_process.exec` will be initialized later on (in `init_important_handles`). */
     g_process.exec = NULL;
 
+    return 0;
+}
+
+int init_process_cmdline(const char* const* argv) {
     /* The command line arguments passed are stored in /proc/self/cmdline as part of the proc fs.
      * They are not separated by space, but by NUL instead. So, it is essential to maintain the
      * cmdline_size also as a member here. */
-
     g_process.cmdline_size = 0;
     memset(g_process.cmdline, '\0', ARRAY_SIZE(g_process.cmdline));
     size_t tmp_size = 0;
 
-    for (int i = 0; i < argc; i++) {
-        if (tmp_size + strlen(argv[i]) + 1 > ARRAY_SIZE(g_process.cmdline))
+    for (const char* const* a = argv; *a; a++) {
+        if (tmp_size + strlen(*a) + 1 > ARRAY_SIZE(g_process.cmdline))
             return -ENOMEM;
 
-        memcpy(g_process.cmdline + tmp_size, argv[i], strlen(argv[i]));
-        tmp_size += strlen(argv[i]) + 1;
+        memcpy(g_process.cmdline + tmp_size, *a, strlen(*a));
+        tmp_size += strlen(*a) + 1;
     }
 
     g_process.cmdline_size = tmp_size;
-
     return 0;
 }
 

libos/src/libos_init.c

@@ -366,7 +366,9 @@ static int read_environs(const char* const* envp) {
         }                                                                   \
     } while (0)
 
-noreturn void libos_init(int argc, const char* const* argv, const char* const* envp) {
+noreturn void libos_init(const char* const* argv, const char* const* envp) {
+    int ret;
+
     g_pal_public_state = PalGetPalPublicState();
     assert(g_pal_public_state);
 
@@ -406,7 +408,7 @@ noreturn void libos_init(int argc, const char* const* argv, const char* const* e
     if (g_pal_public_state->parent_process) {
         struct checkpoint_hdr hdr;
 
-        int ret = read_exact(g_pal_public_state->parent_process, &hdr, sizeof(hdr));
+        ret = read_exact(g_pal_public_state->parent_process, &hdr, sizeof(hdr));
         if (ret < 0) {
             log_error("libos_init: failed to read the whole checkpoint header: %d", ret);
             PalProcessExit(1);
@@ -419,28 +421,37 @@ noreturn void libos_init(int argc, const char* const* argv, const char* const* e
     }
 
     RUN_INIT(init_ipc);
-    RUN_INIT(init_process, argc, argv);
+    RUN_INIT(init_process);
     RUN_INIT(init_mount_root);
     RUN_INIT(init_threading);
     RUN_INIT(init_mount);
-    RUN_INIT(init_important_handles);
+    RUN_INIT(init_std_handles);
+
+    char** expanded_argv;
+    RUN_INIT(init_exec_handle, argv, &expanded_argv);
+    RUN_INIT(init_process_cmdline, expanded_argv ? (const char* const*)expanded_argv : argv);
 
     /* Update log prefix after we initialized `g_process.exec` */
     log_setprefix(libos_get_tcb());
 
     RUN_INIT(init_async_worker);
 
-    char** new_argp;
+    char** new_argv;
     elf_auxv_t* new_auxv;
-    RUN_INIT(init_stack, argv, envp, &new_argp, &new_auxv);
+    RUN_INIT(init_stack, expanded_argv ? (const char* const*)expanded_argv : argv, envp, &new_argv,
+             &new_auxv);
+
+    if (expanded_argv) {
+        free(*expanded_argv);
+        free(expanded_argv);
+    }
 
-    /* TODO: Support running non-ELF executables (scripts) */
     RUN_INIT(init_elf_objects);
     RUN_INIT(init_signal_handling);
     RUN_INIT(init_ipc_worker);
 
     if (g_pal_public_state->parent_process) {
-        int ret = connect_to_process(g_process_ipc_ids.parent_vmid);
+        ret = connect_to_process(g_process_ipc_ids.parent_vmid);
         if (ret < 0) {
             log_error("libos_init: failed to establish IPC connection to parent: %d", ret);
             PalProcessExit(1);
@@ -491,7 +502,7 @@ noreturn void libos_init(int argc, const char* const* argv, const char* const* e
 
     /* At this point, the exec map has been either copied from checkpoint, or initialized in
      * `init_loader`. */
-    execute_elf_object(/*exec_map=*/NULL, new_argp, new_auxv);
+    execute_elf_object(/*exec_map=*/NULL, new_argv, new_auxv);
     /* UNREACHABLE */
 }
 

libos/src/sys/libos_exec.c

@@ -123,6 +123,10 @@ static int libos_syscall_execve_rtld(struct libos_handle* hdl, char** argv,
     char** new_argp;
     elf_auxv_t* new_auxv;
 
+    ret = init_process_cmdline((const char* const*)argv);
+    if (ret < 0)
+        return ret;
+
     /* note the typecast of argv here: the C standard disallows implicit conversion of `char**` ->
      * `const char* const*`, but in reality it is safe to do */
     ret = init_stack((const char* const*)argv, envp, &new_argp, &new_auxv);

libos/test/regression/shebang_test_script.manifest.template

@@ -0,0 +1,24 @@
+loader.entrypoint = "file:{{ gramine.libos }}"
+libos.entrypoint = "shebang_test_script.sh"
+loader.argv0_override = "shebang_test_script.sh"
+
+loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr/{{ arch_libdir }}"
+
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir(libc) }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/usr/{{ arch_libdir }}", uri = "file:/usr/{{ arch_libdir }}" },
+  { path = "/bin", uri = "file:/bin" },
+]
+
+sgx.thread_num = 16
+sgx.nonpie_binary = true
+sgx.debug = true
+
+sgx.trusted_files = [
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir(libc) }}/",
+  "file:/bin/sh",
+  "file:shebang_test_script.sh",
+  "file:scripts/",
+]

libos/test/regression/shebang_test_script.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec scripts/foo.sh

libos/test/regression/test_libos.py

@@ -221,14 +221,24 @@ def test_210_exec_invalid_args(self):
         self.assertIn('execve(invalid-argv) correctly returned error', stdout)
         self.assertIn('execve(invalid-envp) correctly returned error', stdout)
 
-    @unittest.skipIf(USES_MUSL, 'Test uses /bin/sh from the host which is built against Glibc')
+    @unittest.skipIf(USES_MUSL,
+        'Test uses /bin/sh from the host which is usually built against glibc')
     def test_211_exec_script(self):
         stdout, _ = self.run_binary(['exec_script'])
         self.assertIn('Printing Args: '
             'scripts/baz.sh ECHO FOXTROT GOLF scripts/bar.sh '
             'ALPHA BRAVO CHARLIE DELTA '
             'scripts/foo.sh STRING FROM EXECVE', stdout)
 
+    @unittest.skipIf(USES_MUSL,
+        'Test uses /bin/sh from the host which is usually built against glibc')
+    def test_212_shebang_test_script(self):
+        stdout, _ = self.run_binary(['shebang_test_script'])
+        self.assertIn('Printing Args: '
+            'scripts/baz.sh ECHO FOXTROT GOLF scripts/bar.sh '
+            'ALPHA BRAVO CHARLIE DELTA '
+            'scripts/foo.sh', stdout)
+
     def test_220_send_handle(self):
         path = 'tmp/send_handle_test'
         try:

libos/test/regression/tests.toml

@@ -90,6 +90,7 @@ manifests = [
   "select",
   "send_handle",
   "shared_object",
+  "shebang_test_script",
   "sigaction_per_process",
   "sigaltstack",
   "sighandler_reset",