From e92856fe6768d4f09553d6b032fbe2ebcca83bfc Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Thu, 1 Jun 2023 15:33:58 +0800
Subject: [PATCH 14/15] refactor rt_isula_exec and shim log

Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
 src/cmd/isulad-shim/common.c                  |  17 +-
 src/cmd/isulad-shim/common.h                  |   4 +-
 src/cmd/isulad-shim/main.c                    |  19 +--
 src/cmd/isulad-shim/process.c                 |  98 ++++++-----
 .../modules/runtime/isula/isula_rt_ops.c      | 153 ++++++++++--------
 5 files changed, 164 insertions(+), 127 deletions(-)

diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
index 3787cdfb..27836a8c 100644
--- a/src/cmd/isulad-shim/common.c
+++ b/src/cmd/isulad-shim/common.c
@@ -31,11 +31,20 @@
 
 int g_log_fd = -1;
 
+int init_shim_log(void)
+{
+    g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640);
+    if (g_log_fd < 0) {
+        return SHIM_ERR;
+    }
+    return SHIM_OK;
+}
+
 void signal_routine(int sig)
 {
     switch (sig) {
         case SIGALRM:
-            write_message(g_log_fd, ERR_MSG, "runtime timeout");
+            write_message(ERR_MSG, "runtime timeout");
             exit(EXIT_FAILURE);
         default:
             break;
@@ -228,12 +237,12 @@ int generate_random_str(char *id, size_t len)
     return SHIM_OK;
 }
 
-void write_message(int fd, const char *level, const char *fmt, ...)
+void write_message(const char *level, const char *fmt, ...)
 {
 #define MAX_MSG_JSON_TEMPLATE 32
 #define MAX_MESSAGE_CONTENT_LEN 128
 #define MAX_MESSAGE_LEN (MAX_MSG_JSON_TEMPLATE + MAX_MESSAGE_CONTENT_LEN)
-    if (fd < 0) {
+    if (g_log_fd < 0) {
         return;
     }
 
@@ -247,7 +256,7 @@ void write_message(int fd, const char *level, const char *fmt, ...)
     va_end(arg_list);
 
     snprintf(msg, MAX_MESSAGE_LEN - 1, "{\"level\": \"%s\", \"msg\": \"%s\"}\n", level, buf);
-    nwrite = write_nointr_in_total(fd, msg, strlen(msg));
+    nwrite = write_nointr_in_total(g_log_fd, msg, strlen(msg));
     if (nwrite < 0 || (size_t)nwrite != strlen(msg)) {
         return;
     }
diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
index 8cef5de2..a5991cc3 100644
--- a/src/cmd/isulad-shim/common.h
+++ b/src/cmd/isulad-shim/common.h
@@ -58,6 +58,8 @@ extern "C" {
 #define CONTAINER_ACTION_REBOOT 129
 #define CONTAINER_ACTION_SHUTDOWN 130
 
+int init_shim_log(void);
+
 void signal_routine(int sig);
 
 void util_usleep_nointerupt(unsigned long usec);
@@ -107,7 +109,7 @@ bool file_exists(const char *f);
 
 int cmd_combined_output(const char *binary, const char *params[], void *output, int *output_len);
 
-void write_message(int fd, const char *level, const char *fmt, ...);
+void write_message(const char *level, const char *fmt, ...);
 
 int generate_random_str(char *id, size_t len);
 
diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c
index ff06a633..22db251e 100644
--- a/src/cmd/isulad-shim/main.c
+++ b/src/cmd/isulad-shim/main.c
@@ -26,8 +26,6 @@
 #include "common.h"
 #include "process.h"
 
-extern int g_log_fd;
-
 static void set_timeout_exit(unsigned int timeout)
 {
     signal(SIGALRM, signal_routine);
@@ -97,8 +95,11 @@ int main(int argc, char **argv)
     uint64_t timeout = 0;
     pthread_t tid_epoll;
 
-    g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640);
-    if (g_log_fd < 0) {
+    ret = init_shim_log();
+    if (ret != SHIM_OK) {
+        // because shim log init error, print error msg to stderr.
+        // isulad can obtain the reason why shim exits.
+        dprintf(STDERR_FILENO, "failed to init shim log");
         _exit(EXIT_FAILURE);
     }
 
@@ -110,19 +111,19 @@ int main(int argc, char **argv)
 
     ret = set_subreaper();
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "set subreaper failed:%d", ret);
+        write_message(ERR_MSG, "set subreaper failed:%d", ret);
         exit(EXIT_FAILURE);
     }
 
     ret = parse_args(argc, argv, &container_id, &bundle, &rt_name, &log_level, &timeout);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "parse args failed:%d", ret);
+        write_message(ERR_MSG, "parse args failed:%d", ret);
         exit(EXIT_FAILURE);
     }
 
     p = new_process(container_id, bundle, rt_name);
     if (p == NULL) {
-        write_message(g_log_fd, ERR_MSG, "new process failed");
+        write_message(ERR_MSG, "new process failed");
         exit(EXIT_FAILURE);
     }
 
@@ -135,7 +136,7 @@ int main(int argc, char **argv)
         if (p->state->exit_fifo != NULL) {
             int efd = open_no_inherit("exit_fifo", O_WRONLY, -1);
             if (efd < 0) {
-                write_message(g_log_fd, ERR_MSG, "open exit pipe failed:%d", SHIM_SYS_ERR(errno));
+                write_message(ERR_MSG, "open exit pipe failed:%d", SHIM_SYS_ERR(errno));
                 exit(EXIT_FAILURE);
             }
             p->exit_fd = efd;
@@ -145,7 +146,7 @@ int main(int argc, char **argv)
     /* start epoll for io copy */
     ret = process_io_start(p, &tid_epoll);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret);
+        write_message(ERR_MSG, "process io init failed:%d", ret);
         exit(EXIT_FAILURE);
     }
 
diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
index 2e76574d..a9e65fcb 100644
--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
@@ -44,8 +44,6 @@
 #define DEFAULT_IO_COPY_BUF (16 * 1024)
 #define DEFAULT_LOG_FILE_SIZE (4 * 1024)
 
-extern int g_log_fd;
-
 static shim_client_process_state *load_process()
 {
     parser_error err = NULL;
@@ -53,7 +51,7 @@ static shim_client_process_state *load_process()
 
     p_state = shim_client_process_state_parse_file("process.json", NULL, &err);
     if (p_state == NULL) {
-        write_message(g_log_fd, ERR_MSG, "parse process state failed");
+        write_message(ERR_MSG, "parse process state failed");
     }
     /* "err" will definitely be allocated memory in the function above */
     free(err);
@@ -68,7 +66,7 @@ static int open_fifo_noblock(const char *path, mode_t mode)
     /* By default, We consider that the file has been created by isulad */
     fd = open_no_inherit(path, mode | O_NONBLOCK, -1);
     if (fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "open fifo file failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "open fifo file failed:%d", SHIM_SYS_ERR(errno));
         return -1;
     }
 
@@ -107,7 +105,7 @@ static int receive_fd(int sock)
      */
     int ret = recvmsg(sock, &msg, 0);
     if (ret <= 0) {
-        write_message(g_log_fd, ERR_MSG, "get console fd failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "get console fd failed:%d", SHIM_SYS_ERR(errno));
         free(cmptr);
         return -1;
     }
@@ -201,7 +199,7 @@ static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *d
     w_count = write_nointr_in_total(*fd_to, p->buf, r_count);
     if (w_count < 0) {
         /* When any error occurs, set the write fd -1  */
-        write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno));
+        write_message(WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno));
         close(*fd_to);
         *fd_to = -1;
     }
@@ -243,7 +241,7 @@ static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *
     w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count);
     if (w_count < 0) {
         /* When any error occurs, set the write fd -1  */
-        write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno));
+        write_message(WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno));
         close(p->isulad_io->out);
         p->isulad_io->out = -1;
     }
@@ -285,7 +283,7 @@ static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *
     w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count);
     if (w_count < 0) {
         /* When any error occurs, set the write fd -1  */
-        write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno));
+        write_message(WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno));
         close(p->isulad_io->err);
         p->isulad_io->err = -1;
     }
@@ -333,13 +331,13 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epo
 
     conn_fd = accept(p->listen_fd, NULL, NULL);
     if (conn_fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno));
         goto out;
     }
 
     p->recv_fd = receive_fd(conn_fd);
     if (check_fd(p->recv_fd) != true) {
-        write_message(g_log_fd, ERR_MSG, "check console fd failed");
+        write_message(ERR_MSG, "check console fd failed");
         goto out;
     }
 
@@ -348,19 +346,19 @@ static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epo
     // p->isulad_io->in ----> p->recv_fd
     ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
         goto out;
     }
     // p->recv_fd ----> p->isulad_io->out
     ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno));
         goto out;
     }
     // p->isulad_io->resize ----> p->recv_fd
     ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno));
         goto out;
     }
 
@@ -416,7 +414,7 @@ static stdio_t *initialize_io(process_t *p)
     /* don't open resize pipe */
     if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) ||
         (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) {
-        write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
         goto failure;
     }
 
@@ -481,7 +479,7 @@ static int console_init(process_t *p, struct epoll_descr *descr)
 
     fd = socket(AF_UNIX, SOCK_STREAM, 0);
     if (fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "create socket failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "create socket failed:%d", SHIM_SYS_ERR(errno));
         goto failure;
     }
 
@@ -491,13 +489,13 @@ static int console_init(process_t *p, struct epoll_descr *descr)
 
     ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr));
     if (ret < 0) {
-        write_message(g_log_fd, ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "bind console fd failed:%d", SHIM_SYS_ERR(errno));
         goto failure;
     }
 
     ret = listen(fd, 2);
     if (ret < 0) {
-        write_message(g_log_fd, ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "listen console fd failed:%d", SHIM_SYS_ERR(errno));
         goto failure;
     }
 
@@ -505,7 +503,7 @@ static int console_init(process_t *p, struct epoll_descr *descr)
 
     ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d",  p->listen_fd, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d",  p->listen_fd, SHIM_SYS_ERR(errno));
         goto failure;
     }
 
@@ -523,7 +521,7 @@ static int open_terminal_io(process_t *p, struct epoll_descr *descr)
 
     ret = new_temp_console_path(p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "get temp console sock path failed");
+        write_message(ERR_MSG, "get temp console sock path failed");
         return SHIM_ERR;
     }
 
@@ -545,19 +543,19 @@ static int open_generic_io(process_t *p, struct epoll_descr *descr)
     // p->isulad_io->in ----> p->shim_io->in
     ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
         return SHIM_ERR;
     }
     // p->shim_io->out ----> p->isulad_io->out
     ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add  out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add  out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno));
         return SHIM_ERR;
     }
     // p->shim_io->err ----> p->isulad_io->err
     ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno));
         return SHIM_ERR;
     }
 
@@ -608,14 +606,14 @@ static void *io_epoll_loop(void *data)
 
     ret = epoll_loop_open(&descr);
     if (ret != 0) {
-        write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno));
         exit(EXIT_FAILURE);
     }
 
     // sync fd: epoll loop will exit when recive sync fd event.
     ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p);
     if (ret != 0) {
-        write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno));
         exit(EXIT_FAILURE);
     }
 
@@ -625,7 +623,7 @@ static void *io_epoll_loop(void *data)
         ret = open_generic_io(p, &descr);
     }
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno));
         exit(EXIT_FAILURE);
     }
 
@@ -633,7 +631,7 @@ static void *io_epoll_loop(void *data)
 
     ret = epoll_loop(&descr, -1);
     if (ret != 0) {
-        write_message(g_log_fd, ERR_MSG, "epoll loop failed");
+        write_message(ERR_MSG, "epoll loop failed");
         exit(EXIT_FAILURE);
     }
 
@@ -649,7 +647,7 @@ static void *io_epoll_loop(void *data)
     if (fd_out > 0) {
         ret = set_non_block(fd_out);
         if (ret != SHIM_OK) {
-            write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno));
+            write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno));
             exit(EXIT_FAILURE);
         }
 
@@ -664,7 +662,7 @@ static void *io_epoll_loop(void *data)
     if (fd_err > 0) {
         ret = set_non_block(fd_err);
         if (ret != SHIM_OK) {
-            write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno));
+            write_message(ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno));
             exit(EXIT_FAILURE);
         }
 
@@ -696,12 +694,12 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s
 
     log_term = util_common_calloc_s(sizeof(log_terminal));
     if (log_term == NULL) {
-        write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal");
+        write_message(ERR_MSG, "Failed to calloc log_terminal");
         goto clean_out;
     }
 
     if (pthread_rwlock_init(&log_term->log_terminal_rwlock, NULL) != 0) {
-        write_message(g_log_fd, ERR_MSG, "Failed to init isulad conf rwlock");
+        write_message(ERR_MSG, "Failed to init isulad conf rwlock");
         goto clean_out;
     }
 
@@ -777,25 +775,25 @@ static int init_isulad_stdio(process_t *p)
 
     ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s",  p->state->isulad_stdin);
+        write_message(ERR_MSG, "Failed to open in isulad fd: %s",  p->state->isulad_stdin);
         goto failure;
     }
 
     ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s",  p->state->isulad_stdout);
+        write_message(ERR_MSG, "Failed to open out isulad fd: %s",  p->state->isulad_stdout);
         goto failure;
     }
 
     ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s",  p->state->isulad_stderr);
+        write_message(ERR_MSG, "Failed to open err isulad fd: %s",  p->state->isulad_stderr);
         goto failure;
     }
 
     ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize);
     if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s",  p->state->resize_fifo);
+        write_message(ERR_MSG, "Failed to open resize isulad fd: %s",  p->state->resize_fifo);
         goto failure;
     }
     return SHIM_OK;
@@ -862,7 +860,7 @@ process_t *new_process(char *id, char *bundle, char *runtime)
 
     p->sync_fd = eventfd(0, EFD_CLOEXEC);
     if (p->sync_fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno));
+        write_message(ERR_MSG, "Failed to create eventfd: %s", strerror(errno));
         goto failure;
     }
 
@@ -996,7 +994,7 @@ static void process_delete(process_t *p)
 
     cwd = getcwd(NULL, 0);
     if (cwd == NULL) {
-        write_message(g_log_fd, ERR_MSG, "get cwd failed when do process delete");
+        write_message(ERR_MSG, "get cwd failed when do process delete");
         return;
     }
     int nret = snprintf(log_path, PATH_MAX, "%s/log.json", cwd);
@@ -1094,13 +1092,13 @@ int create_process(process_t *p)
     int nread = -1;
 
     if (pipe2(exec_fd, O_CLOEXEC) != 0) {
-        write_message(g_log_fd, ERR_MSG, "create pipe failed when create process:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "create pipe failed when create process:%d", SHIM_SYS_ERR(errno));
         return SHIM_ERR;
     }
 
     pid_t pid = fork();
     if (pid == (pid_t) -1) {
-        write_message(g_log_fd, ERR_MSG, "fork failed when create process:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "fork failed when create process:%d", SHIM_SYS_ERR(errno));
         return SHIM_ERR;
     }
 
@@ -1120,7 +1118,7 @@ int create_process(process_t *p)
     }
     nread = read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff) - 1);
     if (nread > 0) {
-        write_message(g_log_fd, ERR_MSG, "runtime error");
+        write_message(ERR_MSG, "runtime error");
         ret = SHIM_ERR;
         goto out;
     }
@@ -1128,7 +1126,7 @@ int create_process(process_t *p)
     /* block to wait runtime pid exit */
     ret = waitpid(pid, NULL, 0);
     if (ret != pid) {
-        write_message(g_log_fd, ERR_MSG, "wait runtime failed:%d", SHIM_SYS_ERR(errno));
+        write_message(ERR_MSG, "wait runtime failed:%d", SHIM_SYS_ERR(errno));
         ret = SHIM_ERR;
         goto out;
     }
@@ -1136,7 +1134,7 @@ int create_process(process_t *p)
     /* save runtime pid */
     data = read_text_file("pid");
     if (data == NULL) {
-        write_message(g_log_fd, ERR_MSG, "read pid of runtime failed");
+        write_message(ERR_MSG, "read pid of runtime failed");
         goto out;
     }
     int ctr_pid = atoi(data);
@@ -1197,12 +1195,12 @@ static int waitpid_with_timeout(int ctr_pid,  int *status, const uint64_t timeou
     if (*status == CONTAINER_ACTION_REBOOT) {
         nret = setenv("CONTAINER_ACTION", "reboot", 1);
         if (nret != SHIM_OK) {
-            write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno));
+            write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno));
         }
     } else if (*status == CONTAINER_ACTION_SHUTDOWN) {
         nret = setenv("CONTAINER_ACTION", "shutdown", 1);
         if (nret != SHIM_OK) {
-            write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno));
+            write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno));
         }
     }
     return SHIM_OK;
@@ -1225,12 +1223,12 @@ static int wait_container_process_with_timeout(process_t *p, const uint64_t time
             if (*status == CONTAINER_ACTION_REBOOT) {
                 ret = setenv("CONTAINER_ACTION", "reboot", 1);
                 if (ret != SHIM_OK) {
-                    write_message(g_log_fd, WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno));
+                    write_message(WARN_MSG, "set reboot action failed:%d", SHIM_SYS_ERR(errno));
                 }
             } else if (*status == CONTAINER_ACTION_SHUTDOWN) {
                 ret = setenv("CONTAINER_ACTION", "shutdown", 1);
                 if (ret != SHIM_OK) {
-                    write_message(g_log_fd, WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno));
+                    write_message(WARN_MSG, "set shutdown action failed:%d", SHIM_SYS_ERR(errno));
                 }
             }
             return SHIM_OK;
@@ -1260,7 +1258,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const
         // kill container process to ensure process_kill_all effective
         nret = kill(p->ctr_pid, SIGKILL);
         if (nret < 0 && errno != ESRCH) {
-            write_message(g_log_fd, ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid);
+            write_message(ERR_MSG, "Can not kill process (pid=%d) with SIGKILL", p->ctr_pid);
             return SHIM_ERR;
         }
     }
@@ -1270,7 +1268,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const
     // wait atmost 120 seconds
     DO_RETRY_CALL(120, 1000000, nret, try_wait_all_child);
     if (nret != 0) {
-        write_message(g_log_fd, ERR_MSG, "Failed to wait all child after 120 seconds");
+        write_message(ERR_MSG, "Failed to wait all child after 120 seconds");
     }
 
     process_delete(p);
@@ -1280,13 +1278,13 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const
 
     if (p->sync_fd > 0) {
         if (eventfd_write(p->sync_fd, 1)) {
-            write_message(g_log_fd, ERR_MSG, "Failed to write sync fd");
+            write_message(ERR_MSG, "Failed to write sync fd");
         }
     }
 
     nret = pthread_join(tid_epoll, NULL);
     if (nret != 0) {
-        write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread");
+        write_message(ERR_MSG, "Failed to join epoll loop thread");
     }
 
     close(p->sync_fd);
@@ -1298,7 +1296,7 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const
     }
 
     if (ret == SHIM_ERR_TIMEOUT) {
-        write_message(g_log_fd, INFO_MSG, "Wait %d timeout", p->ctr_pid);
+        write_message(INFO_MSG, "Wait %d timeout", p->ctr_pid);
         return SHIM_ERR_TIMEOUT;
     }
 
diff --git a/src/daemon/modules/runtime/isula/isula_rt_ops.c b/src/daemon/modules/runtime/isula/isula_rt_ops.c
index dcc1d8ac..9adaf613 100644
--- a/src/daemon/modules/runtime/isula/isula_rt_ops.c
+++ b/src/daemon/modules/runtime/isula/isula_rt_ops.c
@@ -707,7 +707,7 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
                        int *exit_code, const char* timeout, int* shim_exit_code)
 {
     pid_t pid = 0;
-    int exec_fd[2] = { -1, -1 };
+    int shim_stderr_pipe[2] = { -1, -1 };
     int shim_stdout_pipe[2] = { -1, -1 };
     int num = 0;
     int ret = 0;
@@ -733,21 +733,21 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
         return -1;
     }
 
-    if (pipe2(exec_fd, O_CLOEXEC) != 0) {
-        ERROR("Failed to create pipe for shim create");
+    if (pipe2(shim_stderr_pipe, O_CLOEXEC) != 0) {
+        ERROR("Failed to create pipe for shim stderr");
         return -1;
     }
 
     if (pipe2(shim_stdout_pipe, O_CLOEXEC) != 0) {
-        ERROR("Failed to create pipe for shim exit code");
+        ERROR("Failed to create pipe for shim stdout");
         return -1;
     }
 
     pid = fork();
     if (pid < 0) {
         ERROR("Failed fork for shim parent %s", strerror(errno));
-        close(exec_fd[0]);
-        close(exec_fd[1]);
+        close(shim_stderr_pipe[0]);
+        close(shim_stderr_pipe[1]);
         close(shim_stdout_pipe[0]);
         close(shim_stdout_pipe[1]);
         return -1;
@@ -755,60 +755,64 @@ static int shim_create(bool fg, const char *id, const char *workdir, const char
 
     if (pid == (pid_t)0) {
         if (chdir(workdir) < 0) {
-            (void)dprintf(exec_fd[1], "%s: failed chdir to %s", id, workdir);
+            (void)dprintf(shim_stderr_pipe[1], "%s: failed chdir to %s", id, workdir);
             exit(EXIT_FAILURE);
         }
 
         if (fg) {
+            // child process, dup2 shim_stdout_pipe[1] to STDOUT
+            if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) {
+                (void)dprintf(shim_stderr_pipe[1], "Dup stdout fd error: %s", strerror(errno));
+                exit(EXIT_FAILURE);
+            }
+            // child process, dup2 shim_stderr_pipe[1] to STDERR
+            if (dup2(shim_stderr_pipe[1], STDERR_FILENO) < 0) {
+                (void)dprintf(shim_stderr_pipe[1], "Dup stderr fd error: %s", strerror(errno));
+                exit(EXIT_FAILURE);
+            }
             goto realexec;
         }
 
         // clear NOTIFY_SOCKET from the env to adapt runc create
         if (unsetenv("NOTIFY_SOCKET") != 0) {
-            (void)dprintf(exec_fd[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno));
+            (void)dprintf(shim_stderr_pipe[1], "%s: unset env NOTIFY_SOCKET failed %s", id, strerror(errno));
             exit(EXIT_FAILURE);
         }
 
         pid = fork();
         if (pid < 0) {
-            (void)dprintf(exec_fd[1], "%s: fork shim-process failed %s", id, strerror(errno));
+            (void)dprintf(shim_stderr_pipe[1], "%s: fork shim-process failed %s", id, strerror(errno));
             _exit(EXIT_FAILURE);
         }
         if (pid != 0) {
             if (file_write_int(fpid, pid) != 0) {
-                (void)dprintf(exec_fd[1], "%s: write %s with %d failed", id, fpid, pid);
+                (void)dprintf(shim_stderr_pipe[1], "%s: write %s with %d failed", id, fpid, pid);
             }
             _exit(EXIT_SUCCESS);
         }
 
 realexec:
         /* real shim process. */
-        close(exec_fd[0]);
+        close(shim_stderr_pipe[0]);
         close(shim_stdout_pipe[0]);
-        // child process, dup2 shim_stdout_pipe[1] to STDOUT
-        if (dup2(shim_stdout_pipe[1], STDOUT_FILENO) < 0) {
-            (void)dprintf(exec_fd[1], "Dup fd error: %s", strerror(errno));
-            exit(EXIT_FAILURE);
-        }
         
         if (setsid() < 0) {
-            (void)dprintf(exec_fd[1], "%s: failed setsid for process %d", id, getpid());
+            (void)dprintf(shim_stderr_pipe[1], "%s: failed setsid for process %d", id, getpid());
             exit(EXIT_FAILURE);
         }
-        int ignore_fd[2] = {-1, -1};
-        ignore_fd[0] = exec_fd[1];
-        ignore_fd[1] = shim_stdout_pipe[1];
-        if (util_check_inherited_exclude_fds(true, ignore_fd, 2) != 0) {
-            (void)dprintf(exec_fd[1], "close inherited fds failed");
+
+        if (util_check_inherited(true, shim_stderr_pipe[1]) != 0) {
+            (void)dprintf(shim_stderr_pipe[1], "close inherited fds failed");
+            exit(EXIT_FAILURE);
         }
 
         execvp(SHIM_BINARY, (char * const *)params);
-        (void)dprintf(exec_fd[1], "exec failed: %s", strerror(errno));
+        (void)dprintf(shim_stderr_pipe[1], "exec failed: %s", strerror(errno));
     }
 
-    close(exec_fd[1]);
+    close(shim_stderr_pipe[1]);
     close(shim_stdout_pipe[1]);
-    num = util_read_nointr(exec_fd[0], exec_buff, sizeof(exec_buff) - 1);
+    num = util_read_nointr(shim_stderr_pipe[0], exec_buff, sizeof(exec_buff) - 1);
     if (num > 0) {
         ERROR("Exec failed: %s", exec_buff);
         ret = -1;
@@ -829,9 +833,16 @@ realexec:
         goto out;
     }
 
+    // exit_code is NULL when command is create.
     if (exit_code == NULL) {
         goto out;
     }
+
+    // when exec in background, exit code is shim exit code
+    if (!fg) {
+        *exit_code = *shim_exit_code;
+        goto out;
+    }
     ret = util_read_nointr(shim_stdout_pipe[0], exit_code, sizeof(int));
     if (ret <= 0) {
         *exit_code = 137;
@@ -839,7 +850,7 @@ realexec:
     ret = 0;
 
 out:
-    close(exec_fd[0]);
+    close(shim_stderr_pipe[0]);
     close(shim_stdout_pipe[0]);
     if (ret != 0) {
         show_shim_runtime_errlog(workdir);
@@ -1146,19 +1157,63 @@ err_out:
     return NULL;
 }
 
-int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code)
+static int preparation_exec(const char *id, const char *runtime, const char *workdir, const char *exec_id,
+                            const rt_exec_params_t *params)
 {
-    char *exec_id = NULL;
-    defs_process *process = NULL;
-    const char **runtime_args = NULL;
+    int ret = 0;
     size_t runtime_args_len = 0;
-    char workdir[PATH_MAX] = { 0 };
     char resize_fifo_dir[PATH_MAX] = { 0 };
+    const char **runtime_args = NULL;
+    shim_client_process_state p = { 0 };
+    defs_process *process = NULL;
+
+    ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE);
+    if (ret < 0) {
+        ERROR("failed mkdir exec workdir %s", workdir);
+        return -1;
+    }
+
+    ret = snprintf(resize_fifo_dir, sizeof(resize_fifo_dir), "%s/%s", workdir, RESIZE_FIFO_NAME);
+    if (ret < 0) {
+        ERROR("failed join resize fifo full path");
+        return -1;
+    }
+
+    ret = console_fifo_create(resize_fifo_dir);
+    if (ret < 0) {
+        ERROR("failed create resize fifo file");
+        return -1;
+    }
+
+    process = params->spec;
+    runtime_args_len = get_runtime_args(runtime, &runtime_args);
+
+    p.exec = true;
+    p.isulad_stdin = (char *)params->console_fifos[0];
+    p.isulad_stdout = (char *)params->console_fifos[1];
+    p.isulad_stderr = (char *)params->console_fifos[2];
+    p.resize_fifo = resize_fifo_dir;
+    p.runtime_args = (char **)runtime_args;
+    p.runtime_args_len = runtime_args_len;
+    copy_process(&p, process);
+
+    ret = create_process_json_file(workdir, &p);
+    if (ret != 0) {
+        ERROR("%s: failed create exec json file", id);
+        return -1;
+    }
+
+    return 0;
+}
+
+int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *params, int *exit_code)
+{
     const char *cmd = NULL;
+    char *exec_id = NULL;
     int ret = 0;
-    char bundle[PATH_MAX] = { 0 };
     int pid = 0;
-    shim_client_process_state p = { 0 };
+    char bundle[PATH_MAX] = { 0 };
+    char workdir[PATH_MAX] = { 0 };
     char *timeout = NULL;
     int shim_exit_code = 0;
 
@@ -1166,8 +1221,6 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p
         ERROR("nullptr arguments not allowed");
         return -1;
     }
-    process = params->spec;
-    runtime_args_len = get_runtime_args(runtime, &runtime_args);
 
     ret = snprintf(bundle, sizeof(bundle), "%s/%s", params->rootpath, id);
     if (ret < 0) {
@@ -1191,36 +1244,10 @@ int rt_isula_exec(const char *id, const char *runtime, const rt_exec_params_t *p
         ERROR("failed join exec full path");
         goto out;
     }
-    ret = util_mkdir_p(workdir, DEFAULT_SECURE_DIRECTORY_MODE);
-    if (ret < 0) {
-        ERROR("failed mkdir exec workdir %s", workdir);
-        goto out;
-    }
 
-    ret = snprintf(resize_fifo_dir, sizeof(resize_fifo_dir), "%s/%s", workdir, RESIZE_FIFO_NAME);
-    if (ret < 0) {
-        ERROR("failed join resize fifo full path");
-        goto del_out;
-    }
-
-    ret = console_fifo_create(resize_fifo_dir);
-    if (ret < 0) {
-        ERROR("failed create resize fifo file");
-        goto del_out;
-    }
-
-    p.exec = true;
-    p.isulad_stdin = (char *)params->console_fifos[0];
-    p.isulad_stdout = (char *)params->console_fifos[1];
-    p.isulad_stderr = (char *)params->console_fifos[2];
-    p.resize_fifo = resize_fifo_dir;
-    p.runtime_args = (char **)runtime_args;
-    p.runtime_args_len = runtime_args_len;
-    copy_process(&p, process);
-
-    ret = create_process_json_file(workdir, &p);
+    ret = preparation_exec(id, runtime, workdir, exec_id, params);
     if (ret != 0) {
-        ERROR("%s: failed create exec json file", id);
+        ERROR("%s: failed to preparation for exec %s", id, exec_id);
         goto del_out;
     }
 
-- 
2.25.1

