From 15275529aed5b4f2e317902dfd3390ee114b310f Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Sun, 16 Apr 2023 13:56:18 +0800
Subject: [PATCH 57/57] change isulad-shim epoll struct

Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
 src/cmd/isulad-shim/common.c           |   9 +
 src/cmd/isulad-shim/common.h           |   2 +
 src/cmd/isulad-shim/linked_list.h      | 134 ++++
 src/cmd/isulad-shim/main.c             |  13 +-
 src/cmd/isulad-shim/mainloop.c         | 168 +++++
 src/cmd/isulad-shim/mainloop.h         |  53 ++
 src/cmd/isulad-shim/process.c          | 941 ++++++++++++-------------
 src/cmd/isulad-shim/process.h          |  47 +-
 test/cmd/isulad-shim/CMakeLists.txt    |   1 +
 test/cmd/isulad-shim/isulad-shim_ut.cc |   1 +
 10 files changed, 825 insertions(+), 544 deletions(-)
 create mode 100644 src/cmd/isulad-shim/linked_list.h
 create mode 100644 src/cmd/isulad-shim/mainloop.c
 create mode 100644 src/cmd/isulad-shim/mainloop.h

diff --git a/src/cmd/isulad-shim/common.c b/src/cmd/isulad-shim/common.c
index e1ca96e1..18443385 100644
--- a/src/cmd/isulad-shim/common.c
+++ b/src/cmd/isulad-shim/common.c
@@ -524,4 +524,13 @@ err_out:
     util_free_array(res_array);
     errno = tmp_errno;
     return NULL;
+}
+
+void *util_common_calloc_s(size_t size)
+{
+    if (size == 0 || size > MAX_MEMORY_SIZE) {
+        return NULL;
+    }
+
+    return calloc((size_t)1, size);
 }
\ No newline at end of file
diff --git a/src/cmd/isulad-shim/common.h b/src/cmd/isulad-shim/common.h
index 3de16ace..d733823b 100644
--- a/src/cmd/isulad-shim/common.h
+++ b/src/cmd/isulad-shim/common.h
@@ -128,6 +128,8 @@ char *util_strdup_s(const char *src);
 
 char **util_string_split_multi(const char *src_str, char delim);
 
+void *util_common_calloc_s(size_t size);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/cmd/isulad-shim/linked_list.h b/src/cmd/isulad-shim/linked_list.h
new file mode 100644
index 00000000..3a87990a
--- /dev/null
+++ b/src/cmd/isulad-shim/linked_list.h
@@ -0,0 +1,134 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container linked list function definition
+ ******************************************************************************/
+#ifndef __LINKED_LIST_H
+#define __LINKED_LIST_H
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct linked_list {
+    void *elem;
+    struct linked_list *next;
+    struct linked_list *prev;
+};
+
+/* Iterate through an linked list. */
+#define linked_list_for_each(__iterator, __list) \
+    for ((__iterator) = (__list)->next; \
+         (__iterator) != (__list); \
+         (__iterator) = (__iterator)->next)
+
+/* Iterate safely through an linked list. */
+#define linked_list_for_each_safe(__iterator, __list, __next) \
+    for ((__iterator) = (__list)->next, (__next) = (__iterator)->next; \
+         (__iterator) != (__list); \
+         (__iterator) = (__next), (__next) = (__next)->next)
+
+/* Initialize list. */
+static inline void linked_list_init(struct linked_list *list)
+{
+    list->elem = NULL;
+    list->next = list->prev = list;
+}
+
+/* Add an element to a list. See linked_list_add() and linked_list_add_tail() for an
+ * idiom. */
+static inline void linked_list_add_elem(struct linked_list *list, void *elem)
+{
+    list->elem = elem;
+}
+
+/* Retrieve first element of list. */
+static inline void *linked_list_first_elem(struct linked_list *list)
+{
+    return list->next->elem;
+}
+
+/* Retrieve last element of list. */
+static inline void *linked_list_last_elem(struct linked_list *list)
+{
+    return list->prev->elem;
+}
+
+/* Retrieve first node of list. */
+static inline void *linked_list_first_node(struct linked_list *list)
+{
+    return list->next;
+}
+
+/* Determine if list is empty. */
+static inline int linked_list_empty(struct linked_list *list)
+{
+    return list == list->next;
+}
+
+/* Workhorse to be called from linked_list_add() and linked_list_add_tail(). */
+static inline void __linked_list_add(struct linked_list *newlist,
+                                     struct linked_list *prev,
+                                     struct linked_list *next)
+{
+    next->prev = newlist;
+    newlist->next = next;
+    newlist->prev = prev;
+    prev->next = newlist;
+}
+
+/* Idiom to add an element to the beginning of an linked list */
+static inline void linked_list_add(struct linked_list *head,
+                                   struct linked_list *list)
+{
+    __linked_list_add(list, head, head->next);
+}
+
+/* Idiom to add an element to the end of an linked list */
+static inline void linked_list_add_tail(struct linked_list *head,
+                                        struct linked_list *list)
+{
+    __linked_list_add(list, head->prev, head);
+}
+
+/* Idiom to free an linked list */
+static inline void linked_list_del(const struct linked_list *list)
+{
+    struct linked_list *next = NULL;
+    struct linked_list *prev = NULL;
+
+    next = list->next;
+    prev = list->prev;
+    next->prev = prev;
+    prev->next = next;
+}
+
+/* Return length of the list. */
+static inline size_t linked_list_len(struct linked_list *list)
+{
+    size_t i = 0;
+    struct linked_list *iter = NULL;
+    linked_list_for_each(iter, list) {
+        i++;
+    }
+
+    return i;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
diff --git a/src/cmd/isulad-shim/main.c b/src/cmd/isulad-shim/main.c
index 68e99e53..0cccdf06 100644
--- a/src/cmd/isulad-shim/main.c
+++ b/src/cmd/isulad-shim/main.c
@@ -105,9 +105,9 @@ int main(int argc, char **argv)
     int ret = SHIM_ERR;
     int efd = -1;
     process_t *p = NULL;
-    pthread_t tid_accept;
     // execSync timeout
     uint64_t timeout = 0;
+    pthread_t tid_epoll;
 
     g_log_fd = open_no_inherit(SHIM_LOG_NAME, O_CREAT | O_WRONLY | O_APPEND | O_SYNC, 0640);
     if (g_log_fd < 0) {
@@ -154,18 +154,13 @@ int main(int argc, char **argv)
         }
     }
 
-    /* create main loop and start epoll for io copy */
-    ret = process_io_init(p);
+    /* start epoll for io copy */
+    ret = process_io_start(p, &tid_epoll);
     if (ret != SHIM_OK) {
         write_message(g_log_fd, ERR_MSG, "process io init failed:%d", ret);
         exit(EXIT_FAILURE);
     }
 
-    ret = open_io(p, &tid_accept);
-    if (ret != SHIM_OK) {
-        exit(EXIT_FAILURE);
-    }
-
     ret = create_process(p);
     if (ret != SHIM_OK) {
         if (p->console_sock_path != NULL) {
@@ -176,5 +171,5 @@ int main(int argc, char **argv)
 
     released_timeout_exit();
 
-    return process_signal_handle_routine(p, tid_accept, timeout);
+    return process_signal_handle_routine(p, tid_epoll, timeout);
 }
diff --git a/src/cmd/isulad-shim/mainloop.c b/src/cmd/isulad-shim/mainloop.c
new file mode 100644
index 00000000..53c4d856
--- /dev/null
+++ b/src/cmd/isulad-shim/mainloop.c
@@ -0,0 +1,168 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container mainloop functions
+ ******************************************************************************/
+#include "mainloop.h"
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/epoll.h>
+
+#include "common.h"
+
+struct epoll_loop_handler {
+    epoll_loop_callback_t cb;
+    int cbfd;
+    void *cbdata;
+};
+
+#define MAX_EVENTS 100
+
+/* epoll loop */
+int epoll_loop(struct epoll_descr *descr, int t)
+{
+    int i;
+    int ret = 0;
+    struct epoll_loop_handler *epoll_handler = NULL;
+    struct epoll_event evs[MAX_EVENTS];
+
+    while (1) {
+        int ep_fds = epoll_wait(descr->fd, evs, MAX_EVENTS, t);
+        if (ep_fds < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+            ret = -1;
+            goto out;
+        }
+
+        for (i = 0; i < ep_fds; i++) {
+            epoll_handler = (struct epoll_loop_handler *)(evs[i].data.ptr);
+            if (epoll_handler->cb(epoll_handler->cbfd, evs[i].events, epoll_handler->cbdata, descr) !=
+                EPOLL_LOOP_HANDLE_CONTINUE) {
+                goto out;
+            }
+        }
+
+        if (ep_fds == 0 && t != 0) {
+            if (descr->timeout_cb != NULL) {
+                descr->timeout_cb(descr->timeout_cbdata);
+            }
+            goto out;
+        }
+
+        if (linked_list_empty(&descr->handler_list)) {
+            goto out;
+        }
+    }
+out:
+    return ret;
+}
+
+/* epoll loop add handler */
+int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data)
+{
+    struct epoll_event ev = { 0 };
+    struct epoll_loop_handler *epoll_handler = NULL;
+    struct linked_list *node = NULL;
+
+    // if fd == -1; dose not add handler for it.
+    if (fd < 0) {
+        return 0;
+    }
+
+    epoll_handler = util_common_calloc_s(sizeof(*epoll_handler));
+    if (epoll_handler == NULL) {
+        goto fail_out;
+    }
+
+    epoll_handler->cbfd = fd;
+    epoll_handler->cb = callback;
+    epoll_handler->cbdata = data;
+
+    ev.events = EPOLLIN;
+    ev.data.ptr = epoll_handler;
+
+    if (epoll_ctl(descr->fd, EPOLL_CTL_ADD, fd, &ev) < 0) {
+        goto fail_out;
+    }
+
+    node = util_common_calloc_s(sizeof(struct linked_list));
+    if (node == NULL) {
+        goto fail_out;
+    }
+
+    node->elem = epoll_handler;
+    linked_list_add(&descr->handler_list, node);
+    return 0;
+
+fail_out:
+    (void)epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, &ev);
+    free(epoll_handler);
+    return -1;
+}
+
+/* epoll loop del handler */
+int epoll_loop_del_handler(struct epoll_descr *descr, int fd)
+{
+    struct epoll_loop_handler *epoll_handler = NULL;
+    struct linked_list *index = NULL;
+
+    linked_list_for_each(index, &descr->handler_list) {
+        epoll_handler = index->elem;
+
+        if (fd == epoll_handler->cbfd) {
+            if (epoll_ctl(descr->fd, EPOLL_CTL_DEL, fd, NULL)) {
+                goto fail_out;
+            }
+
+            linked_list_del(index);
+            free(index->elem);
+            free(index);
+            return 0;
+        }
+    }
+
+fail_out:
+    return -1;
+}
+
+/* epoll loop open */
+int epoll_loop_open(struct epoll_descr *descr)
+{
+    descr->fd = epoll_create1(EPOLL_CLOEXEC);
+    if (descr->fd < 0) {
+        return -1;
+    }
+
+    linked_list_init(&(descr->handler_list));
+    descr->timeout_cb = NULL;
+    descr->timeout_cbdata = NULL;
+    return 0;
+}
+
+/* epoll loop close */
+int epoll_loop_close(struct epoll_descr *descr)
+{
+    struct linked_list *index = NULL;
+    struct linked_list *next = NULL;
+
+    linked_list_for_each_safe(index, &(descr->handler_list), next) {
+        linked_list_del(index);
+        free(index->elem);
+        free(index);
+    }
+
+    return close(descr->fd);
+}
diff --git a/src/cmd/isulad-shim/mainloop.h b/src/cmd/isulad-shim/mainloop.h
new file mode 100644
index 00000000..7a4f1cfd
--- /dev/null
+++ b/src/cmd/isulad-shim/mainloop.h
@@ -0,0 +1,53 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2018-2019. All rights reserved.
+ * iSulad licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: tanyifeng
+ * Create: 2018-11-08
+ * Description: provide container mainloop definition
+ ******************************************************************************/
+#ifndef UTILS_CUTILS_MAINLOOP_H
+#define UTILS_CUTILS_MAINLOOP_H
+
+#include <stdint.h>
+#include "linked_list.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void (*epoll_timeout_callback_t)(void *data);
+
+struct epoll_descr {
+    int fd;
+    struct linked_list handler_list;
+    epoll_timeout_callback_t timeout_cb;
+    void *timeout_cbdata;
+};
+
+#define EPOLL_LOOP_HANDLE_CONTINUE 0
+#define EPOLL_LOOP_HANDLE_CLOSE 1
+
+typedef int (*epoll_loop_callback_t)(int fd, uint32_t event, void *data, struct epoll_descr *descr);
+
+extern int epoll_loop(struct epoll_descr *descr, int t);
+
+extern int epoll_loop_add_handler(struct epoll_descr *descr, int fd, epoll_loop_callback_t callback, void *data);
+
+extern int epoll_loop_del_handler(struct epoll_descr *descr, int fd);
+
+extern int epoll_loop_open(struct epoll_descr *descr);
+
+extern int epoll_loop_close(struct epoll_descr *descr);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/cmd/isulad-shim/process.c b/src/cmd/isulad-shim/process.c
index a5e0bd39..c31f8af6 100644
--- a/src/cmd/isulad-shim/process.c
+++ b/src/cmd/isulad-shim/process.c
@@ -28,6 +28,7 @@
 #include <string.h>
 #include <errno.h>
 #include <sys/ioctl.h>
+#include <sys/eventfd.h>
 #include <termios.h> // IWYU pragma: keep
 #include <sys/resource.h> // IWYU pragma: keep
 #include <isula_libutils/json_common.h>
@@ -37,6 +38,7 @@
 
 #include "common.h"
 #include "terminal.h"
+#include "mainloop.h"
 
 #define MAX_EVENTS 100
 #define DEFAULT_IO_COPY_BUF (16 * 1024)
@@ -78,7 +80,7 @@ static int receive_fd(int sock)
     u_char *pfd = NULL;
     int fd = -1;
     int cmsgsize = CMSG_LEN(sizeof(int));
-    struct cmsghdr *cmptr = (struct cmsghdr *)calloc(1, cmsgsize);
+    struct cmsghdr *cmptr = (struct cmsghdr *)util_common_calloc_s(cmsgsize);
     if (cmptr == NULL) {
         return -1;
     }
@@ -128,107 +130,6 @@ static bool check_fd(int fd)
     return true;
 }
 
-static int add_io_dispatch(int epfd, io_thread_t *io_thd, int from, int to)
-{
-    int ret = SHIM_ERR;
-
-    if (io_thd == NULL || io_thd->ioc == NULL) {
-        return SHIM_ERR;
-    }
-
-    io_copy_t *ioc = io_thd->ioc;
-
-    if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
-        return SHIM_ERR;
-    }
-    /* add src fd */
-    if (from != -1 && ioc->fd_from == -1) {
-        ioc->fd_from = from;
-        struct epoll_event ev;
-        ev.events = EPOLLIN;
-        ev.data.ptr = io_thd;
-
-        ret = epoll_ctl(epfd, EPOLL_CTL_ADD, from, &ev);
-        if (ret != SHIM_OK) {
-            write_message(g_log_fd, ERR_MSG, "add fd %d to epoll loop failed:%d", from, SHIM_SYS_ERR(errno));
-            pthread_mutex_unlock(&(ioc->mutex));
-            return SHIM_ERR;
-        }
-    }
-
-    /* add dest fd */
-    if (to != -1) {
-        /* new fd_node_t for dest fd */
-        fd_node_t *fn = (fd_node_t *)calloc(1, sizeof(fd_node_t));
-        if (fn == NULL) {
-            pthread_mutex_unlock(&(ioc->mutex));
-            return SHIM_ERR;
-        }
-        fn->fd = to;
-        fn->is_log = false;
-        if (io_thd->terminal != NULL && to == io_thd->terminal->fd) {
-            fn->is_log = true;
-        }
-        fn->next = NULL;
-
-        if (ioc->fd_to == NULL) {
-            ioc->fd_to = fn;
-        } else {
-            fd_node_t *tmp = ioc->fd_to;
-            while (tmp->next != NULL) {
-                tmp = tmp->next;
-            }
-            tmp->next = fn;
-        }
-    }
-    pthread_mutex_unlock(&(ioc->mutex));
-
-    return SHIM_OK;
-}
-
-static void remove_io_dispatch(io_thread_t *io_thd, int from, int to)
-{
-    if (io_thd == NULL || io_thd->ioc == NULL) {
-        return;
-    }
-    io_copy_t *ioc = io_thd->ioc;
-
-    fd_node_t *tmp = NULL;
-    do {
-        /* remove src fd */
-        if (from != -1 && from == ioc->fd_from) {
-            struct epoll_event ev;
-            ev.events = EPOLLIN;
-            ev.data.fd = ioc->fd_from;
-            (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev);
-        }
-
-        /* remove dest fd */
-        if (ioc->fd_to == NULL) {
-            break;
-        }
-        if (ioc->fd_to->fd == to) {
-            /* remove the first fd node */
-            tmp = ioc->fd_to;
-            ioc->fd_to = ioc->fd_to->next;
-            break;
-        }
-        fd_node_t *pre = ioc->fd_to;
-        tmp = ioc->fd_to->next;
-        while (tmp != NULL && tmp->fd != to) {
-            pre = tmp;
-            tmp = tmp->next;
-        }
-        if (tmp != NULL) {
-            pre->next = tmp->next;
-        }
-    } while (0);
-    if (tmp != NULL) {
-        free(tmp);
-        tmp = NULL;
-    }
-}
-
 static int get_exec_winsize(const char *buf, struct winsize *wsize)
 {
     char **array = NULL;
@@ -261,347 +162,287 @@ out:
     return ret;
 }
 
-static void *do_io_copy(void *data)
+static int sync_exit_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
 {
-    io_thread_t *io_thd = (io_thread_t *)data;
-    if (io_thd == NULL || io_thd->ioc == NULL) {
-        return NULL;
-    }
-    io_copy_t *ioc = io_thd->ioc;
-    char *buf = calloc(1, DEFAULT_IO_COPY_BUF + 1);
-    if (buf == NULL) {
-        return NULL;
-    }
+    return EPOLL_LOOP_HANDLE_CLOSE;
+}
 
-    for (;;) {
-        memset(buf, 0, DEFAULT_IO_COPY_BUF);
-        (void)sem_wait(&(io_thd->sem_thd));
-        if (io_thd->is_stdin && io_thd->shutdown) {
-            break;
-        }
+static int stdin_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
+{
+    process_t *p = (process_t *)cbdata;
+    int r_count = 0;
+    int w_count = 0;
+    int *fd_to = NULL;
 
-        int r_count = read(ioc->fd_from, buf, DEFAULT_IO_COPY_BUF);
-        if (r_count == -1) {
-            if (errno == EAGAIN || errno == EINTR) {
-                continue;
-            }
-            break;
-        } else if (r_count == 0) {
-            /* End of file. The remote has closed the connection */
-            break;
-        } else if (ioc->id != EXEC_RESIZE) {
-            if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
-                continue;
-            }
+    if (events & EPOLLHUP) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
+    }
 
-            fd_node_t *fn = ioc->fd_to;
-            fd_node_t *next = NULL;
-            for (; fn != NULL; fn = next) {
-                next = fn->next;
-                if (fn->is_log) {
-                    shim_write_container_log_file(io_thd->terminal, ioc->id, buf, r_count);
-                } else {
-                    int w_count = write_nointr_in_total(fn->fd, buf, r_count);
-                    if (w_count < 0) {
-                        /* When any error occurs, remove the write fd */
-                        remove_io_dispatch(io_thd, -1, fn->fd);
-                    }
-                }
-            }
-            pthread_mutex_unlock(&(ioc->mutex));
-        } else {
-            if (pthread_mutex_lock(&(ioc->mutex)) != 0) {
-                continue;
-            }
+    if (!(events & EPOLLIN)) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
+    }
 
-            int resize_fd = ioc->fd_to->fd;
-            struct winsize wsize = { 0x00 };
-            if (get_exec_winsize(buf, &wsize) < 0) {
-                break;
-            }
-            if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) {
-                break;
-            }
-            pthread_mutex_unlock(&(ioc->mutex));
-        }
+    (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
 
-        /*
-         In the case of stdout and stderr, maybe numbers of read bytes are not the last msg in pipe.
-         So, when the value of r_count is larger than zero, we need to try reading again to avoid loss msgs.
-        */
-        if (io_thd->shutdown && r_count <= 0) {
-            break;
-        }
+    r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+    if (r_count <= 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
-    struct epoll_event ev;
-    ev.events = EPOLLIN;
-    ev.data.fd = ioc->fd_from;
-    (void)epoll_ctl(io_thd->epfd, EPOLL_CTL_DEL, ioc->fd_from, &ev);
 
-    free(buf);
+    if (p->state->terminal) {
+        fd_to = &(p->recv_fd);
+    } else {
+        fd_to = &(p->shim_io->in);
+    }
+    w_count = write_nointr_in_total(*fd_to, p->buf, r_count);
+    if (w_count < 0) {
+        /* When any error occurs, set the write fd -1  */
+        write_message(g_log_fd, WARN_MSG, "write in_fd %d error:%d", *fd_to, SHIM_SYS_ERR(errno));
+        close(*fd_to);
+        *fd_to = -1;
+    }
 
-    return NULL;
+    return EPOLL_LOOP_HANDLE_CONTINUE;
 }
 
-static void sem_post_inotify_io_copy(int fd, uint32_t event, void *data)
+static int stdout_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
 {
-    io_thread_t *thd = (io_thread_t *)data;
-    if (thd->ioc == NULL || fd != thd->ioc->fd_from) {
-        return;
+    process_t *p = (process_t *)cbdata;
+    int r_count = 0;
+    int w_count = 0;
+
+    if (events & EPOLLHUP) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    if (event & EPOLLIN) {
-        (void)sem_post(&thd->sem_thd);
-    } else if (event & EPOLLHUP) {
-        thd->shutdown = true;
-        (void)sem_post(&thd->sem_thd);
+    if (!(events & EPOLLIN)) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
     }
-}
 
-static int create_io_copy_thread(process_t *p, int std_id)
-{
-    int ret = SHIM_ERR;
-    io_thread_t *io_thd = NULL;
-    io_copy_t *ioc = NULL;
+    (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
 
-    ioc = (io_copy_t *)calloc(1, sizeof(io_copy_t));
-    if (ioc == NULL) {
-        goto failure;
+    if (p->block_read) {
+        r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+    } else {
+        r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF);
     }
-    ioc->id = std_id;
-    ioc->fd_from = -1;
-    ioc->fd_to = NULL;
-    if (pthread_mutex_init(&(ioc->mutex), NULL) != 0) {
-        goto failure;
+    if (r_count <= 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    io_thd = (io_thread_t *)calloc(1, sizeof(io_thread_t));
-    if (io_thd == NULL) {
-        goto failure;
+    shim_write_container_log_file(p->terminal, STDID_OUT, p->buf, r_count);
+
+    if (p->isulad_io->out == -1) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
     }
-    if (sem_init(&io_thd->sem_thd, 0, 0) != 0) {
-        write_message(g_log_fd, ERR_MSG, "sem init failed:%d", SHIM_SYS_ERR(errno));
-        goto failure;
+
+    w_count = write_nointr_in_total(p->isulad_io->out, p->buf, r_count);
+    if (w_count < 0) {
+        /* When any error occurs, set the write fd -1  */
+        write_message(g_log_fd, WARN_MSG, "write out_fd %d error:%d", p->isulad_io->out, SHIM_SYS_ERR(errno));
+        close(p->isulad_io->out);
+        p->isulad_io->out = -1;
     }
-    io_thd->epfd = p->io_loop_fd;
-    io_thd->ioc = ioc;
-    io_thd->shutdown = false;
-    io_thd->is_stdin = std_id == STDID_IN ? true : false;
-    io_thd->terminal = std_id != STDID_IN ? p->terminal : NULL;
 
-    p->io_threads[std_id] = io_thd;
+    return EPOLL_LOOP_HANDLE_CONTINUE;
+}
 
-    ret = pthread_create(&(io_thd->tid), NULL, do_io_copy, io_thd);
-    if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "thread io copy create failed:%d", SHIM_SYS_ERR(errno));
-        goto failure;
+static int stderr_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
+{
+    process_t *p = (process_t *)cbdata;
+    int r_count = 0;
+    int w_count = 0;
+
+    if (events & EPOLLHUP) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    ret = SHIM_OK;
+    if (!(events & EPOLLIN)) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
+    }
 
-    return ret;
+    (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
 
-failure:
-    if (ioc != NULL) {
-        pthread_mutex_destroy(&(ioc->mutex));
-        free(ioc);
+    if (p->block_read) {
+        r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+    } else {
+        r_count = read(fd, p->buf, DEFAULT_IO_COPY_BUF);
     }
-    if (io_thd != NULL) {
-        free(io_thd);
+    if (r_count <= 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    return SHIM_ERR;
-}
-
-static int start_io_copy_threads(process_t *p)
-{
-    int ret = SHIM_ERR;
-    int i;
-
-    /* 4 threads for stdin, stdout, stderr and exec resize */
-    for (i = 0; i < 4; i++) {
-        /*
-        * if the terminal is used, we do not need to active the io copy of stderr pipe,
-        * for stderr and stdout are mixed together
-        */
-        if (i == STDID_ERR && p->state->terminal) {
-            continue;
-        }
+    shim_write_container_log_file(p->terminal, STDID_ERR, p->buf, r_count);
 
-        ret = create_io_copy_thread(p, i);
-        if (ret != SHIM_OK) {
-            return SHIM_ERR;
-        }
+    if (p->isulad_io->err == -1) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
     }
-    return SHIM_OK;
-}
 
-static void destroy_io_thread(process_t *p, int std_id)
-{
-    io_thread_t *io_thd = p->io_threads[std_id];
-    if (io_thd == NULL) {
-        return;
+    w_count = write_nointr_in_total(p->isulad_io->err, p->buf, r_count);
+    if (w_count < 0) {
+        /* When any error occurs, set the write fd -1  */
+        write_message(g_log_fd, WARN_MSG, "write err_fd %d error:%d", p->isulad_io->err, SHIM_SYS_ERR(errno));
+        close(p->isulad_io->err);
+        p->isulad_io->err = -1;
     }
 
-    io_thd->shutdown = true;
-    (void)sem_post(&io_thd->sem_thd);
-    pthread_join(io_thd->tid, NULL);
-    if (io_thd->ioc != NULL) {
-        free(io_thd->ioc);
-    }
-    free(io_thd);
-    p->io_threads[std_id] = NULL;
+    return EPOLL_LOOP_HANDLE_CONTINUE;
 }
 
-/*
-    std_id: channel type
-    isulad_stdio: one side of the isulad fifo file
-    fd: one side of the shim io pipe
-    ---------------------------------------------------------------
-    | CHANNEL |    iSulad Fifo Side     | Flow Direction |   fd   |
-    ---------------------------------------------------------------
-    |  STDIN  |        READ             |      -->       |  WRITE |
-    ---------------------------------------------------------------
-    |  STDOUT |        WRITE            |      <--       |  READ  |
-    ---------------------------------------------------------------
-    |  STDERR |        WRITE            |      <--       |  READ  |
-    ---------------------------------------------------------------
-    |  RESIZE |        READ             |      -->       |  WRITE |
-    ---------------------------------------------------------------
-*/
-static int connect_to_isulad(process_t *p, int std_id, const char *isulad_stdio, int fd)
+static int resize_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
 {
-    mode_t mode;
-    int fd_isulad = -1;
-    int *fd_from = NULL;
-    int *fd_to = NULL;
+    process_t *p = (process_t *)cbdata;
+    int r_count = 0;
+    int resize_fd = -1;
 
-    if (std_id == STDID_IN || std_id == EXEC_RESIZE) {
-        mode = O_RDONLY;
-        fd_from = &fd_isulad;
-        fd_to = &fd;
-    } else {
-        mode = O_WRONLY;
-        fd_from = &fd;
-        fd_to = &fd_isulad;
+    if (events & EPOLLHUP) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    if (isulad_stdio != NULL && file_exists(isulad_stdio)) {
-        fd_isulad = open_fifo_noblock(isulad_stdio, mode);
-        if (fd_isulad < 0) {
-            return SHIM_ERR;
-        }
-        /* open dummy fd to avoid resize epoll hub */
-        if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) {
-            return SHIM_ERR;
-        }
+    if (!(events & EPOLLIN)) {
+        return EPOLL_LOOP_HANDLE_CONTINUE;
     }
 
-    if (*fd_from != -1) {
-        if (std_id != STDID_IN && std_id != EXEC_RESIZE && p->io_threads[std_id]->terminal != NULL) {
-            (void)add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, p->terminal->fd);
-        }
-        return add_io_dispatch(p->io_loop_fd, p->io_threads[std_id], *fd_from, *fd_to);
+    (void)memset(p->buf, 0, DEFAULT_IO_COPY_BUF);
+    r_count = read_nointr(fd, p->buf, DEFAULT_IO_COPY_BUF);
+    if (r_count <= 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
     }
 
-    /* if no I/O source is available, the I/O thread nead to be destroyed */
-    destroy_io_thread(p, std_id);
+    resize_fd = p->recv_fd;
+    struct winsize wsize = { 0x00 };
+    if (get_exec_winsize(p->buf, &wsize) < 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
+    }
+    if (ioctl(resize_fd, TIOCSWINSZ, &wsize) < 0) {
+        return EPOLL_LOOP_HANDLE_CLOSE;
+    }
 
-    return SHIM_OK;
+    return EPOLL_LOOP_HANDLE_CONTINUE;
 }
 
-static void *task_console_accept(void *data)
+static int task_console_accept(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr)
 {
+    process_t *p = (process_t *)cbdata;
     int conn_fd = -1;
-    int recv_fd = -1;
     int ret = SHIM_ERR;
-    console_accept_t *ac = (console_accept_t *)data;
 
-    conn_fd = accept(ac->listen_fd, NULL, NULL);
+    conn_fd = accept(p->listen_fd, NULL, NULL);
     if (conn_fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", ac->listen_fd, SHIM_SYS_ERR(errno));
+        write_message(g_log_fd, ERR_MSG, "accept from fd %d failed:%d", p->listen_fd, SHIM_SYS_ERR(errno));
         goto out;
     }
 
-    recv_fd = receive_fd(conn_fd);
-    if (check_fd(recv_fd) != true) {
+    p->recv_fd = receive_fd(conn_fd);
+    if (check_fd(p->recv_fd) != true) {
         write_message(g_log_fd, ERR_MSG, "check console fd failed");
         goto out;
     }
 
     /* do console io copy */
 
-    /* p.state.stdin---->runtime.console */
-    ret = connect_to_isulad(ac->p, STDID_IN, ac->p->state->isulad_stdin, recv_fd);
+    // p->isulad_io->in ----> p->recv_fd
+    ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
     if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
         goto out;
     }
-
-    /* p.state.stdout<------runtime.console */
-    ret = connect_to_isulad(ac->p, STDID_OUT, ac->p->state->isulad_stdout, recv_fd);
+    // p->recv_fd ----> p->isulad_io->out
+    ret = epoll_loop_add_handler(descr, p->recv_fd, stdout_cb, p);
     if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add recv_fd fd %d to epoll loop failed:%d", p->recv_fd, SHIM_SYS_ERR(errno));
         goto out;
     }
-
-    /* p.state.resize_fifo------>runtime.console */
-    ret = connect_to_isulad(ac->p, EXEC_RESIZE, ac->p->state->resize_fifo, recv_fd);
+    // p->isulad_io->resize ----> p->recv_fd
+    ret = epoll_loop_add_handler(descr, p->isulad_io->resize, resize_cb, p);
     if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add resize fd %d to epoll loop failed:%d", p->isulad_io->resize, SHIM_SYS_ERR(errno));
         goto out;
     }
 
 out:
     /* release listen socket at the first time */
-    close_fd(&ac->listen_fd);
-    if (ac->p->console_sock_path != NULL) {
-        (void)unlink(ac->p->console_sock_path);
-        free(ac->p->console_sock_path);
-        ac->p->console_sock_path = NULL;
-    }
-    free(ac);
-    if (ret != SHIM_OK) {
-        /*
-         * When an error occurs during the receiving of the fd , the process
-         * exits directly. The files created in the working directory will be
-         * deleted by its parent process isulad
-         */
-        exit(EXIT_FAILURE);
+    close_fd(&p->listen_fd);
+    if (p->console_sock_path != NULL) {
+        (void)unlink(p->console_sock_path);
+        free(p->console_sock_path);
+        p->console_sock_path = NULL;
     }
-    return NULL;
+    return ret;
 }
 
-static void *io_epoll_loop(void *data)
+static int stdio_chown(int (*stdio_fd)[2], int uid, int gid)
 {
-    process_t *p = (process_t *)data;
-    int wait_fds = 0;
-    struct epoll_event evs[MAX_EVENTS];
-    int i;
+    int i, j;
 
-    if ((pthread_detach(pthread_self())) != 0) {
-        write_message(g_log_fd, ERR_MSG, "detach thread failed");
-        return NULL;
+    for (i = 0; i < 3; i++) {
+        for (j = 0; j < 2; j++) {
+            int ret = fchown(stdio_fd[i][j], uid, gid);
+            if (ret != SHIM_OK) {
+                return SHIM_ERR;
+            }
+        }
     }
+    return SHIM_OK;
+}
 
-    p->io_loop_fd = epoll_create1(EPOLL_CLOEXEC);
-    if (p->io_loop_fd < 0) {
-        write_message(g_log_fd, ERR_MSG, "epoll create failed:%d", SHIM_SYS_ERR(errno));
-        exit(EXIT_FAILURE);
-    }
-    (void)sem_post(&p->sem_mainloop);
+static void stdio_release(int (*stdio_fd)[2])
+{
+    int i, j;
 
-    for (;;) {
-        wait_fds = epoll_wait(p->io_loop_fd, evs, MAX_EVENTS, -1);
-        if (wait_fds < 0) {
-            if (errno == EINTR) {
-                continue;
+    for (i = 0; i < 3; i++) {
+        for (j = 0; j < 2; j++) {
+            if (stdio_fd[i][j] > 0) {
+                close(stdio_fd[i][j]);
             }
-            _exit(EXIT_FAILURE);
         }
+    }
+}
 
-        for (i = 0; i < wait_fds; i++) {
-            io_thread_t *thd_io = (io_thread_t *)evs[i].data.ptr;
-            sem_post_inotify_io_copy(thd_io->ioc->fd_from, evs[i].events, thd_io);
-        }
+static stdio_t *initialize_io(process_t *p)
+{
+    int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } };
+
+    stdio_t *stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+    p->stdio = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+    if (p->stdio == NULL || stdio == NULL) {
+        goto failure;
+    }
+
+    /* don't open resize pipe */
+    if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) ||
+        (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) {
+        write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
+        goto failure;
     }
+
+    p->stdio->in = stdio_fd[0][0]; // r
+    stdio->in = stdio_fd[0][1]; // w
+    p->stdio->out = stdio_fd[1][1]; // w
+    stdio->out = stdio_fd[1][0]; // r
+    p->stdio->err = stdio_fd[2][1]; // w
+    stdio->err = stdio_fd[2][0]; // r
+    p->stdio->resize = stdio_fd[3][0]; // r
+    stdio->resize = stdio_fd[3][1]; // w
+
+    if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) {
+        goto failure;
+    }
+
+    return stdio;
+
+failure:
+    if (stdio != NULL) {
+        free(stdio);
+        stdio = NULL;
+    }
+    if (p->stdio != NULL) {
+        free(p->stdio);
+        p->stdio = NULL;
+    }
+    stdio_release(stdio_fd);
+
+    return NULL;
 }
 
 static int new_temp_console_path(process_t *p)
@@ -614,7 +455,7 @@ static int new_temp_console_path(process_t *p)
     if (ret != SHIM_OK) {
         return SHIM_ERR;
     }
-    p->console_sock_path = (char *)calloc(1, MAX_CONSOLE_SOCK_LEN + 1);
+    p->console_sock_path = (char *)util_common_calloc_s(MAX_CONSOLE_SOCK_LEN + 1);
     if (p->console_sock_path == NULL) {
         return SHIM_ERR;
     }
@@ -628,12 +469,11 @@ static int new_temp_console_path(process_t *p)
     return SHIM_OK;
 }
 
-static int console_init(process_t *p, pthread_t *tid_accept)
+static int console_init(process_t *p, struct epoll_descr *descr)
 {
     int ret = SHIM_ERR;
     int fd = -1;
     struct sockaddr_un addr;
-    console_accept_t *ac = NULL;
 
     fd = socket(AF_UNIX, SOCK_STREAM, 0);
     if (fd < 0) {
@@ -657,145 +497,182 @@ static int console_init(process_t *p, pthread_t *tid_accept)
         goto failure;
     }
 
-    ac = (console_accept_t *)calloc(1, sizeof(console_accept_t));
-    if (ac == NULL) {
-        goto failure;
-    }
-    ac->p = p;
-    ac->listen_fd = fd;
+    p->listen_fd = fd;
 
-    ret = pthread_create(tid_accept, NULL, task_console_accept, ac);
+    ret = epoll_loop_add_handler(descr, p->listen_fd, task_console_accept, p);
     if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add listen_fd fd %d to epoll loop failed:%d",  p->listen_fd, SHIM_SYS_ERR(errno));
         goto failure;
     }
 
     return SHIM_OK;
 failure:
     close_fd(&fd);
-    if (ac != NULL) {
-        free(ac);
-        ac = NULL;
-    }
     (void)unlink(p->console_sock_path);
 
     return SHIM_ERR;
 }
 
-static int stdio_chown(int (*stdio_fd)[2], int uid, int gid)
+static int open_terminal_io(process_t *p, struct epoll_descr *descr)
 {
-    int i, j;
+    int ret = SHIM_ERR;
 
-    for (i = 0; i < 3; i++) {
-        for (j = 0; j < 2; j++) {
-            int ret = fchown(stdio_fd[i][j], uid, gid);
-            if (ret != SHIM_OK) {
-                return SHIM_ERR;
-            }
-        }
+    ret = new_temp_console_path(p);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "get temp console sock path failed");
+        return SHIM_ERR;
     }
-    return SHIM_OK;
-}
-
-static void stdio_release(int (*stdio_fd)[2])
-{
-    int i, j;
 
-    for (i = 0; i < 3; i++) {
-        for (j = 0; j < 2; j++) {
-            if (stdio_fd[i][j] > 0) {
-                close(stdio_fd[i][j]);
-            }
-        }
-    }
+    /* begin listen from p->console_sock_path */
+    return console_init(p, descr);
 }
 
-static stdio_t *initialize_io(process_t *p)
+static int open_generic_io(process_t *p, struct epoll_descr *descr)
 {
-    int stdio_fd[4][2] = { { -1, -1 }, { -1, -1 }, { -1, -1 }, { -1, -1 } };
-
-    stdio_t *stdio = (stdio_t *)calloc(1, sizeof(stdio_t));
-    p->stdio = (stdio_t *)calloc(1, sizeof(stdio_t));
-    if (p->stdio == NULL || stdio == NULL) {
-        goto failure;
-    }
+    int ret = SHIM_ERR;
 
-    /* don't open resize pipe */
-    if ((pipe2(stdio_fd[0], O_CLOEXEC | O_NONBLOCK) != 0) || (pipe2(stdio_fd[1], O_CLOEXEC | O_NONBLOCK) != 0) ||
-        (pipe2(stdio_fd[2], O_CLOEXEC | O_NONBLOCK) != 0)) {
-        write_message(g_log_fd, ERR_MSG, "open pipe failed when init io:%d", SHIM_SYS_ERR(errno));
-        goto failure;
+    // io: in: w  out/err: r
+    stdio_t *io = initialize_io(p);
+    if (io == NULL) {
+        return SHIM_ERR;
     }
+    p->shim_io = io;
 
-    p->stdio->in = stdio_fd[0][0]; // r
-    stdio->in = stdio_fd[0][1]; // w
-    p->stdio->out = stdio_fd[1][1]; // w
-    stdio->out = stdio_fd[1][0]; // r
-    p->stdio->err = stdio_fd[2][1]; // w
-    stdio->err = stdio_fd[2][0]; // r
-    p->stdio->resize = stdio_fd[3][0]; // r
-    stdio->resize = stdio_fd[3][1]; // w
-
-    if (stdio_chown(stdio_fd, p->state->root_uid, p->state->root_gid) != SHIM_OK) {
-        goto failure;
+    // p->isulad_io->in ----> p->shim_io->in
+    ret = epoll_loop_add_handler(descr, p->isulad_io->in, stdin_cb, p);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add in fd %d to epoll loop failed:%d", p->isulad_io->in, SHIM_SYS_ERR(errno));
+        return SHIM_ERR;
     }
-
-    return stdio;
-
-failure:
-    if (stdio != NULL) {
-        free(stdio);
-        stdio = NULL;
+    // p->shim_io->out ----> p->isulad_io->out
+    ret = epoll_loop_add_handler(descr, p->shim_io->out, stdout_cb, p);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add  out fd %d to epoll loop failed:%d", p->shim_io->out, SHIM_SYS_ERR(errno));
+        return SHIM_ERR;
     }
-    if (p->stdio != NULL) {
-        free(p->stdio);
-        p->stdio = NULL;
+    // p->shim_io->err ----> p->isulad_io->err
+    ret = epoll_loop_add_handler(descr, p->shim_io->err, stderr_cb, p);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "add err fd %d to epoll loop failed:%d", p->shim_io->err, SHIM_SYS_ERR(errno));
+        return SHIM_ERR;
     }
-    stdio_release(stdio_fd);
 
-    return NULL;
+    return SHIM_OK;
 }
 
-static int open_terminal_io(process_t *p, pthread_t *tid_accept)
+static int set_non_block(int fd)
 {
+    int flag = -1;
     int ret = SHIM_ERR;
 
-    ret = new_temp_console_path(p);
-    if (ret != SHIM_OK) {
-        write_message(g_log_fd, ERR_MSG, "get temp console sock path failed");
+    flag = fcntl(fd, F_GETFL, 0);
+    if (flag < 0) {
+        return SHIM_ERR;
+    }
+
+    ret = fcntl(fd, F_SETFL, flag | O_NONBLOCK);
+    if (ret != 0) {
         return SHIM_ERR;
     }
 
-    /* begin listen and accept fd from p->console_sock_path */
-    return console_init(p, tid_accept);
+    return SHIM_OK;
 }
 
-static int open_generic_io(process_t *p)
+/*
+    std_id: channel type
+    isulad_stdio: one side of the isulad fifo file
+    fd: one side of the shim io pipe
+    ---------------------------------------------------------------
+    | CHANNEL |    iSulad Fifo Side     | Flow Direction |   fd   |
+    ---------------------------------------------------------------
+    |  STDIN  |        READ             |      -->       |  WRITE |
+    ---------------------------------------------------------------
+    |  STDOUT |        WRITE            |      <--       |  READ  |
+    ---------------------------------------------------------------
+    |  STDERR |        WRITE            |      <--       |  READ  |
+    ---------------------------------------------------------------
+    |  RESIZE |        READ             |      -->       |  WRITE |
+    ---------------------------------------------------------------
+*/
+static void *io_epoll_loop(void *data)
 {
-    int ret = SHIM_ERR;
+    int ret = 0;
+    int fd_out = -1;
+    int fd_err = -1;
+    process_t *p = (process_t *)data;
+    struct epoll_descr descr;
 
-    // io: in: w  out/err: r
-    stdio_t *io = initialize_io(p);
-    if (io == NULL) {
-        return SHIM_ERR;
+    ret = epoll_loop_open(&descr);
+    if (ret != 0) {
+        write_message(g_log_fd, ERR_MSG, "epoll loop open failed:%d", SHIM_SYS_ERR(errno));
+        exit(EXIT_FAILURE);
     }
-    p->shim_io = io;
-    /* stdin */
-    ret = connect_to_isulad(p, STDID_IN, p->state->isulad_stdin, io->in);
-    if (ret != SHIM_OK) {
-        return SHIM_ERR;
+
+    // sync fd: epoll loop will exit when recive sync fd event.
+    ret = epoll_loop_add_handler(&descr, p->sync_fd, sync_exit_cb, p);
+    if (ret != 0) {
+        write_message(g_log_fd, ERR_MSG, "add sync_fd %d to epoll loop failed:%d", p->sync_fd, SHIM_SYS_ERR(errno));
+        exit(EXIT_FAILURE);
     }
-    /* stdout */
-    ret = connect_to_isulad(p, STDID_OUT, p->state->isulad_stdout, io->out);
-    if (ret != SHIM_OK) {
-        return SHIM_ERR;
+
+    if (p->state->terminal) {
+        ret = open_terminal_io(p, &descr);
+    } else {
+        ret = open_generic_io(p, &descr);
     }
-    /* stderr */
-    ret = connect_to_isulad(p, STDID_ERR, p->state->isulad_stderr, io->err);
     if (ret != SHIM_OK) {
-        return SHIM_ERR;
+        write_message(g_log_fd, ERR_MSG, "open io failed:%d", SHIM_SYS_ERR(errno));
+        exit(EXIT_FAILURE);
     }
 
-    return SHIM_OK;
+    (void)sem_post(&p->sem_mainloop);
+
+    ret = epoll_loop(&descr, -1);
+    if (ret != 0) {
+        write_message(g_log_fd, ERR_MSG, "epoll loop failed");
+        exit(EXIT_FAILURE);
+    }
+
+    // in order to avoid data loss, set fd non-block and read it
+    p->block_read = false;
+    if (p->state->terminal) {
+        fd_out = p->recv_fd;
+    } else {
+        fd_out = p->shim_io->out;
+        fd_err = p->shim_io->err;
+    }
+
+    if (fd_out > 0) {
+        ret = set_non_block(fd_out);
+        if (ret != SHIM_OK) {
+            write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_out, SHIM_SYS_ERR(errno));
+            exit(EXIT_FAILURE);
+        }
+
+        for (;;) {
+            ret = stdout_cb(fd_out, EPOLLIN, p, &descr);
+            if (ret == EPOLL_LOOP_HANDLE_CLOSE) {
+                break;
+            }
+        }
+    }
+
+    if (fd_err > 0) {
+        ret = set_non_block(fd_err);
+        if (ret != SHIM_OK) {
+            write_message(g_log_fd, ERR_MSG, "set fd %d non_block failed:%d", fd_err, SHIM_SYS_ERR(errno));
+            exit(EXIT_FAILURE);
+        }
+
+        for (;;) {
+            ret = stderr_cb(fd_err, EPOLLIN, p, &descr);
+            if (ret == EPOLL_LOOP_HANDLE_CLOSE) {
+                break;
+            }
+        }
+    }
+
+    return NULL;
 }
 
 static void adapt_for_isulad_stdin(process_t *p)
@@ -813,7 +690,7 @@ static int terminal_init(log_terminal **terminal, shim_client_process_state *p_s
 {
     log_terminal *log_term = NULL;
 
-    log_term = calloc(1, sizeof(log_terminal));
+    log_term = util_common_calloc_s(sizeof(log_terminal));
     if (log_term == NULL) {
         write_message(g_log_fd, ERR_MSG, "Failed to calloc log_terminal");
         goto clean_out;
@@ -858,11 +735,90 @@ clean_out:
     return SHIM_ERR;
 }
 
+static int open_isulad_fd(int std_id, const char *isulad_stdio, int *fd)
+{
+    mode_t mode = O_WRONLY;
+
+    if (std_id == STDID_IN || std_id == EXEC_RESIZE) {
+        mode = O_RDONLY;
+    }
+
+    if (isulad_stdio != NULL && file_exists(isulad_stdio)) {
+        *(fd) = open_fifo_noblock(isulad_stdio, mode);
+        if (*(fd) < 0) {
+            return -1;
+        }
+        /* open dummy fd to avoid resize epoll hub */
+        if (std_id == EXEC_RESIZE && open_fifo_noblock(isulad_stdio, O_WRONLY) < 0) {
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+static int init_isulad_stdio(process_t *p)
+{
+    int ret = SHIM_OK;
+    p->isulad_io = (stdio_t *)util_common_calloc_s(sizeof(stdio_t));
+    if (p->isulad_io == NULL) {
+        return SHIM_ERR;
+    }
+
+    p->isulad_io->in = -1;
+    p->isulad_io->out = -1;
+    p->isulad_io->err = -1;
+    p->isulad_io->resize = -1;
+
+    ret = open_isulad_fd(STDID_IN, p->state->isulad_stdin, &p->isulad_io->in);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "Failed to open in isulad fd: %s",  p->state->isulad_stdin);
+        goto failure;
+    }
+
+    ret = open_isulad_fd(STDID_OUT, p->state->isulad_stdout, &p->isulad_io->out);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "Failed to open out isulad fd: %s",  p->state->isulad_stdout);
+        goto failure;
+    }
+
+    ret = open_isulad_fd(STDID_ERR, p->state->isulad_stderr, &p->isulad_io->err);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "Failed to open err isulad fd: %s",  p->state->isulad_stderr);
+        goto failure;
+    }
+
+    ret = open_isulad_fd(EXEC_RESIZE, p->state->resize_fifo, &p->isulad_io->resize);
+    if (ret != SHIM_OK) {
+        write_message(g_log_fd, ERR_MSG, "Failed to open resize isulad fd: %s",  p->state->resize_fifo);
+        goto failure;
+    }
+    return SHIM_OK;
+failure:
+    if (p->isulad_io != NULL) {
+        free(p->isulad_io);
+        p->isulad_io = NULL;
+    }
+    if (p->isulad_io->in > 0) {
+        close(p->isulad_io->in);
+    }
+    if (p->isulad_io->out > 0) {
+        close(p->isulad_io->out);
+    }
+    if (p->isulad_io->err > 0) {
+        close(p->isulad_io->err);
+    }
+    if (p->isulad_io->resize > 0) {
+        close(p->isulad_io->resize);
+    }
+    return SHIM_ERR;
+}
+
 process_t *new_process(char *id, char *bundle, char *runtime)
 {
     shim_client_process_state *p_state;
     process_t *p = NULL;
-    int i;
     int ret;
 
     p_state = load_process();
@@ -870,7 +826,7 @@ process_t *new_process(char *id, char *bundle, char *runtime)
         return NULL;
     }
 
-    p = (process_t *)calloc(1, sizeof(process_t));
+    p = (process_t *)util_common_calloc_s(sizeof(process_t));
     if (p == NULL) {
         return NULL;
     }
@@ -889,16 +845,31 @@ process_t *new_process(char *id, char *bundle, char *runtime)
     p->bundle = bundle;
     p->runtime = runtime;
     p->state = p_state;
-
+    p->block_read = true;
     p->console_sock_path = NULL;
     p->exit_fd = -1;
     p->io_loop_fd = -1;
     p->ctr_pid = -1;
+    p->listen_fd = -1;
+    p->recv_fd = -1;
     p->stdio = NULL;
     p->shim_io = NULL;
+    p->isulad_io = NULL;
 
-    for (i = 0; i < 3; i++) {
-        p->io_threads[i] = NULL;
+    p->sync_fd = eventfd(0, EFD_CLOEXEC);
+    if (p->sync_fd < 0) {
+        write_message(g_log_fd, ERR_MSG, "Failed to create eventfd: %s", strerror(errno));
+        goto failure;
+    }
+
+    ret = init_isulad_stdio(p);
+    if (ret != SHIM_OK) {
+        goto failure;
+    }
+
+    p->buf = util_common_calloc_s(DEFAULT_IO_COPY_BUF + 1);
+    if (p->buf == NULL) {
+        goto failure;
     }
 
     return p;
@@ -909,28 +880,11 @@ failure:
     return NULL;
 }
 
-int open_io(process_t *p, pthread_t *tid_accept)
+int process_io_start(process_t *p, pthread_t *tid_epoll)
 {
     int ret = SHIM_ERR;
 
-    ret = start_io_copy_threads(p);
-    if (ret != SHIM_OK) {
-        return SHIM_ERR;
-    }
-
-    if (p->state->terminal) {
-        return open_terminal_io(p, tid_accept);
-    }
-
-    return open_generic_io(p);
-}
-
-int process_io_init(process_t *p)
-{
-    int ret = SHIM_ERR;
-
-    pthread_t tid_loop;
-    ret = pthread_create(&tid_loop, NULL, io_epoll_loop, p);
+    ret = pthread_create(tid_epoll, NULL, io_epoll_loop, p);
     if (ret != SHIM_OK) {
         return SHIM_SYS_ERR(errno);
     }
@@ -1091,8 +1045,8 @@ static void exec_runtime_process(process_t *p, int exec_fd)
     }
 
     char *cwd = getcwd(NULL, 0);
-    char *log_path = (char *)calloc(1, PATH_MAX);
-    char *pid_path = (char *)calloc(1, PATH_MAX);
+    char *log_path = (char *)util_common_calloc_s(PATH_MAX);
+    char *pid_path = (char *)util_common_calloc_s(PATH_MAX);
     if (cwd == NULL || log_path == NULL || pid_path == NULL) {
         (void)dprintf(exec_fd, "memory error: %s", strerror(errno));
         _exit(EXIT_FAILURE);
@@ -1294,13 +1248,11 @@ static int wait_container_process_with_timeout(process_t *p, const unsigned int
 
 }
 
-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout)
+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout)
 {
-    int i;
     int nret = 0;
     int ret = 0;
     int status = 0;
-    struct timespec ts;
 
     ret = wait_container_process_with_timeout(p, timeout, &status);
     if (ret == SHIM_ERR_TIMEOUT) {
@@ -1324,27 +1276,20 @@ int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, cons
     if (p->exit_fd > 0) {
         (void)write_nointr(p->exit_fd, &status, sizeof(int));
     }
-    // wait for task_console_accept thread termination. In order to make sure that
-    // the io_copy connection is established and io_thread is not used by multiple threads.
-    if (p->state->terminal) {
-        if (clock_gettime(CLOCK_REALTIME, &ts) == -1) {
-            write_message(g_log_fd, ERR_MSG, "Failed to get realtime");
-            nret = pthread_join(tid_accept, NULL);
-        } else {
-            // Set the maximum waiting time to 60s to prevent stuck.
-            ts.tv_sec += 60;
-            nret = pthread_timedjoin_np(tid_accept, NULL, &ts);
-        }
 
-        if (nret != 0) {
-            write_message(g_log_fd, ERR_MSG, "Failed to join task_console_accept thread");
+    if (p->sync_fd > 0) {
+        if (eventfd_write(p->sync_fd, 1)) {
+            write_message(g_log_fd, ERR_MSG, "Failed to write sync fd");
         }
     }
 
-    for (i = 0; i < 3; i++) {
-        destroy_io_thread(p, i);
+    nret = pthread_join(tid_epoll, NULL);
+    if (nret != 0) {
+        write_message(g_log_fd, ERR_MSG, "Failed to join epoll loop thread");
     }
 
+    close(p->sync_fd);
+
     if (!p->state->exec) {
         // if log did not contain "/n", print remaind container log when exit isulad-shim
         shim_write_container_log_file(p->terminal, STDID_OUT, NULL, 0);
diff --git a/src/cmd/isulad-shim/process.h b/src/cmd/isulad-shim/process.h
index 7e3259e8..4cb6462c 100644
--- a/src/cmd/isulad-shim/process.h
+++ b/src/cmd/isulad-shim/process.h
@@ -40,64 +40,37 @@ typedef struct {
     int resize;
 } stdio_t;
 
-typedef struct fd_node {
-    int fd;
-    bool is_log;
-    struct fd_node *next;
-} fd_node_t;
-
-typedef struct {
-    int fd_from;
-    fd_node_t *fd_to;
-    int id;// 0,1,2,3
-    pthread_mutex_t mutex;
-} io_copy_t;
-
-typedef struct {
-    int epfd;
-    pthread_t tid;
-    pthread_attr_t attr;
-    sem_t sem_thd;
-    io_copy_t *ioc;
-    bool shutdown;
-    bool is_stdin;
-    log_terminal *terminal;// just used by stdout and stderr
-} io_thread_t;
-
 typedef struct process {
     char *id;
     char *bundle;
     char *runtime;
-    char *console_sock_path;// pty socket path
+    char *console_sock_path; // pty socket path
     int io_loop_fd;
     int exit_fd;
     int ctr_pid;
+    int sync_fd;
+    int listen_fd;
+    int recv_fd;
+    bool block_read;
     log_terminal *terminal;
-    stdio_t *stdio;// shim to on runtime side, in:r out/err: w
+    stdio_t *stdio; // shim to on runtime side, in:r out/err: w
     stdio_t *shim_io; // shim io on isulad side, in: w  out/err: r
-    io_thread_t *io_threads[4];// stdin,stdout,stderr,exec_resize
+    stdio_t *isulad_io; // isulad io, in:r out/err: w
     shim_client_process_state *state;
     sem_t sem_mainloop;
+    char *buf;
 } process_t;
 
-typedef struct {
-    int listen_fd;
-    process_t *p;
-} console_accept_t;
-
 typedef struct {
     int pid;
     int status;
 } process_exit_t;
 
-
-
 process_t* new_process(char *id, char *bundle, char *runtime);
 
-int open_io(process_t *p, pthread_t *tid_accept);
-int process_io_init(process_t *p);
+int process_io_start(process_t *p, pthread_t *tid_epoll);
 int create_process(process_t *p);
-int process_signal_handle_routine(process_t *p, const pthread_t tid_accept, const unsigned int timeout);
+int process_signal_handle_routine(process_t *p, const pthread_t tid_epoll, const unsigned int timeout);
 
 #ifdef __cplusplus
 }
diff --git a/test/cmd/isulad-shim/CMakeLists.txt b/test/cmd/isulad-shim/CMakeLists.txt
index e5c1cd6e..122538ff 100644
--- a/test/cmd/isulad-shim/CMakeLists.txt
+++ b/test/cmd/isulad-shim/CMakeLists.txt
@@ -6,6 +6,7 @@ add_executable(${EXE}
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/process.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/common.c
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/terminal.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/isulad-shim/mainloop.c
     isulad-shim_ut.cc)
 
 target_include_directories(${EXE} PUBLIC
diff --git a/test/cmd/isulad-shim/isulad-shim_ut.cc b/test/cmd/isulad-shim/isulad-shim_ut.cc
index 34ecd452..8c116ac8 100644
--- a/test/cmd/isulad-shim/isulad-shim_ut.cc
+++ b/test/cmd/isulad-shim/isulad-shim_ut.cc
@@ -11,6 +11,7 @@
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 
+#include "mainloop.h"
 #include "process.h"
 #include "common.h"
 
-- 
2.25.1

