From 0ab63e4c7a5010b78581525359c08af3a0d219cc Mon Sep 17 00:00:00 2001
From: Lemmy Huang <huangliming5@huawei.com>
Date: Wed, 10 Jul 2024 14:09:04 +0800
Subject: [PATCH] cleancode: refactor lwipsock.h

Signed-off-by: Lemmy Huang <huangliming5@huawei.com>
---
 src/api/api_msg.c                    |   4 -
 src/api/lwipgz_dir.mk                |   3 +-
 src/api/lwipgz_posix_api.c           |  11 +-
 src/api/lwipgz_sock.c                | 156 ++++++++++++++++++
 src/api/sockets.c                    | 238 +++------------------------
 src/core/init.c                      |  10 ++
 src/include/lwip/priv/sockets_priv.h |   2 +-
 src/include/lwip/sockets.h           |  41 +++--
 src/include/lwipgz_posix_api.h       |   2 -
 src/include/lwipgz_sock.h            | 130 ++++++---------
 src/include/lwipopts.h               |   8 -
 11 files changed, 273 insertions(+), 332 deletions(-)
 create mode 100644 src/api/lwipgz_sock.c

diff --git a/src/api/api_msg.c b/src/api/api_msg.c
index 91608ca..4234084 100644
--- a/src/api/api_msg.c
+++ b/src/api/api_msg.c
@@ -661,10 +661,6 @@ accept_function(void *arg, struct tcp_pcb *newpcb, err_t err)
     API_EVENT(conn, NETCONN_EVT_RCVPLUS, 0);
   }
 
-#if GAZELLE_ENABLE
-  LWIP_DEBUGF(API_MSG_DEBUG, ("libos incoming connection established\n"));
-  SET_CONN_TYPE_LIBOS(newconn);
-#endif
   return ERR_OK;
 }
 #endif /* LWIP_TCP */
diff --git a/src/api/lwipgz_dir.mk b/src/api/lwipgz_dir.mk
index 49cc973..e832578 100644
--- a/src/api/lwipgz_dir.mk
+++ b/src/api/lwipgz_dir.mk
@@ -1,3 +1,4 @@
-SRC = api_lib.c api_msg.c err.c netbuf.c netdb.c netifapi.c sockets.c tcpip.c sys_arch.c lwipgz_posix_api.c
+SRC = api_lib.c api_msg.c err.c netbuf.c netdb.c netifapi.c sockets.c tcpip.c \
+	  sys_arch.c lwipgz_posix_api.c lwipgz_sock.c
 
 $(eval $(call register_dir, api, $(SRC)))
diff --git a/src/api/lwipgz_posix_api.c b/src/api/lwipgz_posix_api.c
index b916c75..c045496 100644
--- a/src/api/lwipgz_posix_api.c
+++ b/src/api/lwipgz_posix_api.c
@@ -51,17 +51,11 @@ static int chld_is_epfd(int fd)
     return 0;
 }
 
-static struct lwip_sock *chld_get_socket(int fd)
-{
-    return NULL;
-}
-
 void posix_api_fork(void)
 {
     /* lstack helper api */
     posix_api->ues_posix = 1;
     posix_api->is_epfd = chld_is_epfd;
-    posix_api->get_socket = chld_get_socket;
 }
 
 
@@ -115,12 +109,9 @@ int posix_api_init(void)
     CHECK_DLSYM_RET_RETURN(posix_api->ioctl_fn = dlsym(handle, "ioctl"));
     CHECK_DLSYM_RET_RETURN(posix_api->select_fn = dlsym(handle, "select"));
 
-    /* lstack helper api */
-    posix_api->get_socket = get_socket;
-    posix_api->epoll_close_fn = lstack_epoll_close;
-
     /* support fork */
     posix_api->ues_posix = 1;
+    lwip_sock_init();
     return ERR_OK;
 
 err_out:
diff --git a/src/api/lwipgz_sock.c b/src/api/lwipgz_sock.c
new file mode 100644
index 0000000..4c1ecd0
--- /dev/null
+++ b/src/api/lwipgz_sock.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2001-2004 Swedish Institute of Computer Science.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+ * OF SUCH DAMAGE.
+ *
+ * This file is part of the lwIP TCP/IP stack.
+ *
+ * Author: Huawei Technologies
+ *
+ */
+
+#include <sys/socket.h>
+#include <fcntl.h>
+
+#include "lwipgz_sock.h"
+#include "lwipgz_posix_api.h"
+#include "lwip/tcp.h"
+
+extern struct lwip_sock *sockets;
+
+static int socket_sys_type(enum netconn_type type)
+{
+  int sys_type;
+  switch (NETCONNTYPE_GROUP(type)) {
+    case NETCONN_RAW:
+      sys_type = SOCK_RAW;
+      break;
+    case NETCONN_UDPLITE:
+    case NETCONN_UDP:
+      sys_type = SOCK_DGRAM;
+      break;
+    case NETCONN_TCP:
+      sys_type = SOCK_STREAM;
+      break;
+    default:
+      sys_type = -1;
+      break;
+  }
+  return sys_type;
+}
+
+static int socket_new_sysfd(struct netconn *newconn, int flags)
+{
+  int domain = NETCONNTYPE_ISIPV6(newconn->type) ? AF_INET6 : AF_INET;
+  int protocol = 0;
+  int type = socket_sys_type(newconn->type) | flags;
+
+  return posix_api->socket_fn(domain, type, protocol);
+}
+
+/* reference tag: alloc_socket() */
+int gazelle_alloc_socket(struct netconn *newconn, int accepted, int flags)
+{
+  int fd;
+  struct lwip_sock *sock;
+
+  /* only support SOCK_CLOEXEC and SOCK_NONBLOCK */
+  if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)){
+    set_errno(EINVAL);
+    return -1;
+  }
+  if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
+    flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
+
+  fd = socket_new_sysfd(newconn, flags);
+  if (fd < 0)
+    return -1;
+  sock = get_socket_by_fd(fd);
+  if (sock == NULL)
+    goto out;
+
+  sock->conn       = newconn;
+  sock->lastdata.pbuf = NULL;
+#if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
+  LWIP_ASSERT("sock->select_waiting == 0", sock->select_waiting == 0);
+  sock->rcvevent   = 0;
+  /* TCP sendbuf is empty, but the socket is not yet writable until connected
+   * (unless it has been created by accept()). */
+  sock->sendevent  = (NETCONNTYPE_GROUP(newconn->type) == NETCONN_TCP ? (accepted != 0) : 1);
+  sock->errevent   = 0;
+#endif /* LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL */
+
+  if (do_lwip_init_sock(fd) != 0)
+    goto out;
+
+  if (accepted) {
+    int ret = 0;
+    struct tcp_pcb *pcb = newconn->pcb.tcp;
+    if (pcb != NULL && pcb->client_rx_ring != NULL && pcb->client_tx_ring != NULL) {
+        ret = find_same_node_memzone(pcb, sock);
+    }
+    if (pcb == NULL || ret != 0) {
+        goto out;
+    }
+  }
+
+  netconn_set_nonblocking(newconn, flags & SOCK_NONBLOCK);
+  return fd;
+
+out:
+  if (sock != NULL)
+    sock->conn = NULL;
+  posix_api->close_fn(fd);
+  return -1;
+}
+
+/* reference tag: free_socket() */
+void gazelle_free_socket(struct lwip_sock *sock, int fd)
+{
+  do_lwip_clean_sock(fd);
+  posix_api->close_fn(fd);
+}
+
+void lwip_sock_init(void)
+{
+  if (unlikely(sockets == NULL)) {
+    sockets = calloc(MEMP_NUM_NETCONN, sizeof(struct lwip_sock));
+    LWIP_ASSERT("sockets != NULL", sockets != NULL);
+    memset(sockets, 0, MEMP_NUM_NETCONN * sizeof(struct lwip_sock));
+  }
+  return;
+}
+
+void lwip_exit(void)
+{
+  /*
+   * LwIP has the following two parts of memory application, but
+   * it is unnecessary to release all memory in sequentially,
+   * which increases complexity. Therefore, we rely on the process
+   * reclamation mechanism of the system to release memory.
+   * 1. a sockets table of the process.
+   * 2. a batch of hugepage memory of each thread.
+   */
+  return;
+}
diff --git a/src/api/sockets.c b/src/api/sockets.c
index ae14407..c52f286 100644
--- a/src/api/sockets.c
+++ b/src/api/sockets.c
@@ -54,7 +54,6 @@
 #include "lwip/netif.h"
 #include "lwip/priv/tcpip_priv.h"
 #include "lwip/mld6.h"
-#include "lwip/api.h"
 #if LWIP_CHECKSUM_ON_COPY
 #include "lwip/inet_chksum.h"
 #endif
@@ -64,7 +63,6 @@
 #endif
 
 #if GAZELLE_ENABLE
-#include <stdarg.h>
 #include "lwipgz_sock.h"
 #include "lwipgz_posix_api.h"
 #endif
@@ -99,7 +97,6 @@
 /* Address length safe read and write */
 #if LWIP_SOCKET_HAVE_SA_LEN
 
-
 #if LWIP_IPV4
 #define IP4ADDR_SOCKADDR_SET_LEN(sin) \
       (sin)->sin_len = sizeof(struct sockaddr_in)
@@ -138,34 +135,18 @@
 #endif /* LWIP_SOCKET_HAVE_SA_LEN */
 
 #if LWIP_IPV4
-#if GAZELLE_ENABLE
-#define IP4ADDR_PORT_TO_SOCKADDR(sin, ipaddr, port) do { \
-      (sin)->sin_family = AF_INET; \
-      (sin)->sin_port = lwip_htons((port)); \
-      inet_addr_from_ip4addr(&(sin)->sin_addr, ipaddr); \
-      memset((sin)->sin_zero, 0, SIN_ZERO_LEN); }while(0)
-#else
 #define IP4ADDR_PORT_TO_SOCKADDR(sin, ipaddr, port) do { \
       IP4ADDR_SOCKADDR_SET_LEN(sin); \
       (sin)->sin_family = AF_INET; \
       (sin)->sin_port = lwip_htons((port)); \
       inet_addr_from_ip4addr(&(sin)->sin_addr, ipaddr); \
       memset((sin)->sin_zero, 0, SIN_ZERO_LEN); }while(0)
-#endif /* GAZELLE_ENABLE */
 #define SOCKADDR4_TO_IP4ADDR_PORT(sin, ipaddr, port) do { \
     inet_addr_to_ip4addr(ip_2_ip4(ipaddr), &((sin)->sin_addr)); \
     (port) = lwip_ntohs((sin)->sin_port); }while(0)
 #endif /* LWIP_IPV4 */
 
 #if LWIP_IPV6
-#if GAZELLE_ENABLE
-#define IP6ADDR_PORT_TO_SOCKADDR(sin6, ipaddr, port) do { \
-      (sin6)->sin6_family = AF_INET6; \
-      (sin6)->sin6_port = lwip_htons((port)); \
-      (sin6)->sin6_flowinfo = 0; \
-      inet6_addr_from_ip6addr(&(sin6)->sin6_addr, ipaddr); \
-      (sin6)->sin6_scope_id = ip6_addr_zone(ipaddr); }while(0)
-#else
 #define IP6ADDR_PORT_TO_SOCKADDR(sin6, ipaddr, port) do { \
       IP6ADDR_SOCKADDR_SET_LEN(sin6); \
       (sin6)->sin6_family = AF_INET6; \
@@ -173,7 +154,6 @@
       (sin6)->sin6_flowinfo = 0; \
       inet6_addr_from_ip6addr(&(sin6)->sin6_addr, ipaddr); \
       (sin6)->sin6_scope_id = ip6_addr_zone(ipaddr); }while(0)
-#endif /* GAZELLE_ENABLE */
 #define SOCKADDR6_TO_IP6ADDR_PORT(sin6, ipaddr, port) do { \
     inet6_addr_to_ip6addr(ip_2_ip6(ipaddr), &((sin6)->sin6_addr)); \
     if (ip6_addr_has_scope(ip_2_ip6(ipaddr), IP6_UNKNOWN)) { \
@@ -326,11 +306,10 @@ static void lwip_socket_drop_registered_mld6_memberships(int s);
 #endif /* LWIP_IPV6_MLD */
 
 /** The global array of available sockets */
-#if GAZELLE_ENABLE
-uint32_t sockets_num;
-struct lwip_sock *sockets;
-#else
+#if !GAZELLE_ENABLE
 static struct lwip_sock sockets[NUM_SOCKETS];
+#else /* GAZELLE_ENABLE */
+struct lwip_sock *sockets = NULL;
 #endif /* GAZELLE_ENABLE */
 
 #if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
@@ -354,7 +333,7 @@ static struct lwip_select_cb *select_cb_list;
 
 /* Forward declaration of some functions */
 #if LWIP_SOCKET_SELECT || LWIP_SOCKET_POLL
-void event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len);
+static void event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len);
 #define DEFAULT_SOCKET_EVENTCB event_callback
 static void select_check_waiters(int s, int has_recvevent, int has_sendevent, int has_errevent);
 #else
@@ -480,14 +459,8 @@ static struct lwip_sock *
 tryget_socket_unconn_nouse(int fd)
 {
   int s = fd - LWIP_SOCKET_OFFSET;
-
-#if GAZELLE_ENABLE
-  if ((s < 0) || (s >= sockets_num))
-#else
-  if ((s < 0) || (s >= NUM_SOCKETS))
-#endif /* GAZELLE_ENABLE */
-  {
-    LWIP_DEBUGF(SOCKETS_DEBUG , ("tryget_socket_unconn(%d): invalid\n", fd));
+  if ((s < 0) || (s >= NUM_SOCKETS)) {
+    LWIP_DEBUGF(SOCKETS_DEBUG, ("tryget_socket_unconn(%d): invalid\n", fd));
     return NULL;
   }
   return &sockets[s];
@@ -550,18 +523,13 @@ tryget_socket(int fd)
  * @param fd externally used socket index
  * @return struct lwip_sock for the socket or NULL if not found
  */
-#if GAZELLE_ENABLE
 struct lwip_sock *
 get_socket(int fd)
-#else
-static struct lwip_sock *
-get_socket(int fd)
-#endif /* GAZELLE_ENABLE */
 {
   struct lwip_sock *sock = tryget_socket(fd);
   if (!sock) {
     if ((fd < LWIP_SOCKET_OFFSET) || (fd >= (LWIP_SOCKET_OFFSET + NUM_SOCKETS))) {
-      LWIP_DEBUGF(SOCKETS_DEBUG , ("get_socket(%d): invalid\n", fd));
+      LWIP_DEBUGF(SOCKETS_DEBUG, ("get_socket(%d): invalid\n", fd));
     }
     set_errno(EBADF);
     return NULL;
@@ -580,10 +548,7 @@ get_socket(int fd)
 struct lwip_sock *
 get_socket_by_fd(int fd)
 {
-  if ((fd < LWIP_SOCKET_OFFSET) || (fd >= sockets_num + LWIP_SOCKET_OFFSET)) {
-    return NULL;
-  }
-  return &sockets[fd - LWIP_SOCKET_OFFSET];
+  return tryget_socket_unconn_nouse(fd);
 }
 #endif /* GAZELLE_ENABLE */
 
@@ -593,85 +558,19 @@ get_socket_by_fd(int fd)
  * @param newconn the netconn for which to allocate a socket
  * @param accepted 1 if socket has been created by accept(),
  *                 0 if socket has been created by socket()
- * @param flags  only support SOCK_CLOEXEC and SOCK_NONBLOCK
  * @return the index of the new socket; -1 on error
  */
 static int
 alloc_socket(struct netconn *newconn, int accepted, int flags)
 {
+#if GAZELLE_ENABLE
+  return gazelle_alloc_socket(newconn, accepted, flags);
+#endif /* GAZELLE_ENABLE */
+
   int i;
   SYS_ARCH_DECL_PROTECT(lev);
   LWIP_UNUSED_ARG(accepted);
 
-#if GAZELLE_ENABLE
-  int type, protocol = 0;
-  int domain = NETCONNTYPE_ISIPV6(newconn->type) ? AF_INET6 : AF_INET;
-  switch (NETCONNTYPE_GROUP(newconn->type)) {
-    case NETCONN_RAW:
-      type = SOCK_RAW;
-      break;
-    case NETCONN_UDPLITE:
-    case NETCONN_UDP:
-      type = SOCK_DGRAM;
-      break;
-    case NETCONN_TCP:
-      type = SOCK_STREAM;
-      break;
-    default:
-      type = -1;
-      break;
-  }
-
-  /*add CLOEXEC OR NONBLOCK OR NONE*/
-  type |= flags;
-
-  SYS_ARCH_PROTECT(lev);
-  i = posix_api->socket_fn(domain, type, protocol);
-  if (i == -1) {
-    goto err;
-  }
-
-  if ((flags & O_NONBLOCK) != 0){
-    netconn_set_nonblocking(newconn, flags & O_NONBLOCK);
-  }
-
-  if ((i < LWIP_SOCKET_OFFSET) || (i >= sockets_num + LWIP_SOCKET_OFFSET)) {
-    goto err;
-  }
-
-  if (!sockets[i].conn && (sockets[i].select_waiting == 0)) {
-    /*initialize state as NETCONN_HOST | NETCONN_LIBOS,
-     *if connection accepted and alloc_socket called, it can be only NETCONN_LIBOS*/
-    if (accepted)
-      SET_CONN_TYPE_LIBOS(newconn);
-    else
-      SET_CONN_TYPE_LIBOS_OR_HOST(newconn);
-    sockets[i].conn       = newconn;
-    /* The socket is not yet known to anyone, so no need to protect
-       after having marked it as used. */
-    SYS_ARCH_UNPROTECT(lev);
-    sockets[i].lastdata.pbuf = NULL;
-    sockets[i].rcvevent   = 0;
-    /* TCP sendbuf is empty, but the socket is not yet writable until connected
-     * (unless it has been created by accept()). */
-    sockets[i].sendevent  = (NETCONNTYPE_GROUP(newconn->type) == NETCONN_TCP ? (accepted != 0) : 1);
-    sockets[i].errevent   = 0;
-    sockets[i].same_node_rx_ring = NULL;
-    sockets[i].same_node_rx_ring_mz = NULL;
-    sockets[i].same_node_tx_ring = NULL;
-    sockets[i].same_node_tx_ring_mz = NULL;
-    return i + LWIP_SOCKET_OFFSET;
-  } else {
-    lwip_close(i);
-    do_lwip_clean_sock(i);
-  }
-
-err:
-  posix_api->close_fn(i);
-  SYS_ARCH_UNPROTECT(lev);
-  return -1;
-#else /* GAZELLE_ENABLE */
-
   /* allocate a new socket identifier */
   for (i = 0; i < NUM_SOCKETS; ++i) {
     /* Protect socket array */
@@ -703,8 +602,6 @@ err:
     SYS_ARCH_UNPROTECT(lev);
   }
   return -1;
-
-#endif /* GAZELLE_ENABLE */
 }
 
 /** Free a socket (under lock)
@@ -761,7 +658,7 @@ free_socket_free_elements(int is_tcp, struct netconn *conn, union lwip_sock_last
  * @param is_tcp != 0 for TCP sockets, used to free lastdata
  */
 static void
-free_socket(struct lwip_sock *sock, int is_tcp)
+free_socket(struct lwip_sock *sock, int is_tcp, int fd)
 {
   int freed;
   struct netconn *conn;
@@ -771,16 +668,15 @@ free_socket(struct lwip_sock *sock, int is_tcp)
   /* Protect socket array */
   SYS_ARCH_PROTECT(lev);
 
-#if GAZELLE_ENABLE
-  /* remove sock from same_node_recv_lit */
-  list_del_node_null(&sock->recv_list);
-#endif
-
   freed = free_socket_locked(sock, is_tcp, &conn, &lastdata);
   SYS_ARCH_UNPROTECT(lev);
   /* don't use 'sock' after this line, as another task might have allocated it */
 
   if (freed) {
+#if GAZELLE_ENABLE
+    /* fd must be closed after free_socket_locked */
+    gazelle_free_socket(sock, fd);
+#endif /* GAZELLE_ENABLE */
     free_socket_free_elements(is_tcp, conn, &lastdata);
   }
 }
@@ -832,27 +728,8 @@ lwip_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags)
     done_socket(sock);
     return -1;
   }
-#if GAZELLE_ENABLE
-  LWIP_ASSERT("invalid socket index", (newsock >= LWIP_SOCKET_OFFSET) && (newsock < sockets_num + LWIP_SOCKET_OFFSET));
-  do_lwip_init_sock(newsock);
-#else
   LWIP_ASSERT("invalid socket index", (newsock >= LWIP_SOCKET_OFFSET) && (newsock < NUM_SOCKETS + LWIP_SOCKET_OFFSET));
-#endif /* GAZELLE_ENABLE */
   nsock = &sockets[newsock - LWIP_SOCKET_OFFSET];
-#if GAZELLE_ENABLE
-  int ret = 0;
-  struct tcp_pcb *pcb = newconn->pcb.tcp;
-  if (pcb != NULL && pcb->client_rx_ring != NULL && pcb->client_tx_ring != NULL) {
-    ret = find_same_node_memzone(pcb, nsock);
-  }
-  if (pcb == NULL || ret != 0) {
-      netconn_delete(newconn);
-      free_socket(nsock, 1);
-      set_errno(ENOTCONN);
-      done_socket(sock);
-      return -1;
-  }
-#endif
 
   /* See event_callback: If data comes in right away after an accept, even
    * though the server task might not have created a new socket yet.
@@ -882,20 +759,16 @@ lwip_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags)
     err = netconn_peer(newconn, &naddr, &port);
     if (err != ERR_OK) {
       LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_accept(%d): netconn_peer failed, err=%d\n", s, err));
-      free_socket(nsock, 1);
+      free_socket(nsock, 1, newsock);
       set_errno(err_to_errno(err));
       done_socket(sock);
       return -1;
     }
 
     IPADDR_PORT_TO_SOCKADDR(&tempaddr, &naddr, port);
-#if !GAZELLE_ENABLE
     if (*addrlen > IPADDR_SOCKADDR_GET_LEN(&tempaddr)) {
       *addrlen = IPADDR_SOCKADDR_GET_LEN(&tempaddr);
     }
-#else
-        *addrlen = LWIP_MIN(*addrlen, sizeof(tempaddr));
-#endif /* GAZELLE_ENABLE */
     MEMCPY(addr, &tempaddr, *addrlen);
 
     LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_accept(%d) returning new sock=%d addr=", s, newsock));
@@ -924,6 +797,7 @@ lwip_bind(int s, const struct sockaddr *name, socklen_t namelen)
   ip_addr_t local_addr;
   u16_t local_port;
   err_t err;
+
   sock = get_socket(s);
   if (!sock) {
     return -1;
@@ -978,6 +852,7 @@ lwip_close(int s)
   err_t err;
 
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_close(%d)\n", s));
+
   sock = get_socket(s);
   if (!sock) {
     return -1;
@@ -1005,7 +880,7 @@ lwip_close(int s)
     return -1;
   }
 
-  free_socket(sock, is_tcp);
+  free_socket(sock, is_tcp, s);
   set_errno(0);
   return 0;
 }
@@ -1064,11 +939,6 @@ lwip_connect(int s, const struct sockaddr *name, socklen_t namelen)
     return -1;
   }
 
-#if GAZELLE_ENABLE
-  LWIP_DEBUGF(SOCKETS_DEBUG, ("libos connect succeed fd=%d\n",s));
-  SET_CONN_TYPE_LIBOS(sock->conn);
-#endif /* GAZELLE_ENABLE */
-
   LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_connect(%d) succeeded\n", s));
   set_errno(0);
   done_socket(sock);
@@ -1261,13 +1131,11 @@ lwip_sock_make_addr(struct netconn *conn, ip_addr_t *fromaddr, u16_t port,
 #endif /* LWIP_IPV4 && LWIP_IPV6 */
 
   IPADDR_PORT_TO_SOCKADDR(&saddr, fromaddr, port);
-#if !GAZELLE_ENABLE
   if (*fromlen < IPADDR_SOCKADDR_GET_LEN(&saddr)) {
     truncated = 1;
   } else if (*fromlen > IPADDR_SOCKADDR_GET_LEN(&saddr)) {
     *fromlen = IPADDR_SOCKADDR_GET_LEN(&saddr);
   }
-#endif
   MEMCPY(from, &saddr, *fromlen);
   return truncated;
 }
@@ -1472,7 +1340,6 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
       done_socket(sock);
       return -1;
     }
-
     ret = (ssize_t)LWIP_MIN(LWIP_MIN(len, datagram_len), SSIZE_MAX);
     if (fromlen) {
       *fromlen = msg.msg_namelen;
@@ -1950,12 +1817,8 @@ lwip_socket(int domain, int type, int protocol)
 
   LWIP_UNUSED_ARG(domain); /* @todo: check this */
 
-  int flags = type & ~SOCK_TYPE_MASK;
-  type &= SOCK_TYPE_MASK;
-
-
   /* create a netconn */
-  switch (type) {
+  switch (type & SOCK_TYPE_MASK) {
     case SOCK_RAW:
       conn = netconn_new_with_proto_and_callback(DOMAIN_TO_NETCONN_TYPE(domain, NETCONN_RAW),
              (u8_t)protocol, DEFAULT_SOCKET_EVENTCB);
@@ -1993,15 +1856,7 @@ lwip_socket(int domain, int type, int protocol)
     return -1;
   }
 
-  if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)){
-    set_errno(EINVAL);
-    return -1;
-  }
-
-  if (SOCK_NONBLOCK != O_NONBLOCK && (flags & SOCK_NONBLOCK))
-    flags = (flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
-
-  i = alloc_socket(conn, 0, flags);
+  i = alloc_socket(conn, 0, type & ~SOCK_TYPE_MASK);
 
   if (i == -1) {
     netconn_delete(conn);
@@ -2753,7 +2608,7 @@ lwip_poll_should_wake(const struct lwip_select_cb *scb, int fd, int has_recveven
  *   NETCONN_EVT_ERROR
  * This requirement will be asserted in select_check_waiters()
  */
-void
+static void
 event_callback(struct netconn *conn, enum netconn_evt evt, u16_t len)
 {
   int s, check_waiters;
@@ -3051,14 +2906,9 @@ lwip_getaddrname(int s, struct sockaddr *name, socklen_t *namelen, u8_t local)
   ip_addr_debug_print_val(SOCKETS_DEBUG, naddr);
   LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F")\n", port));
 
-#if !GAZELLE_ENABLE
   if (*namelen > IPADDR_SOCKADDR_GET_LEN(&saddr)) {
     *namelen = IPADDR_SOCKADDR_GET_LEN(&saddr);
   }
-#else
-  u8_t sa_len = IP_IS_V4_VAL(naddr) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6);
-  *namelen = LWIP_MIN(*namelen, sa_len);
-#endif
   MEMCPY(name, &saddr, *namelen);
 
   set_errno(0);
@@ -3082,13 +2932,12 @@ int
 lwip_getsockopt(int s, int level, int optname, void *optval, socklen_t *optlen)
 {
   int err;
+  struct lwip_sock *sock = get_socket(s);
 #if !LWIP_TCPIP_CORE_LOCKING
   err_t cberr;
   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
 #endif /* !LWIP_TCPIP_CORE_LOCKING */
 
-  struct lwip_sock *sock = get_socket(s);
-
   if (!sock) {
     return -1;
   }
@@ -3533,13 +3382,12 @@ int
 lwip_setsockopt(int s, int level, int optname, const void *optval, socklen_t optlen)
 {
   int err = 0;
+  struct lwip_sock *sock = get_socket(s);
 #if !LWIP_TCPIP_CORE_LOCKING
   err_t cberr;
   LWIP_SETGETSOCKOPT_DATA_VAR_DECLARE(data);
 #endif /* !LWIP_TCPIP_CORE_LOCKING */
 
-  struct lwip_sock *sock = get_socket(s);
-
   if (!sock) {
     return -1;
   }
@@ -4249,7 +4097,7 @@ lwip_setsockopt_impl(int s, int level, int optname, const void *optval, socklen_
 int
 lwip_ioctl(int s, long cmd, ...)
 {
-  struct lwip_sock *sock = posix_api->get_socket(s);
+  struct lwip_sock *sock = get_socket(s);
   u8_t val;
 
 #if LWIP_SO_RCVBUF
@@ -4428,12 +4276,7 @@ lwip_fcntl(int s, int cmd, int val)
       break;
     default:
       LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_fcntl(%d, UNIMPL: %d, %d)\n", s, cmd, val));
-#if GAZELLE_ENABLE
-      set_errno(0); /* not yet implemented, but we return 0 for compatilbe with app */
-      ret = 0;
-#else
       set_errno(ENOSYS); /* not yet implemented */
-#endif
       break;
   }
   done_socket(sock);
@@ -5400,31 +5243,4 @@ out:
 }
 #endif /* (LWIP_IPV4 && LWIP_IGMP) || (LWIP_IPV6 && LWIP_IPV6_MLD)*/
 
-#if GAZELLE_ENABLE
-void lwip_sock_init(void)
-{
-  if (sockets_num == 0) {
-    sockets_num = NUM_SOCKETS;
-    sockets = calloc(sockets_num, sizeof(struct lwip_sock));
-    LWIP_ASSERT("sockets != NULL", sockets != NULL);
-    memset(sockets, 0, sockets_num * sizeof(struct lwip_sock));
-  }
-  return;
-}
-
-void lwip_exit(void)
-{
-  /*
-   * LwIP has the following two parts of memory application, but
-   * it is unnecessary to release all memory in sequentially,
-   * which increases complexity. Therefore, we rely on the process
-   * reclamation mechanism of the system to release memory.
-   * 1. a sockets table of the process.
-   * 2. a batch of hugepage memory of each thread.
-   */
-  return;
-}
-
-#endif /* GAZELLE_ENABLE */
-
 #endif /* LWIP_SOCKET */
diff --git a/src/core/init.c b/src/core/init.c
index 821c6c7..9f130a6 100644
--- a/src/core/init.c
+++ b/src/core/init.c
@@ -61,6 +61,10 @@
 #include "netif/ppp/ppp_opts.h"
 #include "netif/ppp/ppp_impl.h"
 
+#if GAZELLE_ENABLE
+#include "lwipgz_memp.h"
+#endif  /* GAZELLE_ENABLE */
+
 #ifndef LWIP_SKIP_PACKING_CHECK
 
 #ifdef PACK_STRUCT_USE_INCLUDES
@@ -351,6 +355,12 @@ lwip_init(void)
   LWIP_ASSERT("Struct packing not implemented correctly. Check your lwIP port.", sizeof(struct packed_struct_test) == PACKED_STRUCT_TEST_EXPECTED_SIZE);
 #endif
 
+#if GAZELLE_ENABLE
+  if (hugepage_init() != 0) {
+    rte_exit(EXIT_FAILURE, "hugepage init failed\n");
+  }
+#endif  /* GAZELLE_ENABLE */
+
   /* Modules initialization */
   stats_init();
 
diff --git a/src/include/lwip/priv/sockets_priv.h b/src/include/lwip/priv/sockets_priv.h
index 432f09b..f938e84 100644
--- a/src/include/lwip/priv/sockets_priv.h
+++ b/src/include/lwip/priv/sockets_priv.h
@@ -45,7 +45,7 @@
 #include "lwip/sockets.h"
 #include "lwip/sys.h"
 
-/* move some definitions to the lwipgz_sock.h for libnet to use, and
+/* move some definitions to the lwipgz_sock.h for `gazelle` to use, and
  * at the same time avoid conflict between lwip/sockets.h and sys/socket.h
  */
 #include "lwipgz_sock.h"
diff --git a/src/include/lwip/sockets.h b/src/include/lwip/sockets.h
index 038afba..9f5bc81 100644
--- a/src/include/lwip/sockets.h
+++ b/src/include/lwip/sockets.h
@@ -61,7 +61,7 @@ extern "C" {
 #endif
 
 /* sockaddr and pals include length fields */
-#define LWIP_SOCKET_HAVE_SA_LEN  1
+#define LWIP_SOCKET_HAVE_SA_LEN  0
 
 /* If your port already typedef's sa_family_t, define SA_FAMILY_T_DEFINED
    to prevent this code from redefining it. */
@@ -468,6 +468,28 @@ typedef struct ipv6_mreq {
 #define IPTOS_PREC_PRIORITY             0x20
 #define IPTOS_PREC_ROUTINE              0x00
 
+#if GAZELLE_ENABLE
+#ifndef SOCK_TYPE_MASK
+#define SOCK_TYPE_MASK  0xf
+#endif
+
+#ifndef FIONBIO
+#define FIONBIO    0x5421 /* same as define in asm-generic/ioctls.h */
+#endif
+#ifndef O_NONBLOCK
+#define O_NONBLOCK 04000 /* same as define in bits/fcntl-linux.h */
+#endif
+#ifndef O_CLOEXEC
+#define O_CLOEXEC  02000000 /* same as define in bits/fcntl-linux.h */
+#endif
+
+#ifndef SOCK_NONBLOCK
+#define SOCK_NONBLOCK O_NONBLOCK
+#endif
+#ifndef SOCK_CLOEXEC
+#define SOCK_CLOEXEC  O_CLOEXEC
+#endif
+#endif /* GAZELLE_ENABLE */
 
 /*
  * Commands for ioctlsocket(),  taken from the BSD file fcntl.h.
@@ -683,23 +705,6 @@ int fcntl(int s, int cmd, ...);
 #endif /* LWIP_POSIX_SOCKETS_IO_NAMES */
 #endif /* LWIP_COMPAT_SOCKETS == 2 */
 
-#ifndef O_CLOEXEC
-#define O_CLOEXEC       02000000
-#endif
-
-#ifndef SOCK_TYPE_MASK
-#define SOCK_TYPE_MASK  0xf
-#endif
-
-#ifndef SOCK_CLOEXEC
-#define SOCK_CLOEXEC    O_CLOEXEC
-#endif
-
-#ifndef SOCK_NONBLOCK
-#define SOCK_NONBLOCK   O_NONBLOCK
-#endif
-
-
 int lwip_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
 int lwip_accept4(int s, struct sockaddr *addr, socklen_t *addrlen, int flags);
 int lwip_bind(int s, const struct sockaddr *name, socklen_t namelen);
diff --git a/src/include/lwipgz_posix_api.h b/src/include/lwipgz_posix_api.h
index 4e827c1..d64d854 100644
--- a/src/include/lwipgz_posix_api.h
+++ b/src/include/lwipgz_posix_api.h
@@ -71,10 +71,8 @@ typedef struct {
     int (*epoll_create1_fn)(int size);
     int (*epoll_ctl_fn)(int epfd, int op, int fd, struct epoll_event *event);
     int (*epoll_wait_fn)(int epfd, struct epoll_event *events, int maxevents, int timeout);
-    int (*epoll_close_fn)(int epfd);
     int (*eventfd_fn)(unsigned int initval, int flags);
     int (*is_epfd)(int fd);
-    struct lwip_sock* (*get_socket)(int fd);
     int (*sigaction_fn)(int signum, const struct sigaction *act, struct sigaction *oldact);
     int (*poll_fn)(struct pollfd *fds, nfds_t nfds, int timeout);
     int (*ioctl_fn)(int fd, int cmd, ...);
diff --git a/src/include/lwipgz_sock.h b/src/include/lwipgz_sock.h
index 01b39e4..f5581cb 100644
--- a/src/include/lwipgz_sock.h
+++ b/src/include/lwipgz_sock.h
@@ -33,12 +33,61 @@
 #ifndef __LWIPGZ_SOCK_H__
 #define __LWIPGZ_SOCK_H__
 
-#include <semaphore.h>
 #include "lwip/opt.h"
 #include "lwip/api.h"
 
+#if GAZELLE_ENABLE
+#include <semaphore.h>
+#include <rte_common.h>
+#include <rte_memzone.h>
 #include "lwipgz_event.h"
 
+#define set_errno(err) do { errno = (err); } while(0)
+
+struct lwip_sock *get_socket(int fd);
+struct lwip_sock *get_socket_by_fd(int fd);
+int gazelle_alloc_socket(struct netconn *newconn, int accepted, int flags);
+void gazelle_free_socket(struct lwip_sock *sock, int fd);
+void lwip_sock_init(void);
+void lwip_exit(void);
+
+
+extern int do_lwip_init_sock(int fd);
+extern void do_lwip_clean_sock(int fd);
+
+extern void do_lwip_add_recvlist(int32_t fd);
+extern void do_lwip_connected_callback(struct netconn *conn);
+
+extern struct pbuf *do_lwip_udp_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size);
+extern struct pbuf *do_lwip_tcp_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size);
+extern void do_lwip_get_from_sendring_over(struct lwip_sock *sock);
+extern ssize_t do_lwip_read_from_lwip(struct lwip_sock *sock, int32_t flags, u8_t apiflags);
+
+struct sock_time_stamp {
+    uint64_t rpc_time_stamp;
+    uint64_t mbox_time_stamp;
+};
+extern void lstack_calculate_aggregate(int type, uint32_t len);
+extern void time_stamp_transfer_pbuf(struct pbuf *pbuf_old, struct pbuf *pbuf_new);
+extern void time_stamp_record(int fd, struct pbuf *pbuf);
+
+// 8M
+#define SAME_NODE_RING_LEN (unsigned long long)(8388608)
+#define SAME_NODE_RING_MASK (unsigned long long)(8388608 - 1)
+#define RING_NAME_LEN 32
+struct same_node_ring {
+    const struct rte_memzone *mz;
+    unsigned long long sndbegin;
+    unsigned long long sndend;
+};
+extern err_t same_node_ring_create(struct rte_ring **ring, int size, int port, char *name, char *rx);
+extern err_t create_same_node_ring(struct tcp_pcb *pcb);
+extern err_t find_same_node_ring(struct tcp_pcb *pcb);
+extern err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock);
+
+#endif /* GAZELLE_ENABLE */
+
+
 /* move some definitions to the lwipgz_sock.h for libnet to use, and
  * at the same time avoid conflict between lwip/sockets.h and sys/socket.h
  */
@@ -60,29 +109,6 @@ union lwip_sock_lastdata {
   struct pbuf *pbuf;
 };
 
-#if GAZELLE_ENABLE
-struct protocol_stack;
-struct wakeup_poll;
-struct rte_ring;
-#include <rte_common.h>
-#include <rte_memzone.h>
-
-// 8M
-#define SAME_NODE_RING_LEN (unsigned long long)(8388608)
-#define SAME_NODE_RING_MASK (unsigned long long)(8388608 - 1)
-#define RING_NAME_LEN 32
-struct same_node_ring {
-    const struct rte_memzone *mz;
-    unsigned long long sndbegin;
-    unsigned long long sndend;
-};
-
-struct sock_time_stamp {
-    uint64_t rpc_time_stamp;
-    uint64_t mbox_time_stamp;
-};
-#endif
-
 /** Contains all internal pointers and states used for a socket */
 struct lwip_sock {
   /** sockets currently are built on netconns, each socket has one netconn */
@@ -130,10 +156,10 @@ struct lwip_sock {
 
   char pad3 __rte_cache_aligned;
   /* nerver change */
-  struct wakeup_poll *wakeup;
-  epoll_data_t ep_data;
   struct lwip_sock *listen_next; /* listenfd list */
   struct protocol_stack *stack;
+  struct wakeup_poll *wakeup;
+  epoll_data_t ep_data;
   struct rte_ring *recv_ring;
   struct rte_ring *send_ring;
 
@@ -145,57 +171,7 @@ struct lwip_sock {
   uint8_t already_bind_numa;
 
   struct sock_time_stamp stamp;
-#endif
-};
-
-#if GAZELLE_ENABLE
-static inline unsigned same_node_ring_count(struct lwip_sock *sock)
-{
-  const unsigned long long cur_begin = __atomic_load_n(&sock->same_node_rx_ring->sndbegin, __ATOMIC_RELAXED);
-  const unsigned long long cur_end = __atomic_load_n(&sock->same_node_rx_ring->sndend, __ATOMIC_RELAXED);
-
-  return cur_end - cur_begin;
-}
-#endif
-
-#ifndef set_errno
-#define set_errno(err) do { if (err) { errno = (err); } } while(0)
-#endif
-
-
-/* --------------------------------------------------
- * --------------- LIBNET references ----------------
- * --------------------------------------------------
- */
-#if GAZELLE_ENABLE
-extern uint32_t sockets_num;
-extern struct lwip_sock *sockets;
-
-extern void do_lwip_init_sock(int32_t fd);
-extern void do_lwip_clean_sock(int32_t fd);
-extern void do_lwip_connected_callback(struct netconn *conn);
-extern struct pbuf *do_lwip_udp_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size);
-extern struct pbuf *do_lwip_tcp_get_from_sendring(struct lwip_sock *sock, uint16_t remain_size);
-extern void do_lwip_get_from_sendring_over(struct lwip_sock *sock);
-extern ssize_t do_lwip_read_from_lwip(struct lwip_sock *sock, int32_t flags, u8_t apiflags);
-extern void do_lwip_add_recvlist(int32_t fd);
-
-extern void netif_poll(struct netif *netif);
-extern err_t netif_loop_output(struct netif *netif, struct pbuf *p);
-extern err_t find_same_node_memzone(struct tcp_pcb *pcb, struct lwip_sock *nsock);
-extern err_t same_node_memzone_create(const struct rte_memzone **zone, int size, int port, char *name, char *);
-extern err_t same_node_ring_create(struct rte_ring **ring, int size, int port, char *name, char *rx);
-extern err_t create_same_node_ring(struct tcp_pcb *pcb);
-extern err_t find_same_node_ring(struct tcp_pcb *pcb);
-extern void lstack_calculate_aggregate(int type, uint32_t len);
-extern void time_stamp_transfer_pbuf(struct pbuf *pbuf_old, struct pbuf *pbuf_new);
-extern void time_stamp_record(int fd, struct pbuf *pbuf);
-
 #endif /* GAZELLE_ENABLE */
-
-struct lwip_sock *get_socket(int s);
-struct lwip_sock *get_socket_by_fd(int s);
-void lwip_sock_init(void);
-void lwip_exit(void);
+};
 
 #endif /* __LWIPGZ_SOCK_H__ */
diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h
index e428f2a..31856ce 100644
--- a/src/include/lwipopts.h
+++ b/src/include/lwipopts.h
@@ -247,14 +247,6 @@
 
 #define SO_REUSE 1
 
-#ifndef FIONBIO
-#define FIONBIO 0x5421 /* same as define in asm-generic/ioctls.h */
-#endif
-
-#ifndef O_NONBLOCK
-#define O_NONBLOCK 04000 /* same as define in bits/fcntl-linux.h */
-#endif
-
 #define SIOCSHIWAT 1
 
 #define LWIP_SO_RCVTIMEO 1
-- 
2.33.0

