From 5ec7b92cf94a4b7eb4587763aab9f814286bfe27 Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Thu, 27 Jul 2023 20:02:33 +1400
Subject: [PATCH 02/11] add health check abnormal test_cases

Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
 CI/install_depends.sh                         |  1 +
 CI/pr-gateway.sh                              |  1 +
 CI/test_cases/container_cases/health_check.sh | 90 +++++++++++++++----
 3 files changed, 76 insertions(+), 16 deletions(-)

diff --git a/CI/install_depends.sh b/CI/install_depends.sh
index 88922686..df0474c7 100755
--- a/CI/install_depends.sh
+++ b/CI/install_depends.sh
@@ -91,6 +91,7 @@ check_make_status make_cni_plugins ${build_log_cni_plugins} &
 cd ~
 git clone https://gitee.com/src-openeuler/lxc.git
 cd lxc
+git checkout origin/openEuler-22.03-LTS-SP2
 tar xf lxc-4.0.3.tar.gz
 cd lxc-4.0.3
 mv ../*.patch .
diff --git a/CI/pr-gateway.sh b/CI/pr-gateway.sh
index b3da52d0..8223ddf5 100755
--- a/CI/pr-gateway.sh
+++ b/CI/pr-gateway.sh
@@ -35,6 +35,7 @@ cd ~
 rm -rf lxc
 git clone https://gitee.com/src-openeuler/lxc.git
 pushd lxc
+git checkout origin/openEuler-22.03-LTS-SP2
 rm -rf lxc-4.0.3
 ./apply-patches || exit 1
 pushd lxc-4.0.3
diff --git a/CI/test_cases/container_cases/health_check.sh b/CI/test_cases/container_cases/health_check.sh
index efb357e0..5e6b5641 100755
--- a/CI/test_cases/container_cases/health_check.sh
+++ b/CI/test_cases/container_cases/health_check.sh
@@ -43,18 +43,30 @@ function do_retry()
     return 1
 }
 
-function inspect_container_status()
+function inspect_container_health_status()
 {
     [[ $(isula inspect -f '{{.State.Health.Status}}' ${1}) == "${2}" ]]
     return $?
 }
 
+function inspect_container_status()
+{
+    [[ $(isula inspect -f '{{.State.Status}}' ${1}) == "${2}" ]]
+    return $?
+}
+
 function inspect_container_exitcode()
 {
     [[ $(isula inspect -f '{{.State.ExitCode}}' ${1}) == "${2}" ]]
     return $?
 }
 
+function inspect_container_failing_streak()
+{
+    [[ $(isula inspect -f '{{.State.Health.FailingStreak}}' ${1}) == "${2}" ]]
+    return $?
+}
+
 function test_health_check_paraments()
 {
     local ret=0
@@ -73,29 +85,29 @@ function test_health_check_paraments()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
     # start period : 2s => do health check => interval: 2s => do health check => exit on unhealthy
-    [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} running
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
     # finish first health check
     sleep 10
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} starting
     # keep starting status with health check return non-zero at always until status change to unhealthy
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 6 # finish second health check
 
     success=1
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} unhealthy
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
     # validate --health-retries option
-    [[ $(isula inspect -f '{{.State.Health.FailingStreak}}' ${container_name}) == "2" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
+    do_retry ${retry_limit} ${retry_interval} inspect_container_failing_streak  ${container_name} 2
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check failing streak: not 2" && ((ret++))
 
-    [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "exited" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} exited
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not exited" && ((ret++))
 
-    [[ $(isula inspect -f '{{.State.ExitCode}}' ${container_name}) == "137" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_exitcode  ${container_name} 137
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container exit code: not 137" && ((ret++))
 
     isula rm -f ${container_name}
@@ -123,24 +135,24 @@ function test_health_check_normally()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
     # start period : 0s => interval: 2s => do health check => interval: 2s => do health check => ...
-    [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} running
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
     # Health check has been performed yet
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} starting
     # Initial status when the container is still starting
     [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 8 # finish first health check
     
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} healthy
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} healthy
     # When the health check returns successfully, status immediately becomes healthy
     [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not healthy" && ((ret++))
 
     kill -9 $(isula inspect -f '{{.State.Pid}}' ${container_name})
     
     # Wait for the container to be killed
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} unhealthy
     # The container process exits abnormally and the health check status becomes unhealthy
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
@@ -174,17 +186,17 @@ function test_health_check_timeout()
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
 
     # start period : 0s => interval: 5s => do health check(1s timeout) => unhealthy(exited)
-    [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} running
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
     # Health check has been performed yet
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} starting
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} starting
     # Initial status when the container is still starting
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
 
     sleep 7 # finish first health check
     
-    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} unhealthy
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} unhealthy
     # The container process exits and the health check status becomes unhealthy
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
@@ -229,6 +241,50 @@ function test_health_check_monitor()
     return ${ret}
 }
 
+function test_health_check_abnormal()
+{
+    local ret=0
+    local image="busybox"
+    local retry_limit=10
+    local retry_interval=1
+    local test="health check abnormal => (${FUNCNAME[@]})"
+
+    msg_info "${test} starting..."
+
+    isula images | grep ${image}
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
+
+    container_name="health_check_abnormal"
+    isula run -itd --runtime $1 -n ${container_name} --health-cmd="sleep 999" --health-timeout=1000000s --health-retries=1  ${image} /bin/sh
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++))
+
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} running
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
+
+    # Health check has been performed yet
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} starting
+    # Initial status when the container is still starting
+    [[ $? -ne 0 ]]  && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
+
+    sleep 30 # wait health check exec
+
+    isula stop -t 0 ${container_name} &
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to stop container: ${container_name}" && ((ret++))
+
+    do_retry ${retry_limit} ${retry_interval} inspect_container_status  ${container_name} exited
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not exited" && ((ret++))
+
+    do_retry ${retry_limit} ${retry_interval} inspect_container_health_status  ${container_name} unhealthy
+    # The container process exits and the health check status becomes unhealthy
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
+
+    isula rm -f ${container_name}
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to remove container: ${container_name}" && ((ret++))
+
+    msg_info "${test} finished with return ${ret}..."
+    return ${ret}
+}
+
 declare -i ans=0
 
 for element in ${RUNTIME_LIST[@]};
@@ -244,7 +300,9 @@ do
 
     test_health_check_monitor $element || ((ans++))
 
+    test_health_check_abnormal $element || ((ans++))
+
     msg_info "${test} finished with return ${ans}..."
 done
 
-show_result ${ans} "${curr_path}/${0}"
+show_result ${ans} "${curr_path}/${0}"
\ No newline at end of file
-- 
2.25.1

