From e04d2f1d8382e7b3e81ab4725a21562147ad1727 Mon Sep 17 00:00:00 2001
From: zhongtao <zhongtao17@huawei.com>
Date: Mon, 22 May 2023 12:50:32 +0800
Subject: [PATCH 7/9] reinforce cri_stream.sh and health_check.sh

Signed-off-by: zhongtao <zhongtao17@huawei.com>
---
 CI/test_cases/container_cases/cri_stream.sh   | 30 ++++++++++++----
 CI/test_cases/container_cases/health_check.sh | 34 ++++++++++++++-----
 2 files changed, 50 insertions(+), 14 deletions(-)

diff --git a/CI/test_cases/container_cases/cri_stream.sh b/CI/test_cases/container_cases/cri_stream.sh
index 8b5440d3..bfe90208 100755
--- a/CI/test_cases/container_cases/cri_stream.sh
+++ b/CI/test_cases/container_cases/cri_stream.sh
@@ -58,6 +58,9 @@ function set_up()
 function test_cri_exec_fun()
 {
     local ret=0
+    local retry_limit=20
+    local retry_interval=1
+    local success=1
     local test="test_cri_exec_fun => (${FUNCNAME[@]})"
     msg_info "${test} starting..."
     declare -a fun_pids
@@ -74,9 +77,15 @@ function test_cri_exec_fun()
     done
     wait ${abn_pids[*]// /|}
 
-    sleep 2
-    ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
-    [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
+    for i in $(seq 1 "$retry_limit"); do
+        ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
+        if [ $? -ne 0 ]; then
+            success=0
+            break;
+        fi
+        sleep $retry_interval
+    done
+    [[ $success -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
 
     msg_info "${test} finished with return ${ret}..."
     return ${ret}
@@ -85,6 +94,9 @@ function test_cri_exec_fun()
 function test_cri_exec_abn
 {
     local ret=0
+    local retry_limit=20
+    local retry_interval=1
+    local success=1
     local test="test_cri_exec_abn => (${FUNCNAME[@]})"
     msg_info "${test} starting..."
 
@@ -92,10 +104,16 @@ function test_cri_exec_abn
     pid=$!
     sleep 3
     kill -9 $pid
-    sleep 2
 
-    ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
-    [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
+    for i in $(seq 1 "$retry_limit"); do
+        ps -T -p $(cat /var/run/isulad.pid) | grep IoCopy
+        if [ $? -ne 0 ]; then
+            success=0
+            break;
+        fi
+        sleep $retry_interval
+    done
+    [[ $success -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - residual IO copy thread in CRI exec operation" && ((ret++))
 
     msg_info "${test} finished with return ${ret}..."
     return ${ret}
diff --git a/CI/test_cases/container_cases/health_check.sh b/CI/test_cases/container_cases/health_check.sh
index 1542bd09..28af6149 100755
--- a/CI/test_cases/container_cases/health_check.sh
+++ b/CI/test_cases/container_cases/health_check.sh
@@ -123,6 +123,9 @@ function test_health_check_timeout()
 {
     local ret=0
     local image="busybox"
+    local retry_limit=10
+    local retry_interval=1
+    local success=1
     local test="list && inspect image info test => (${FUNCNAME[@]})"
 
     msg_info "${test} starting..."
@@ -139,16 +142,31 @@ function test_health_check_timeout()
     [[ $(isula inspect -f '{{.State.Status}}' ${container_name}) == "running" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container status: not running" && ((ret++))
 
-    sleep 2 # Health check has been performed yet
-
+    # Health check has been performed yet
+    for i in $(seq 1 "$retry_limit"); do
+        [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "starting" ]]
+        if [ $? -eq 0 ]; then
+            success=0
+            break;
+        fi
+        sleep $retry_interval
+    done
     # Initial status when the container is still starting
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "starting" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
-
-    sleep 10 # finish first health check
+    [[ $success -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not starting" && ((ret++))
+
+    sleep 7 # finish first health check
+    
+    success=1
+    for i in $(seq 1 "$retry_limit"); do
+        [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "unhealthy" ]]
+        if [ $? -eq 0 ]; then
+            success=0
+            break;
+        fi
+        sleep $retry_interval
+    done
     # The container process exits and the health check status becomes unhealthy
-    [[ $(isula inspect -f '{{.State.Health.Status}}' ${container_name}) == "unhealthy" ]]
-    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
+    [[ $success -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container health check status: not unhealthy" && ((ret++))
 
     [[ $(isula inspect -f '{{.State.ExitCode}}' ${container_name}) == "137" ]]
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} -  incorrent container exit code: not 137" && ((ret++))
-- 
2.40.1