From 24bec3c29babce28ea1a85f571ecdd97a6ffc1b0 Mon Sep 17 00:00:00 2001 From: jikai Date: Tue, 2 Apr 2024 11:50:19 +0000 Subject: [PATCH] upgrade from upstream Signed-off-by: jikai --- 0001-code-improve-for-sandbox.cc.patch | 4 +- ...r-with-protobuf-25.1-and-grpc-1.60.x.patch | 4 +- ...-point-remains-under-special-circums.patch | 4 +- ...anup-if-the-directory-does-not-exist.patch | 4 +- ...letes-the-temporary-files-it-creates.patch | 4 +- 0006-skip-devmapper-ut.patch | 4 +- ...-update-annotations-and-add-ci-cases.patch | 4 +- ...x-for-device-cgroup-ulimt-oci-update.patch | 4 +- 0009-improve-dt-for-oci-spec-update.patch | 4 +- ...n-container-with-dev-volume-testcase.patch | 4 +- ...add-cpu-usage-nano-cores-for-sandbox.patch | 4 +- ...in-ServiceWorkThread-to-prevent-the-.patch | 4 +- 0013-restore-name-for-rename-failed.patch | 4 +- ...ulad-to-pull-load-image-with-symlink.patch | 4 +- ...lace-http-parser-dependency-with-lcr.patch | 4 +- ...led-log-information-for-load-sandbox.patch | 4 +- ...oncurrency-competition-between-the-r.patch | 4 +- 0018-add-concurrent-load-test.patch | 4 +- ...-of-the-host-path-for-archive-when-c.patch | 4 +- 0020-bugfix-for-wrong-goto-branch.patch | 4 +- ...wrong-dynamic-allocation-object-type.patch | 4 +- 0022-add-swap-usage-in-cri.patch | 4 +- ...benchmark-result-of-perf-test-in-cri.patch | 4 +- ...dd-support-for-systemd-cgroup-driver.patch | 4 +- ...d-ci-cases-for-systemd-cgroup-driver.patch | 4 +- ...stemd_cgroup-CI-test-to-manual-cases.patch | 4 +- ...re-add-support-for-cgroup-v2-metrics.patch | 4 +- ...ervisor-to-notify-sandbox-exit-event.patch | 4 +- 0029-refactor-cgroup-module.patch | 4 +- ...-adaptor-unit-test-for-cgroup-module.patch | 4 +- ...ot-support-isulad-setting-cpu_rt-opt.patch | 4 +- ...ulad-cannot-set-cpu_rt-parameters-wh.patch | 4 +- ...x-container-bool-value-uninitialized.patch | 4 +- 0034-bugfix-for-cpurt.sh.patch | 4 +- ...om-killed-event-and-update-to-cri-of.patch | 868 ++++++ 0036-add-ci-cases-for-oomkilled-monitor.patch | 279 ++ 0037-add-cgroup-v2-doc.patch | 252 ++ ...n-ubuntu-container-bug-in-inspect.sh.patch | 27 + 0039-add-support-for-GetContainerEvents.patch | 2601 +++++++++++++++++ ...ix-cpurt-init-bug-for-systemd-cgroup.patch | 74 + 0041-fix-message-queue-concurrent-bug.patch | 41 + ...cify-runtime-as-runc-for-oom-test-CI.patch | 26 + 0043-set-oomkilled-in-cri.patch | 27 + iSulad.spec | 17 +- 44 files changed, 4279 insertions(+), 69 deletions(-) create mode 100644 0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch create mode 100644 0036-add-ci-cases-for-oomkilled-monitor.patch create mode 100644 0037-add-cgroup-v2-doc.patch create mode 100644 0038-fix-run-ubuntu-container-bug-in-inspect.sh.patch create mode 100644 0039-add-support-for-GetContainerEvents.patch create mode 100644 0040-fix-cpurt-init-bug-for-systemd-cgroup.patch create mode 100644 0041-fix-message-queue-concurrent-bug.patch create mode 100644 0042-specify-runtime-as-runc-for-oom-test-CI.patch create mode 100644 0043-set-oomkilled-in-cri.patch diff --git a/0001-code-improve-for-sandbox.cc.patch b/0001-code-improve-for-sandbox.cc.patch index 8b3d678..f804203 100644 --- a/0001-code-improve-for-sandbox.cc.patch +++ b/0001-code-improve-for-sandbox.cc.patch @@ -1,7 +1,7 @@ From 9497e03709a035805effd96eaa21f6c221a79e94 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Fri, 19 Jan 2024 17:12:30 +0800 -Subject: [PATCH 1/6] code improve for sandbox.cc +Subject: [PATCH 01/43] code improve for sandbox.cc Signed-off-by: zhongtao --- @@ -23,5 +23,5 @@ index 359cfbad..7b6496ed 100644 ERROR("Failed to get sandbox config json for sandbox: '%s'", m_id.c_str()); } -- -2.25.1 +2.34.1 diff --git a/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch index beebaaf..f43ce86 100644 --- a/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch +++ b/0002-fix-compile-error-with-protobuf-25.1-and-grpc-1.60.x.patch @@ -1,7 +1,7 @@ From 71f8d4accbec5153b362281bbaf9a516ccd083f5 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Mon, 22 Jan 2024 15:55:16 +0800 -Subject: [PATCH 2/6] fix compile error with protobuf 25.1 and grpc 1.60.x +Subject: [PATCH 02/43] fix compile error with protobuf 25.1 and grpc 1.60.x Signed-off-by: zhongtao --- @@ -251,5 +251,5 @@ index f43b0f97..5a7cb2ea 100644 add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) -- -2.25.1 +2.34.1 diff --git a/0003-bugfix-for-mount-point-remains-under-special-circums.patch b/0003-bugfix-for-mount-point-remains-under-special-circums.patch index c9e2442..4465d41 100644 --- a/0003-bugfix-for-mount-point-remains-under-special-circums.patch +++ b/0003-bugfix-for-mount-point-remains-under-special-circums.patch @@ -1,7 +1,7 @@ From cd018d3c1ebff2a328912d99fc43c9a7e4f60704 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Thu, 25 Jan 2024 11:24:59 +0800 -Subject: [PATCH 3/6] bugfix for mount point remains under special +Subject: [PATCH 03/43] bugfix for mount point remains under special circumstances Signed-off-by: zhongtao @@ -119,5 +119,5 @@ index 0a7309c9..e4c302bc 100644 } -- -2.25.1 +2.34.1 diff --git a/0004-do-not-cleanup-if-the-directory-does-not-exist.patch b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch index e9feb7a..d643d8e 100644 --- a/0004-do-not-cleanup-if-the-directory-does-not-exist.patch +++ b/0004-do-not-cleanup-if-the-directory-does-not-exist.patch @@ -1,7 +1,7 @@ From 7f13d95572040d30b70edbfac3c4b7350ee8855c Mon Sep 17 00:00:00 2001 From: zhongtao Date: Fri, 26 Jan 2024 12:59:45 +0800 -Subject: [PATCH 4/6] do not cleanup if the directory does not exist +Subject: [PATCH 04/43] do not cleanup if the directory does not exist Signed-off-by: zhongtao --- @@ -67,5 +67,5 @@ index 08151f42..16dba630 100644 return; } -- -2.25.1 +2.34.1 diff --git a/0005-module-only-deletes-the-temporary-files-it-creates.patch b/0005-module-only-deletes-the-temporary-files-it-creates.patch index bc0b096..50e6515 100644 --- a/0005-module-only-deletes-the-temporary-files-it-creates.patch +++ b/0005-module-only-deletes-the-temporary-files-it-creates.patch @@ -1,7 +1,7 @@ From 69dcd191afbdea5a178fb96a21e28537c2fc6a75 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Sat, 27 Jan 2024 11:16:37 +0800 -Subject: [PATCH 5/6] module only deletes the temporary files it creates +Subject: [PATCH 05/43] module only deletes the temporary files it creates Signed-off-by: zhongtao --- @@ -147,5 +147,5 @@ index 751a8727..aed3057a 100644 ERROR("image tmp work path too long"); ret = -1; -- -2.25.1 +2.34.1 diff --git a/0006-skip-devmapper-ut.patch b/0006-skip-devmapper-ut.patch index 6374132..912b3c7 100644 --- a/0006-skip-devmapper-ut.patch +++ b/0006-skip-devmapper-ut.patch @@ -1,7 +1,7 @@ From b290e7fb553c5cc6746c9dcfe4896098f74bc7d7 Mon Sep 17 00:00:00 2001 From: jikai Date: Tue, 30 Jan 2024 12:35:58 +0800 -Subject: [PATCH 6/6] skip devmapper ut +Subject: [PATCH 06/43] skip devmapper ut Signed-off-by: jikai --- @@ -22,5 +22,5 @@ index c1d26ff1..9bb984cd 100755 # build fuzz -- -2.25.1 +2.34.1 diff --git a/0007-update-annotations-and-add-ci-cases.patch b/0007-update-annotations-and-add-ci-cases.patch index 65a550e..07b857e 100644 --- a/0007-update-annotations-and-add-ci-cases.patch +++ b/0007-update-annotations-and-add-ci-cases.patch @@ -1,7 +1,7 @@ From ed4b71b2027a6e9fdf15931fe93aa9e0bb3dc79d Mon Sep 17 00:00:00 2001 From: leizhongkai Date: Wed, 31 Jan 2024 18:17:52 +0800 -Subject: [PATCH 07/26] update annotations and add ci cases +Subject: [PATCH 07/43] update annotations and add ci cases Signed-off-by: leizhongkai --- @@ -170,5 +170,5 @@ index cc49d85f..62e340b1 100644 +} + -- -2.25.1 +2.34.1 diff --git a/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch index 76c6fab..d433c46 100644 --- a/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch +++ b/0008-bug-fix-for-device-cgroup-ulimt-oci-update.patch @@ -1,7 +1,7 @@ From fe3413bb8ebae90f29ce3cc02373f3fc2b5d2fd2 Mon Sep 17 00:00:00 2001 From: jikai Date: Mon, 22 Jan 2024 20:19:29 +0800 -Subject: [PATCH 08/26] bug fix for device/cgroup/ulimt oci update +Subject: [PATCH 08/43] bug fix for device/cgroup/ulimt oci update Signed-off-by: jikai --- @@ -180,5 +180,5 @@ index 62e340b1..464b4fb4 100644 oci_runtime_spec *load_oci_config(const char *rootpath, const char *name) { -- -2.25.1 +2.34.1 diff --git a/0009-improve-dt-for-oci-spec-update.patch b/0009-improve-dt-for-oci-spec-update.patch index b383bf9..07c4b82 100644 --- a/0009-improve-dt-for-oci-spec-update.patch +++ b/0009-improve-dt-for-oci-spec-update.patch @@ -1,7 +1,7 @@ From 82dd5a1db70fdb3f4934a3f9c0ee290ce5bee1b2 Mon Sep 17 00:00:00 2001 From: jikai Date: Sat, 27 Jan 2024 15:30:05 +0800 -Subject: [PATCH 09/26] improve dt for oci spec update +Subject: [PATCH 09/43] improve dt for oci spec update Signed-off-by: jikai --- @@ -277,5 +277,5 @@ index ad903a3f..47e4ca6e 100644 struct capabilities_lens { size_t bounding_len; -- -2.25.1 +2.34.1 diff --git a/0010-open-run-container-with-dev-volume-testcase.patch b/0010-open-run-container-with-dev-volume-testcase.patch index 16f5305..2381125 100644 --- a/0010-open-run-container-with-dev-volume-testcase.patch +++ b/0010-open-run-container-with-dev-volume-testcase.patch @@ -1,7 +1,7 @@ From 44d15a7451a922ca7266b756d3f9a83908199cb3 Mon Sep 17 00:00:00 2001 From: zhangxiaoyu Date: Tue, 23 Jan 2024 10:35:59 +0800 -Subject: [PATCH 10/26] open run container with dev volume testcase +Subject: [PATCH 10/43] open run container with dev volume testcase Signed-off-by: zhangxiaoyu --- @@ -26,5 +26,5 @@ index 545d5099..04bf437a 100755 [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to run container with image: ${image}" && ((ret++)) -- -2.25.1 +2.34.1 diff --git a/0011-add-cpu-usage-nano-cores-for-sandbox.patch b/0011-add-cpu-usage-nano-cores-for-sandbox.patch index 4c5b2eb..27be05d 100644 --- a/0011-add-cpu-usage-nano-cores-for-sandbox.patch +++ b/0011-add-cpu-usage-nano-cores-for-sandbox.patch @@ -1,7 +1,7 @@ From 3dc12d7806fda8d5ceee183595e993079bee4056 Mon Sep 17 00:00:00 2001 From: jikai Date: Fri, 12 Jan 2024 17:38:09 +0800 -Subject: [PATCH 11/26] add cpu usage nano cores for sandbox +Subject: [PATCH 11/43] add cpu usage nano cores for sandbox Signed-off-by: jikai --- @@ -77,5 +77,5 @@ index 2bd28007..c3d98b8c 100644 void GetFilterPodSandbox(const runtime::v1::PodSandboxStatsFilter *filter, std::vector &podSandboxIDs, Errors &error); -- -2.25.1 +2.34.1 diff --git a/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch index 9da5648..9d70f92 100644 --- a/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch +++ b/0012-sleep-some-time-in-ServiceWorkThread-to-prevent-the-.patch @@ -1,7 +1,7 @@ From 384cf7870c155d41f742b1928a4cb1b56aa46c94 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 6 Feb 2024 20:05:05 +0800 -Subject: [PATCH 12/26] sleep some time in ServiceWorkThread to prevent the CPU +Subject: [PATCH 12/43] sleep some time in ServiceWorkThread to prevent the CPU from being occupied all the time Signed-off-by: zhongtao @@ -23,5 +23,5 @@ index 6319a67f..a8d89b36 100644 } -- -2.25.1 +2.34.1 diff --git a/0013-restore-name-for-rename-failed.patch b/0013-restore-name-for-rename-failed.patch index b6966ff..be7d59c 100644 --- a/0013-restore-name-for-rename-failed.patch +++ b/0013-restore-name-for-rename-failed.patch @@ -1,7 +1,7 @@ From 2df7a67ad2cb0249b18ca5eba46f9aab8f72038f Mon Sep 17 00:00:00 2001 From: zhongtao Date: Sun, 18 Feb 2024 11:32:55 +0800 -Subject: [PATCH 13/26] restore name for rename failed +Subject: [PATCH 13/43] restore name for rename failed Signed-off-by: zhongtao --- @@ -25,5 +25,5 @@ index 2a71e82a..c02cc830 100644 static int container_rename(container_t *cont, const char *new_name) -- -2.25.1 +2.34.1 diff --git a/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch index 122729d..4a1fa1f 100644 --- a/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch +++ b/0014-2371-Allow-iSulad-to-pull-load-image-with-symlink.patch @@ -1,7 +1,7 @@ From fd4c80b8de768d7132cef0720cd46167173a653b Mon Sep 17 00:00:00 2001 From: xuxuepeng Date: Mon, 19 Feb 2024 01:05:18 +0000 -Subject: [PATCH 14/26] !2371 Allow iSulad to pull/load image with symlink * +Subject: [PATCH 14/43] !2371 Allow iSulad to pull/load image with symlink * Allow iSulad to pull/load image with symlink --- @@ -41,5 +41,5 @@ index e4c302bc..52b51162 100644 a = archive_read_new(); if (a == NULL) { -- -2.25.1 +2.34.1 diff --git a/0015-Replace-http-parser-dependency-with-lcr.patch b/0015-Replace-http-parser-dependency-with-lcr.patch index 5d46b1b..eccb44e 100644 --- a/0015-Replace-http-parser-dependency-with-lcr.patch +++ b/0015-Replace-http-parser-dependency-with-lcr.patch @@ -1,7 +1,7 @@ From fb76605985166c4d2172270c8d633ed26d62f698 Mon Sep 17 00:00:00 2001 From: xuxuepeng Date: Mon, 19 Feb 2024 23:52:47 +0800 -Subject: [PATCH 15/26] Replace http-parser dependency with lcr +Subject: [PATCH 15/43] Replace http-parser dependency with lcr Signed-off-by: xuxuepeng --- @@ -423,5 +423,5 @@ index 885375f2..d851ba96 100644 #include "isula_libutils/log.h" -- -2.25.1 +2.34.1 diff --git a/0016-add-more-detailed-log-information-for-load-sandbox.patch b/0016-add-more-detailed-log-information-for-load-sandbox.patch index 6d6b985..db8bf17 100644 --- a/0016-add-more-detailed-log-information-for-load-sandbox.patch +++ b/0016-add-more-detailed-log-information-for-load-sandbox.patch @@ -1,7 +1,7 @@ From e4facfcd2947b5277789d58a452090b61ca2d383 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Wed, 21 Feb 2024 15:04:27 +0800 -Subject: [PATCH 16/26] add more detailed log information for load sandbox +Subject: [PATCH 16/43] add more detailed log information for load sandbox Signed-off-by: zhongtao --- @@ -22,5 +22,5 @@ index d3db4fb4..cee444f4 100644 } -- -2.25.1 +2.34.1 diff --git a/0017-bugfix-for-the-concurrency-competition-between-the-r.patch b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch index 42e0435..796370b 100644 --- a/0017-bugfix-for-the-concurrency-competition-between-the-r.patch +++ b/0017-bugfix-for-the-concurrency-competition-between-the-r.patch @@ -1,7 +1,7 @@ From 0099190e7f18e890185e36c5a657e9ce95179bc8 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Fri, 1 Mar 2024 15:04:09 +0800 -Subject: [PATCH 17/26] bugfix for the concurrency competition between the +Subject: [PATCH 17/43] bugfix for the concurrency competition between the reuse layer and the creation layer Signed-off-by: zhongtao @@ -79,5 +79,5 @@ index aed3057a..66fa0076 100644 // parent_chain_id = NULL means no parent chain match from now on, so no longer need // to get layers by compressed digest to reuse layer. -- -2.25.1 +2.34.1 diff --git a/0018-add-concurrent-load-test.patch b/0018-add-concurrent-load-test.patch index 012fc8d..2256aba 100644 --- a/0018-add-concurrent-load-test.patch +++ b/0018-add-concurrent-load-test.patch @@ -1,7 +1,7 @@ From 2af906d42a155a7b779dce017a2779b96dba2b61 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Fri, 1 Mar 2024 15:04:35 +0800 -Subject: [PATCH 18/26] add concurrent load test +Subject: [PATCH 18/43] add concurrent load test Signed-off-by: zhongtao --- @@ -69,5 +69,5 @@ index 52b713d4..a2cada5f 100755 show_result ${ans} "${curr_path}/${0}" -- -2.25.1 +2.34.1 diff --git a/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch index 4ec6b4a..e1a2fda 100644 --- a/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch +++ b/0019-get-the-realpath-of-the-host-path-for-archive-when-c.patch @@ -1,7 +1,7 @@ From 96dfd32ee5d9a133ad63af13723402f10cd7cf7b Mon Sep 17 00:00:00 2001 From: zhongtao Date: Mon, 11 Mar 2024 15:50:45 +0800 -Subject: [PATCH 19/26] get the realpath of the host path for archive when cp +Subject: [PATCH 19/43] get the realpath of the host path for archive when cp Signed-off-by: zhongtao --- @@ -59,5 +59,5 @@ index bbe4c3b2..fe514acc 100644 ERROR("Can not archive path: %s", path); goto cleanup; -- -2.25.1 +2.34.1 diff --git a/0020-bugfix-for-wrong-goto-branch.patch b/0020-bugfix-for-wrong-goto-branch.patch index 6cf7505..7c8318c 100644 --- a/0020-bugfix-for-wrong-goto-branch.patch +++ b/0020-bugfix-for-wrong-goto-branch.patch @@ -1,7 +1,7 @@ From c67760ce928f67d9a8beeaf2e2d51c8f2239f69e Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 12 Mar 2024 11:15:26 +0800 -Subject: [PATCH 20/26] bugfix for wrong goto branch +Subject: [PATCH 20/43] bugfix for wrong goto branch Signed-off-by: zhongtao --- @@ -31,5 +31,5 @@ index fe514acc..13343922 100644 DEBUG("chroot tar stream srcdir(%s) srcbase(%s) rebase(%s)", srcdir, srcbase, rebase); -- -2.25.1 +2.34.1 diff --git a/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch index aa9476c..bb08db6 100644 --- a/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch +++ b/0021-bugfix-for-wrong-dynamic-allocation-object-type.patch @@ -1,7 +1,7 @@ From 0ef23c6caae4a97228705574b0c8f3445c6e65dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E7=BA=A2=E5=BC=BA?= <277922995@qq.com> Date: Wed, 13 Mar 2024 17:00:16 +0800 -Subject: [PATCH 21/26] bugfix for wrong dynamic allocation object type +Subject: [PATCH 21/43] bugfix for wrong dynamic allocation object type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @@ -25,5 +25,5 @@ index e7aa81b8..871f5f39 100644 ERROR("Out of memory"); return -1; -- -2.25.1 +2.34.1 diff --git a/0022-add-swap-usage-in-cri.patch b/0022-add-swap-usage-in-cri.patch index f6218e7..cb773e6 100644 --- a/0022-add-swap-usage-in-cri.patch +++ b/0022-add-swap-usage-in-cri.patch @@ -1,7 +1,7 @@ From ed569ccbf7e5029e83c40521255e0e406f285bae Mon Sep 17 00:00:00 2001 From: jikai Date: Fri, 12 Jan 2024 11:31:59 +0800 -Subject: [PATCH 22/26] add swap usage in cri +Subject: [PATCH 22/43] add swap usage in cri Signed-off-by: jikai --- @@ -76,5 +76,5 @@ index 2c92cc59..745154bb 100644 if (blkio == NULL) { return; -- -2.25.1 +2.34.1 diff --git a/0023-add-benchmark-result-of-perf-test-in-cri.patch b/0023-add-benchmark-result-of-perf-test-in-cri.patch index 00a2f0a..92efaeb 100644 --- a/0023-add-benchmark-result-of-perf-test-in-cri.patch +++ b/0023-add-benchmark-result-of-perf-test-in-cri.patch @@ -1,7 +1,7 @@ From 16a0cf7e9c2c059cb5537f48a022e63df457f186 Mon Sep 17 00:00:00 2001 From: jikai Date: Sat, 2 Mar 2024 11:49:08 +0800 -Subject: [PATCH 23/26] add benchmark, result of perf test in cri +Subject: [PATCH 23/43] add benchmark, result of perf test in cri Signed-off-by: jikai --- @@ -252,5 +252,5 @@ index 00000000..54ee24f5 +crictl --runtime-endpoint $runtime rmp -af +rm -rf $tmpdir -- -2.25.1 +2.34.1 diff --git a/0024-add-support-for-systemd-cgroup-driver.patch b/0024-add-support-for-systemd-cgroup-driver.patch index 5ef3187..d6c363b 100644 --- a/0024-add-support-for-systemd-cgroup-driver.patch +++ b/0024-add-support-for-systemd-cgroup-driver.patch @@ -1,7 +1,7 @@ From 167af3ce0cff3906c9976b249432d41167b15eb2 Mon Sep 17 00:00:00 2001 From: jikai Date: Thu, 11 Jan 2024 17:06:57 +0800 -Subject: [PATCH 24/26] add support for systemd cgroup driver +Subject: [PATCH 24/43] add support for systemd cgroup driver Signed-off-by: jikai --- @@ -519,5 +519,5 @@ index 464b4fb4..b4d2b0f6 100644 } -- -2.25.1 +2.34.1 diff --git a/0025-add-ci-cases-for-systemd-cgroup-driver.patch b/0025-add-ci-cases-for-systemd-cgroup-driver.patch index dfbfc18..2d2ac71 100644 --- a/0025-add-ci-cases-for-systemd-cgroup-driver.patch +++ b/0025-add-ci-cases-for-systemd-cgroup-driver.patch @@ -1,7 +1,7 @@ From f5f100f5b244be2debebe815aaed3afad8950daf Mon Sep 17 00:00:00 2001 From: jikai Date: Tue, 6 Feb 2024 17:33:17 +0800 -Subject: [PATCH 25/26] add ci cases for systemd cgroup driver +Subject: [PATCH 25/43] add ci cases for systemd cgroup driver Signed-off-by: jikai --- @@ -215,5 +215,5 @@ index 47e4ca6e..6c42216d 100644 { parser_error err = nullptr; -- -2.25.1 +2.34.1 diff --git a/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch index 7f6b431..ea60213 100644 --- a/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch +++ b/0026-move-systemd_cgroup-CI-test-to-manual-cases.patch @@ -1,7 +1,7 @@ From b93647205db5c4a5d74fb245c9b1e15ca1ffd3fe Mon Sep 17 00:00:00 2001 From: jikai Date: Sat, 16 Mar 2024 09:35:22 +0800 -Subject: [PATCH 26/26] move systemd_cgroup CI test to manual cases +Subject: [PATCH 26/43] move systemd_cgroup CI test to manual cases Signed-off-by: jikai --- @@ -14,5 +14,5 @@ similarity index 100% rename from CI/test_cases/container_cases/systemd_cgroup.sh rename to CI/test_cases/manual_cases/systemd_cgroup.sh -- -2.25.1 +2.34.1 diff --git a/0027-feature-add-support-for-cgroup-v2-metrics.patch b/0027-feature-add-support-for-cgroup-v2-metrics.patch index 0db5ef4..f5b4073 100644 --- a/0027-feature-add-support-for-cgroup-v2-metrics.patch +++ b/0027-feature-add-support-for-cgroup-v2-metrics.patch @@ -1,7 +1,7 @@ From 7c7cd82619ed1f7e36d34da1afc2b417a90b3040 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Tue, 23 Jan 2024 17:18:51 +0800 -Subject: [PATCH 27/34] =?UTF-8?q?=E3=80=90feature=E3=80=91add=20support=20?= +Subject: [PATCH 27/43] =?UTF-8?q?=E3=80=90feature=E3=80=91add=20support=20?= =?UTF-8?q?for=20cgroup=20v2=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 @@ -1080,5 +1080,5 @@ index 64f41496..1c084595 100644 } #endif -- -2.25.1 +2.34.1 diff --git a/0028-use-supervisor-to-notify-sandbox-exit-event.patch b/0028-use-supervisor-to-notify-sandbox-exit-event.patch index e139bbb..379eb90 100644 --- a/0028-use-supervisor-to-notify-sandbox-exit-event.patch +++ b/0028-use-supervisor-to-notify-sandbox-exit-event.patch @@ -1,7 +1,7 @@ From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001 From: jikai Date: Wed, 20 Mar 2024 15:48:42 +0800 -Subject: [PATCH 28/34] use supervisor to notify sandbox exit event +Subject: [PATCH 28/43] use supervisor to notify sandbox exit event Signed-off-by: jikai --- @@ -250,5 +250,5 @@ index bef884fb..8189efd6 100644 } #endif -- -2.25.1 +2.34.1 diff --git a/0029-refactor-cgroup-module.patch b/0029-refactor-cgroup-module.patch index 2b93c7b..f5e853d 100644 --- a/0029-refactor-cgroup-module.patch +++ b/0029-refactor-cgroup-module.patch @@ -1,7 +1,7 @@ From c26604ff3150babae729890c549f2784212073a1 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Wed, 20 Mar 2024 15:53:56 +0800 -Subject: [PATCH 29/34] refactor cgroup module +Subject: [PATCH 29/43] refactor cgroup module Signed-off-by: zhongtao --- @@ -2534,5 +2534,5 @@ index 7ed8e837..88c6b354 100644 ERROR("Failed to get cpu rt controller mnt root path"); return -1; -- -2.25.1 +2.34.1 diff --git a/0030-adaptor-unit-test-for-cgroup-module.patch b/0030-adaptor-unit-test-for-cgroup-module.patch index 3a816ad..66fd4dd 100644 --- a/0030-adaptor-unit-test-for-cgroup-module.patch +++ b/0030-adaptor-unit-test-for-cgroup-module.patch @@ -1,7 +1,7 @@ From 59e7ea0f16e83e0bdbc39bdc41d1ade8d3db885e Mon Sep 17 00:00:00 2001 From: zhongtao Date: Thu, 22 Feb 2024 09:52:30 +0800 -Subject: [PATCH 30/34] adaptor unit test for cgroup module +Subject: [PATCH 30/43] adaptor unit test for cgroup module Signed-off-by: zhongtao --- @@ -347,5 +347,5 @@ index cc309352..27d07330 100644 ${CMAKE_BINARY_DIR}/conf ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/sha256 -- -2.25.1 +2.34.1 diff --git a/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch index ba3a9a5..ecca55a 100644 --- a/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch +++ b/0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch @@ -1,7 +1,7 @@ From 8e11a1eea62cb8061f1613379ff83bd9a721fa50 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Wed, 31 Jan 2024 18:10:46 +0800 -Subject: [PATCH 31/34] cgroup v2 does not support isulad setting cpu_rt +Subject: [PATCH 31/43] cgroup v2 does not support isulad setting cpu_rt options Signed-off-by: zhongtao @@ -67,5 +67,5 @@ index 5fb55689..619e36d1 100644 return ret; } -- -2.25.1 +2.34.1 diff --git a/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch index f059de1..1568edb 100644 --- a/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch +++ b/0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch @@ -1,7 +1,7 @@ From 1ab0f4608fb749b50aa6f8d8188db23aa8a6e1ac Mon Sep 17 00:00:00 2001 From: zhongtao Date: Thu, 1 Feb 2024 10:48:45 +0800 -Subject: [PATCH 32/34] add test that isulad cannot set cpu_rt parameters when +Subject: [PATCH 32/43] add test that isulad cannot set cpu_rt parameters when adding cgroup v2 Signed-off-by: zhongtao @@ -26,5 +26,5 @@ index bdc43a5e..23d3baed 100755 isula pull ${image} [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${image}" && return ${FAILURE} -- -2.25.1 +2.34.1 diff --git a/0033-fix-sandbox-container-bool-value-uninitialized.patch b/0033-fix-sandbox-container-bool-value-uninitialized.patch index 52ad07d..2f88424 100644 --- a/0033-fix-sandbox-container-bool-value-uninitialized.patch +++ b/0033-fix-sandbox-container-bool-value-uninitialized.patch @@ -1,7 +1,7 @@ From f62df3dedbbe11bb56e6da7dd610c573fd3ed828 Mon Sep 17 00:00:00 2001 From: jikai Date: Mon, 25 Mar 2024 10:01:56 +0800 -Subject: [PATCH 33/34] fix sandbox container bool value uninitialized +Subject: [PATCH 33/43] fix sandbox container bool value uninitialized Signed-off-by: jikai --- @@ -22,5 +22,5 @@ index 7b34cc7f..a8090d5a 100644 nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); if (nret < 0 || (size_t)nret >= sizeof(bundle)) { -- -2.25.1 +2.34.1 diff --git a/0034-bugfix-for-cpurt.sh.patch b/0034-bugfix-for-cpurt.sh.patch index 13db455..d8b52bb 100644 --- a/0034-bugfix-for-cpurt.sh.patch +++ b/0034-bugfix-for-cpurt.sh.patch @@ -1,7 +1,7 @@ From 411483ad9b2a0c50190f9b56779d41889c895014 Mon Sep 17 00:00:00 2001 From: zhongtao Date: Wed, 27 Mar 2024 10:29:11 +0800 -Subject: [PATCH 34/34] bugfix for cpurt.sh +Subject: [PATCH 34/43] bugfix for cpurt.sh Signed-off-by: zhongtao --- @@ -43,5 +43,5 @@ index 23d3baed..64dcd81f 100755 done -- -2.25.1 +2.34.1 diff --git a/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch new file mode 100644 index 0000000..85f7f62 --- /dev/null +++ b/0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch @@ -0,0 +1,868 @@ +From 947cf87a87ec49409ae509e5142b8134454d1547 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 28 Mar 2024 12:51:09 +0000 +Subject: [PATCH 35/43] monitor cgroup oom killed event and update to cri of + container + +Signed-off-by: jikai +--- + src/daemon/common/cgroup/cgroup.c | 91 +++++++++- + src/daemon/common/cgroup/cgroup.h | 5 + + src/daemon/common/cgroup/cgroup_common.h | 13 ++ + src/daemon/common/cgroup/cgroup_v1.c | 160 ++++++++++++++++++ + src/daemon/common/cgroup/cgroup_v2.c | 138 ++++++++++++++- + .../v1/v1_cri_container_manager_service.cc | 3 + + src/daemon/modules/api/container_api.h | 5 +- + .../container/container_events_handler.c | 12 +- + .../modules/container/container_state.c | 15 ++ + .../modules/container/restore/restore.c | 10 +- + .../modules/container/supervisor/supervisor.c | 54 +++++- + src/daemon/modules/events/collector.c | 7 +- + .../modules/service/service_container.c | 11 +- + 13 files changed, 498 insertions(+), 26 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index 837b514a..d3f1445a 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -133,4 +133,93 @@ char *common_get_own_cgroup_path(const char *subsystem) + } + + return g_cgroup_ops.get_own_cgroup_path(subsystem); +-} +\ No newline at end of file ++} ++ ++char *common_convert_cgroup_path(const char *cgroup_path) ++{ ++ char *token = NULL; ++ char result[PATH_MAX + 1] = {0}; ++ __isula_auto_array_t char **arr = NULL; ++ ++ if (cgroup_path == NULL) { ++ ERROR("Invalid NULL cgroup path"); ++ return NULL; ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(cgroup_path, ".slice")) { ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for systemd cgroup, cgroup_path should have the form slice:prefix:id, ++ // convert it to a true path, such as from test-a.slice:isulad:id ++ // to test.slice/test-a.slice/isulad-id.scope ++ arr = util_string_split_n(cgroup_path, ':', 3); ++ if (arr == NULL || util_array_len((const char **)arr) != 3) { ++ ERROR("Invalid systemd cgroup parent"); ++ return NULL; ++ } ++ ++ token = strchr(arr[0], '-'); ++ while (token != NULL) { ++ *token = '\0'; ++ if (strlen(arr[0]) > PATH_MAX || strlen(result) + 1 + strlen(".slice") > ++ PATH_MAX - strlen(arr[0])) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ *token = '-'; ++ return NULL; ++ } ++ if (result[0] != '\0') { ++ strcat(result, "/"); ++ } ++ strcat(result, arr[0]); ++ strcat(result, ".slice"); ++ *token = '-'; ++ token = strchr(token + 1, '-'); ++ } ++ ++ // Add /arr[0]/arr[1]-arr[2].scope, 3 include two slashes and one dash ++ if (strlen(cgroup_path) > PATH_MAX || strlen(result) + 3 + strlen(".scope") > ++ PATH_MAX - strlen(arr[0] - strlen(arr[1]) - strlen(arr[2]))) { ++ ERROR("Invalid systemd cgroup parent: exceeds max length of path"); ++ return NULL; ++ } ++ ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[0]); ++ (void)strcat(result, "/"); ++ (void)strcat(result, arr[1]); ++ (void)strcat(result, "-"); ++ (void)strcat(result, arr[2]); ++ (void)strcat(result, ".scope"); ++ ++ return util_strdup_s(result); ++} ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ if (g_cgroup_ops.get_cgroup_oom_handler == NULL) { ++ ERROR("Unimplmented get_cgroup_oom_handler op"); ++ return NULL; ++ } ++ ++ return g_cgroup_ops.get_cgroup_oom_handler(fd, name, cgroup_path, exit_fifo); ++} ++ ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info) ++{ ++ if (info == NULL) { ++ return; ++ } ++ ++ if (info->oom_event_fd >= 0) { ++ close(info->oom_event_fd); ++ } ++ if (info->cgroup_file_fd >= 0) { ++ close(info->cgroup_file_fd); ++ } ++ ++ free(info->name); ++ free(info->cgroup_memory_event_path); ++ free(info); ++} +diff --git a/src/daemon/common/cgroup/cgroup.h b/src/daemon/common/cgroup/cgroup.h +index 1efc3ca6..8c76d99d 100644 +--- a/src/daemon/common/cgroup/cgroup.h ++++ b/src/daemon/common/cgroup/cgroup.h +@@ -41,6 +41,11 @@ int common_get_cgroup_mnt_and_root_path(const char *subsystem, char **mountpoint + char *common_get_init_cgroup_path(const char *subsystem); + char *common_get_own_cgroup_path(const char *subsystem); + ++char *common_convert_cgroup_path(const char *cgroup_path); ++ ++cgroup_oom_handler_info_t *common_get_cgroup_oom_handler(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); ++void common_free_cgroup_oom_handler_info(cgroup_oom_handler_info_t *info); ++ + #ifdef __cplusplus + } + #endif +diff --git a/src/daemon/common/cgroup/cgroup_common.h b/src/daemon/common/cgroup/cgroup_common.h +index 2a0935cb..e3912bf0 100644 +--- a/src/daemon/common/cgroup/cgroup_common.h ++++ b/src/daemon/common/cgroup/cgroup_common.h +@@ -116,6 +116,17 @@ typedef struct { + cgroup_pids_metrics_t cgpids_metrics; + } cgroup_metrics_t; + ++#define CGROUP_OOM_HANDLE_CONTINUE false ++#define CGROUP_OOM_HANDLE_CLOSE true ++ ++typedef struct _cgroup_oom_handler_info_t { ++ int oom_event_fd; ++ int cgroup_file_fd; ++ char *name; ++ char *cgroup_memory_event_path; ++ bool (*oom_event_handler)(int, void *); ++} cgroup_oom_handler_info_t; ++ + typedef struct { + int (*get_cgroup_version)(void); + int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, +@@ -128,6 +139,8 @@ typedef struct { + + char *(*get_init_cgroup_path)(const char *subsystem); + char *(*get_own_cgroup_path)(const char *subsystem); ++ ++ cgroup_oom_handler_info_t *(*get_cgroup_oom_handler)(int fd, const char *name, const char *cgroup_path, const char *exit_fifo); + } cgroup_ops; + + #ifdef __cplusplus +diff --git a/src/daemon/common/cgroup/cgroup_v1.c b/src/daemon/common/cgroup/cgroup_v1.c +index 51cf7512..41f3110a 100644 +--- a/src/daemon/common/cgroup/cgroup_v1.c ++++ b/src/daemon/common/cgroup/cgroup_v1.c +@@ -12,14 +12,20 @@ + * Create: 2023-03-29 + * Description: provide cgroup v1 functions + ******************************************************************************/ ++#ifndef _GNU_SOURCE ++#define _GNU_SOURCE ++#endif ++ + #include "cgroup.h" + + #include + #include ++#include + + #include "utils.h" + #include "sysinfo.h" + #include "err_msg.h" ++#include "events_sender_api.h" + + #define CGROUP_HUGETLB_LIMIT "hugetlb.%s.limit_in_bytes" + #define CGROUP_MOUNT_PATH_PREFIX "/sys/fs/cgroup/" +@@ -1045,6 +1051,159 @@ static char *common_get_cgroup_path(const char *path, const char *subsystem) + return res; + } + ++static bool oom_cb_cgroup_v1(int fd, void *cbdata) ++{ ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ /* Try to read cgroup.event_control and known if the cgroup was removed ++ * if the cgroup was removed and only one event received, ++ * we know that it is a cgroup removal event rather than an oom event ++ */ ++ bool cgroup_removed = false; ++ if (info == NULL) { ++ ERROR("Invalide callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (access(info->cgroup_memory_event_path, F_OK) < 0) { ++ DEBUG("Cgroup event path was removed"); ++ cgroup_removed = true; ++ } ++ ++ uint64_t event_count; ++ ssize_t num_read = util_read_nointr(fd, &event_count, sizeof(uint64_t)); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read == 0) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (num_read != sizeof(uint64_t)) { ++ ERROR("Failed to read full oom event from eventfd"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 0) { ++ ERROR("Unexpected event count when reading for oom event"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (event_count == 1 && cgroup_removed) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ INFO("OOM event detected"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++} ++ ++static char *get_memory_cgroup_path_v1(const char *cgroup_path) ++{ ++ int nret = 0; ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ __isula_auto_free char *mnt = NULL; ++ __isula_auto_free char *root = NULL; ++ char fpath[PATH_MAX] = { 0 }; ++ ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path"); ++ return NULL; ++ } ++ ++ nret = get_cgroup_mnt_and_root_path_v1("memory", &mnt, &root); ++ if (nret != 0 || mnt == NULL || root == NULL) { ++ ERROR("Can not find cgroup mnt and root path for subsystem 'memory'"); ++ return NULL; ++ } ++ ++ // When iSulad is run inside docker, the root is based of the host cgroup. ++ // Replace root to "/" ++ if (strncmp(root, "/docker/", strlen("/docker/")) == 0) { ++ root[1] = '\0'; ++ } ++ ++ nret = snprintf(fpath, sizeof(fpath), "%s/%s", mnt, root); ++ if (nret < 0 || (size_t)nret >= sizeof(fpath)) { ++ ERROR("Failed to print string"); ++ return NULL; ++ } ++ ++ return util_path_join(fpath, converted_cgroup_path); ++} ++ ++static cgroup_oom_handler_info_t *get_cgroup_oom_handler_v1(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *memory_cgroup_path = NULL; ++ __isula_auto_free char *memory_cgroup_oom_control_path = NULL; ++ __isula_auto_free char *data = NULL; ++ __isula_auto_close int cgroup_event_control_fd = -1; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ info->name = util_strdup_s(name); ++ info->cgroup_file_fd = -1; ++ info->oom_event_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v1; ++ ++ memory_cgroup_path = get_memory_cgroup_path_v1(cgroup_path); ++ if (memory_cgroup_path == NULL) { ++ ERROR("Failed to get memory cgroup path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(memory_cgroup_path, "cgroup.event_control"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ cgroup_event_control_fd = util_open(info->cgroup_memory_event_path, O_WRONLY | O_CLOEXEC, 0); ++ if (cgroup_event_control_fd < 0) { ++ ERROR("Failed to open %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ memory_cgroup_oom_control_path = util_path_join(memory_cgroup_path, "memory.oom_control"); ++ if (memory_cgroup_oom_control_path == NULL) { ++ ERROR("Failed to join memory cgroup file path"); ++ goto cleanup; ++ } ++ ++ info->cgroup_file_fd = util_open(memory_cgroup_oom_control_path, O_RDONLY | O_CLOEXEC, 0); ++ if (info->cgroup_file_fd < 0) { ++ ERROR("Failed to open %s", memory_cgroup_oom_control_path); ++ goto cleanup; ++ } ++ ++ info->oom_event_fd = eventfd(0, EFD_CLOEXEC); ++ if (info->oom_event_fd < 0) { ++ ERROR("Failed to create oom eventfd"); ++ goto cleanup; ++ } ++ ++ if (asprintf(&data, "%d %d", info->oom_event_fd, info->cgroup_file_fd) < 0 || ++ util_write_nointr(cgroup_event_control_fd, data, strlen(data)) < 0) { ++ ERROR("Failed to write to cgroup.event_control"); ++ goto cleanup; ++ } ++ ++ return info; ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + char *get_init_cgroup_path_v1(const char *subsystem) + { + return common_get_cgroup_path("/proc/1/cgroup", subsystem); +@@ -1071,5 +1230,6 @@ int cgroup_v1_ops_init(cgroup_ops *ops) + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_path_v1; + ops->get_init_cgroup_path = get_init_cgroup_path_v1; + ops->get_own_cgroup_path = get_own_cgroup_v1; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v1; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/common/cgroup/cgroup_v2.c b/src/daemon/common/cgroup/cgroup_v2.c +index 65cf90d8..a36258f0 100644 +--- a/src/daemon/common/cgroup/cgroup_v2.c ++++ b/src/daemon/common/cgroup/cgroup_v2.c +@@ -17,12 +17,14 @@ + #include + #include + #include ++#include + + #include + + #include "utils.h" + #include "path.h" + #include "sysinfo.h" ++#include "events_sender_api.h" + + // Cgroup V2 Item Definition + #define CGROUP2_CPU_WEIGHT "cpu.weight" +@@ -408,10 +410,143 @@ static int get_cgroup_metrics_v2(const char *cgroup_path, cgroup_metrics_t *cgro + + static int get_cgroup_mnt_and_root_v2(const char *subsystem, char **mountpoint, char **root) + { +- *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ if (mountpoint != NULL) { ++ *mountpoint = util_strdup_s(CGROUP_ISULAD_PATH); ++ } + return 0; + } + ++static bool oom_cb_cgroup_v2(int fd, void *cbdata) ++{ ++ const size_t events_size = sizeof(struct inotify_event) + NAME_MAX + 1; ++ char events[events_size]; ++ cgroup_oom_handler_info_t *info = (cgroup_oom_handler_info_t *)cbdata; ++ ++ if (info == NULL) { ++ ERROR("Invalid callback data"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ ssize_t num_read = util_read_nointr(fd, &events, events_size); ++ if (num_read < 0) { ++ ERROR("Failed to read oom event from eventfd in v2"); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ if (((struct inotify_event *)events)->mask & ( IN_DELETE | IN_DELETE_SELF)) { ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_file FILE *fp = fopen(info->cgroup_memory_event_path, "re"); ++ if (fp == NULL) { ++ ERROR("Failed to open cgroups file: %s", info->cgroup_memory_event_path); ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ __isula_auto_free char *line = NULL; ++ size_t len = 0; ++ ssize_t read; ++ while ((read = getline(&line, &len, fp)) != -1) { ++ int count; ++ const char *oom_str = "oom "; ++ const char *oom_kill_str = "oom_kill "; ++ const int oom_len = strlen(oom_str), oom_kill_len = strlen(oom_kill_str); ++ ++ if (read >= oom_kill_len + 2 && memcmp(line, oom_kill_str, oom_kill_len) == 0) { ++ len = oom_kill_len; ++ } else if (read >= oom_len + 2 && memcmp(line, oom_str, oom_len) == 0) { ++ len = oom_len; ++ } else { ++ continue; ++ } ++ ++ // to make use of util_safe_int, it requires it ends with '\0' ++ line[strcspn(line, "\n")] = '\0'; ++ if (util_safe_int(&line[len], &count) < 0) { ++ ERROR("Failed to parse: %s", &line[len]); ++ continue; ++ } ++ ++ if (count == 0) { ++ continue; ++ } ++ ++ INFO("OOM event detected in cgroup v2"); ++ (void)isulad_monitor_send_container_event(info->name, OOM, -1, 0, NULL, NULL); ++ ++ return CGROUP_OOM_HANDLE_CLOSE; ++ } ++ ++ return CGROUP_OOM_HANDLE_CONTINUE; ++} ++ ++static char *get_real_cgroup_path_v2(const char *cgroup_path) ++{ ++ __isula_auto_free char *converted_cgroup_path = NULL; ++ converted_cgroup_path = common_convert_cgroup_path(cgroup_path); ++ if (converted_cgroup_path == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return NULL; ++ } ++ ++ return util_path_join(CGROUP_MOUNTPOINT, converted_cgroup_path); ++} ++ ++cgroup_oom_handler_info_t *get_cgroup_oom_handler_v2(int fd, const char *name, const char *cgroup_path, const char *exit_fifo) ++{ ++ __isula_auto_free char *real_cgroup_path = NULL; ++ if (name == NULL || cgroup_path == NULL || exit_fifo == NULL) { ++ ERROR("Invalid arguments"); ++ return NULL; ++ } ++ ++ cgroup_oom_handler_info_t *info = util_common_calloc_s(sizeof(cgroup_oom_handler_info_t)); ++ if (info == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ info->name = util_strdup_s(name); ++ info->oom_event_fd = -1; ++ info->cgroup_file_fd = -1; ++ info->oom_event_handler = oom_cb_cgroup_v2; ++ ++ real_cgroup_path = get_real_cgroup_path_v2(cgroup_path); ++ if (real_cgroup_path == NULL) { ++ ERROR("Failed to transfer cgroup path: %s", cgroup_path); ++ goto cleanup; ++ } ++ ++ info->cgroup_memory_event_path = util_path_join(real_cgroup_path, "memory.events"); ++ if (info->cgroup_memory_event_path == NULL) { ++ ERROR("Failed to join path"); ++ goto cleanup; ++ } ++ ++ if ((info->oom_event_fd = inotify_init()) < 0) { ++ ERROR("Failed to init inotify fd"); ++ goto cleanup; ++ } ++ ++ if (inotify_add_watch(info->oom_event_fd, info->cgroup_memory_event_path, IN_MODIFY) < 0) { ++ ERROR("Failed to watch inotify fd for %s", info->cgroup_memory_event_path); ++ goto cleanup; ++ } ++ ++ // watch exit fifo for container exit, so we can close the inotify fd ++ // because inotify cannot watch cgroup file delete event ++ if (inotify_add_watch(info->oom_event_fd, exit_fifo, IN_DELETE | IN_DELETE_SELF) < 0) { ++ ERROR("Failed to watch inotify fd for %s", exit_fifo); ++ goto cleanup; ++ } ++ ++ return info; ++ ++cleanup: ++ common_free_cgroup_oom_handler_info(info); ++ return NULL; ++} ++ + int get_cgroup_version_v2() + { + return CGROUP_VERSION_2; +@@ -426,5 +561,6 @@ int cgroup_v2_ops_init(cgroup_ops *ops) + ops->get_cgroup_info = get_cgroup_info_v2; + ops->get_cgroup_metrics = get_cgroup_metrics_v2; + ops->get_cgroup_mnt_and_root_path = get_cgroup_mnt_and_root_v2; ++ ops->get_cgroup_oom_handler = get_cgroup_oom_handler_v2; + return 0; + } +\ No newline at end of file +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index 47a33c2c..cac5c0ba 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1055,6 +1055,9 @@ void ContainerManagerService::UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 43d66d64..830fd696 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -221,6 +221,8 @@ void container_state_set_restarting(container_state_t *s, int exit_code); + void container_state_set_paused(container_state_t *s); + void container_state_reset_paused(container_state_t *s); + ++void container_state_set_oom_killed(container_state_t *s); ++ + void container_state_set_dead(container_state_t *s); + + void container_state_increase_restart_count(container_state_t *s); +@@ -269,8 +271,7 @@ bool container_is_valid_state_string(const char *state); + + void container_update_health_monitor(const char *container_id); + +-extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container); ++extern int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont); + + extern char *container_exit_fifo_create(const char *cont_state_path); + +diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c +index b84f1ad5..109a628c 100644 +--- a/src/daemon/modules/container/container_events_handler.c ++++ b/src/daemon/modules/container/container_events_handler.c +@@ -114,7 +114,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events + bool has_been_manually_stopped = false; + + /* only handle Exit event */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +@@ -187,6 +187,16 @@ static int container_state_changed(container_t *cont, const struct isulad_events + } + + break; ++ ++ case EVENTS_TYPE_OOM: { ++ container_lock(cont); ++ container_state_set_oom_killed(cont->state); ++ if (container_state_to_disk(cont)) { ++ WARN("Failed to save container \"%s\" to disk", id); ++ } ++ container_unlock(cont); ++ break; ++ } + default: + /* ignore garbage */ + break; +diff --git a/src/daemon/modules/container/container_state.c b/src/daemon/modules/container/container_state.c +index f31959fa..452a2b26 100644 +--- a/src/daemon/modules/container/container_state.c ++++ b/src/daemon/modules/container/container_state.c +@@ -154,6 +154,7 @@ void container_state_set_running(container_state_t *s, const pid_ppid_info_t *pi + state->paused = false; + } + state->exit_code = 0; ++ state->oom_killed = false; + + if (pid_info != NULL) { + state->pid = pid_info->pid; +@@ -222,6 +223,19 @@ void container_state_set_paused(container_state_t *s) + container_state_unlock(s); + } + ++void container_state_set_oom_killed(container_state_t *s) ++{ ++ if (s == NULL || s->state == NULL) { ++ return; ++ } ++ ++ container_state_lock(s); ++ ++ s->state->oom_killed = true; ++ ++ container_state_unlock(s); ++} ++ + /* state reset paused */ + void container_state_reset_paused(container_state_t *s) + { +@@ -573,6 +587,7 @@ container_inspect_state *container_state_to_inspect_state(container_state_t *s) + state->running = s->state->running; + state->paused = s->state->paused; + state->restarting = s->state->restarting; ++ state->oom_killed = s->state->oom_killed; + state->pid = s->state->pid; + + state->exit_code = s->state->exit_code; +diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c +index 76868e28..52f68d21 100644 +--- a/src/daemon/modules/container/restore/restore.c ++++ b/src/daemon/modules/container/restore/restore.c +@@ -24,6 +24,7 @@ + #include + #include + #include ++#include + + #include "isulad_config.h" + +@@ -44,6 +45,8 @@ + #include "utils_file.h" + #include "utils_timestamp.h" + #include "id_name_manager.h" ++#include "cgroup.h" ++#include "specs_api.h" + + /* restore supervisor */ + static int restore_supervisor(const container_t *cont) +@@ -55,9 +58,7 @@ static int restore_supervisor(const container_t *cont) + char *exit_fifo = NULL; + char *id = cont->common_config->id; + char *statepath = cont->state_path; +- char *runtime = cont->runtime; + pid_ppid_info_t pid_info = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id); + if (nret < 0 || (size_t)nret >= sizeof(container_state)) { +@@ -91,11 +92,8 @@ static int restore_supervisor(const container_t *cont) + pid_info.ppid = cont->state->state->p_pid; + pid_info.start_time = cont->state->state->start_time; + pid_info.pstart_time = cont->state->state->p_start_time; +-#ifdef ENABLE_CRI_API_V1 +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); +-#endif + +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, &pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + goto out; +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 63289283..1b7da383 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -41,6 +41,8 @@ + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif ++#include "cgroup.h" ++#include "specs_api.h" + + pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER; + struct epoll_descr g_supervisor_descr; +@@ -269,24 +271,52 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol + return EPOLL_LOOP_HANDLE_CONTINUE; + } + ++static int oom_handle_cb(int fd, uint32_t events, void *cbdata, struct epoll_descr *descr) ++{ ++ cgroup_oom_handler_info_t *oom_handler_info = (cgroup_oom_handler_info_t *)cbdata; ++ bool close_oom_handler = CGROUP_OOM_HANDLE_CLOSE; ++ // supervisor only handle one oom event, so we remove the handler directly ++ if (oom_handler_info != NULL && oom_handler_info->oom_event_handler != NULL) { ++ close_oom_handler = oom_handler_info->oom_event_handler(fd, oom_handler_info); ++ } ++ ++ if (close_oom_handler == CGROUP_OOM_HANDLE_CLOSE) { ++ supervisor_handler_lock(); ++ epoll_loop_del_handler(&g_supervisor_descr, fd); ++ supervisor_handler_unlock(); ++ ++ common_free_cgroup_oom_handler_info(oom_handler_info); ++ } ++ ++ return EPOLL_LOOP_HANDLE_CONTINUE; ++} ++ + /* supervisor add exit monitor */ +-int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name, +- const char *runtime, bool sandbox_container) ++int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const pid_ppid_info_t *pid_info, const container_t *cont) + { + int ret = 0; + struct supervisor_handler_data *data = NULL; ++ cgroup_oom_handler_info_t *oom_handler_info = NULL; ++ __isula_auto_free char *cgroup_path = NULL; + + if (fd < 0) { + ERROR("Invalid exit fifo fd"); + return -1; + } + +- if (pid_info == NULL || name == NULL || runtime == NULL) { ++ if (pid_info == NULL || cont == NULL || cont->common_config == NULL) { + ERROR("Invalid input arguments"); + close(fd); + return -1; + } + ++ cgroup_path = merge_container_cgroups_path(cont->common_config->id, cont->hostconfig); ++ if (cgroup_path == NULL) { ++ ERROR("Failed to get cgroup path"); ++ close(fd); ++ return -1; ++ } ++ + data = util_common_calloc_s(sizeof(struct supervisor_handler_data)); + if (data == NULL) { + ERROR("Memory out"); +@@ -295,15 +325,26 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + } + + data->fd = fd; +- data->name = util_strdup_s(name); +- data->runtime = util_strdup_s(runtime); +- data->is_sandbox_container = sandbox_container; ++ data->name = util_strdup_s(cont->common_config->id); ++ data->runtime = util_strdup_s(cont->runtime); ++#ifdef ENABLE_CRI_API_V1 ++ data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++#endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; + data->pid_info.ppid = pid_info->ppid; + data->pid_info.pstart_time = pid_info->pstart_time; ++ oom_handler_info = common_get_cgroup_oom_handler(fd, cont->common_config->id, cgroup_path, exit_fifo); + + supervisor_handler_lock(); ++ if (oom_handler_info != NULL) { ++ ret = epoll_loop_add_handler(&g_supervisor_descr, oom_handler_info->oom_event_fd, oom_handle_cb, oom_handler_info); ++ if (ret != 0) { ++ ERROR("Failed to add handler for oom event"); ++ goto err; ++ } ++ } ++ + ret = epoll_loop_add_handler(&g_supervisor_descr, fd, supervisor_exit_cb, data); + if (ret != 0) { + ERROR("Failed to add handler for exit fifo"); +@@ -314,6 +355,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf + + err: + supervisor_handler_data_free(data); ++ common_free_cgroup_oom_handler_info(oom_handler_info); + out: + supervisor_handler_unlock(); + return ret; +diff --git a/src/daemon/modules/events/collector.c b/src/daemon/modules/events/collector.c +index fb4a7fea..af688742 100644 +--- a/src/daemon/modules/events/collector.c ++++ b/src/daemon/modules/events/collector.c +@@ -133,6 +133,9 @@ static container_events_type_t lcrsta2Evetype(int value) + case THAWED: + et = EVENTS_TYPE_THAWED; + break; ++ case OOM: ++ et = EVENTS_TYPE_OOM; ++ break; + default: + et = EVENTS_TYPE_EXIT; + break; +@@ -822,8 +825,8 @@ static int post_event_to_events_hander(const struct isulad_events_format *events + return -1; + } + +- /* only post STOPPED event to events_hander */ +- if (events->type != EVENTS_TYPE_STOPPED1) { ++ /* only post STOPPED event and OOM event to events_hander */ ++ if (events->type != EVENTS_TYPE_STOPPED1 && events->type != EVENTS_TYPE_OOM) { + return 0; + } + +diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c +index a8090d5a..eb7ce4f4 100644 +--- a/src/daemon/modules/service/service_container.c ++++ b/src/daemon/modules/service/service_container.c +@@ -275,14 +275,13 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi + return; + } + +-static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container, +- const char *pidfile, int exit_fifo_fd, +- const pid_ppid_info_t *pid_info) ++static int do_post_start_on_success(container_t *cont, int exit_fifo_fd, ++ const char *exit_fifo, const pid_ppid_info_t *pid_info) + { + int ret = 0; + + // exit_fifo_fd was closed in container_supervisor_add_exit_monitor +- if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) { ++ if (container_supervisor_add_exit_monitor(exit_fifo_fd, exit_fifo, pid_info, cont)) { + ERROR("Failed to add exit monitor to supervisor"); + ret = -1; + } +@@ -750,7 +749,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + oci_runtime_spec *oci_spec = NULL; + rt_create_params_t create_params = { 0 }; + rt_start_params_t start_params = { 0 }; +- bool sandbox_container = false; + + nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id); + if (nret < 0 || (size_t)nret >= sizeof(bundle)) { +@@ -899,7 +897,6 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + if (cont->common_config->sandbox_info != NULL) { + create_params.task_addr = cont->common_config->sandbox_info->task_address; + } +- sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); + #endif + + if (runtime_create(id, runtime, &create_params) != 0) { +@@ -924,7 +921,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo + + ret = runtime_start(id, runtime, &start_params, pid_info); + if (ret == 0) { +- if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) { ++ if (do_post_start_on_success(cont, exit_fifo_fd, exit_fifo, pid_info) != 0) { + ERROR("Failed to do post start on runtime start success"); + ret = -1; + goto clean_resources; +-- +2.34.1 + diff --git a/0036-add-ci-cases-for-oomkilled-monitor.patch b/0036-add-ci-cases-for-oomkilled-monitor.patch new file mode 100644 index 0000000..8f4f67e --- /dev/null +++ b/0036-add-ci-cases-for-oomkilled-monitor.patch @@ -0,0 +1,279 @@ +From 0111a575f829b946068dcb11286f0d84363cfc3d Mon Sep 17 00:00:00 2001 +From: jikai +Date: Thu, 28 Mar 2024 12:51:53 +0000 +Subject: [PATCH 36/43] add ci cases for oomkilled monitor + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 14 ++++++++ + test/cgroup/cpu/CMakeLists.txt | 2 ++ + .../image/oci/oci_config_merge/CMakeLists.txt | 1 + + test/image/oci/registry/CMakeLists.txt | 1 + + test/mocks/sender_mock.cc | 34 +++++++++++++++++++ + test/mocks/sender_mock.h | 31 +++++++++++++++++ + test/network/network_ns/CMakeLists.txt | 1 + + test/runtime/isula/CMakeLists.txt | 1 + + test/runtime/lcr/CMakeLists.txt | 1 + + test/specs/specs/CMakeLists.txt | 1 + + test/specs/specs_extend/CMakeLists.txt | 1 + + test/specs/verify/CMakeLists.txt | 1 + + test/volume/CMakeLists.txt | 3 +- + 13 files changed, 91 insertions(+), 1 deletion(-) + create mode 100644 test/mocks/sender_mock.cc + create mode 100644 test/mocks/sender_mock.h + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index cde9ea1f..b4f4a785 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -27,6 +27,7 @@ function test_inspect_spec() + { + local ret=0 + local image="busybox" ++ local ubuntu_image="ubuntu" + local test="container inspect test => (${FUNCNAME[@]})" + + msg_info "${test} starting..." +@@ -37,6 +38,12 @@ function test_inspect_spec() + isula images | grep busybox + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++)) + ++ isula pull ${ubuntu_image} ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to pull image: ${ubuntu_image}" && return ${FAILURE} ++ ++ isula images | grep ubuntu ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${ubuntu_image}" && ((ret++)) ++ + containername=test_inspect + + isula create --name $containername --ipc host --pid host --uts host --restart=on-failure:10 --hook-spec ${test_data_path}/test-hookspec.json --cpu-shares 100 --memory 5MB --memory-reservation 4MB --cpu-period 1000000 --cpu-quota 200000 --cpuset-cpus 1 --cpuset-mems 0 --kernel-memory 50M --pids-limit=10000 --volume /home:/root --env a=1 $image /bin/sh ls +@@ -139,6 +146,13 @@ function test_inspect_spec() + + isula rm -f $containername + ++ isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ ++ isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" ++ [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) ++ ++ isula rm -f $containername ++ + msg_info "${test} finished with return ${ret}..." + return ${ret} + } +diff --git a/test/cgroup/cpu/CMakeLists.txt b/test/cgroup/cpu/CMakeLists.txt +index 30bfc417..9c3cfa12 100644 +--- a/test/cgroup/cpu/CMakeLists.txt ++++ b/test/cgroup/cpu/CMakeLists.txt +@@ -13,6 +13,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd/command_parser.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/daemon_arguments.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config/isulad_config.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + cgroup_cpu_ut.cc) + + target_include_directories(${EXE} PUBLIC +@@ -23,6 +24,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/cgroup ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/api + ${CMAKE_BINARY_DIR}/conf + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cmd +diff --git a/test/image/oci/oci_config_merge/CMakeLists.txt b/test/image/oci/oci_config_merge/CMakeLists.txt +index d13ec738..ffd3999d 100644 +--- a/test/image/oci/oci_config_merge/CMakeLists.txt ++++ b/test/image/oci/oci_config_merge/CMakeLists.txt +@@ -35,6 +35,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/containers_store_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/container_unix_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/specs.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/modules/spec/parse_volume.c +diff --git a/test/image/oci/registry/CMakeLists.txt b/test/image/oci/registry/CMakeLists.txt +index 5b5bc3f5..6166c2d0 100644 +--- a/test/image/oci/registry/CMakeLists.txt ++++ b/test/image/oci/registry/CMakeLists.txt +@@ -44,6 +44,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/oci_image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/http_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../mocks/sender_mock.cc + registry_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/mocks/sender_mock.cc b/test/mocks/sender_mock.cc +new file mode 100644 +index 00000000..26028d7f +--- /dev/null ++++ b/test/mocks/sender_mock.cc +@@ -0,0 +1,34 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-29 ++ * Description: provide collector mock ++ ******************************************************************************/ ++ ++#include "sender_mock.h" ++ ++namespace { ++MockEventSender *g_sender_mock = nullptr; ++} ++ ++void MockEventSender_SetMock(MockEventSender *mock) ++{ ++ g_sender_mock = mock; ++} ++ ++int isulad_monitor_send_container_event(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations) ++{ ++ if (g_sender_mock != nullptr) { ++ return g_sender_mock->IsuladMonitorEventSendContainerEvent(name, state, pid, exit_code, args, extra_annations); ++ } ++ return 0; ++} +diff --git a/test/mocks/sender_mock.h b/test/mocks/sender_mock.h +new file mode 100644 +index 00000000..f4fe75f0 +--- /dev/null ++++ b/test/mocks/sender_mock.h +@@ -0,0 +1,31 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-30 ++ * Description: provide sender mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++#define _ISULAD_TEST_MOCKS_SENDER_MOCK_H ++ ++#include ++#include "events_sender_api.h" ++ ++class MockEventSender { ++public: ++ MOCK_METHOD6(IsuladMonitorEventSendContainerEvent, int(const char *name, runtime_state_t state, int pid, int exit_code, ++ const char *args, const char *extra_annations)); ++}; ++ ++void MockEventSender_SetMock(MockEventSender *mock); ++ ++#endif ++ +diff --git a/test/network/network_ns/CMakeLists.txt b/test/network/network_ns/CMakeLists.txt +index 71b8039d..6f3f36a0 100644 +--- a/test/network/network_ns/CMakeLists.txt ++++ b/test/network/network_ns/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/selinux_label_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + network_ns_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/isula/CMakeLists.txt b/test/runtime/isula/CMakeLists.txt +index c1f0a5cc..15636623 100644 +--- a/test/runtime/isula/CMakeLists.txt ++++ b/test/runtime/isula/CMakeLists.txt +@@ -31,6 +31,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/isula/isula_rt_ops.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + isula_rt_ops_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/runtime/lcr/CMakeLists.txt b/test/runtime/lcr/CMakeLists.txt +index c3b93d67..5b2ed11a 100644 +--- a/test/runtime/lcr/CMakeLists.txt ++++ b/test/runtime/lcr/CMakeLists.txt +@@ -29,6 +29,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/namespace_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/container_unix_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/engine_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/modules/runtime/engines/lcr/lcr_rt_ops.c + lcr_rt_ops_ut.cc) +diff --git a/test/specs/specs/CMakeLists.txt b/test/specs/specs/CMakeLists.txt +index 45f688f9..12c11f51 100644 +--- a/test/specs/specs/CMakeLists.txt ++++ b/test/specs/specs/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/specs_extend/CMakeLists.txt b/test/specs/specs_extend/CMakeLists.txt +index 1b737089..2fd37e1c 100644 +--- a/test/specs/specs_extend/CMakeLists.txt ++++ b/test/specs/specs_extend/CMakeLists.txt +@@ -43,6 +43,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/isulad_config_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + specs_extend_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/specs/verify/CMakeLists.txt b/test/specs/verify/CMakeLists.txt +index b0602127..7f000cd1 100644 +--- a/test/specs/verify/CMakeLists.txt ++++ b/test/specs/verify/CMakeLists.txt +@@ -38,6 +38,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/image_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/storage_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/sender_mock.cc + verify_ut.cc) + + target_include_directories(${EXE} PUBLIC +diff --git a/test/volume/CMakeLists.txt b/test/volume/CMakeLists.txt +index 27d07330..1f9dac03 100644 +--- a/test/volume/CMakeLists.txt ++++ b/test/volume/CMakeLists.txt +@@ -24,6 +24,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v1.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_v2.c + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/daemon/common/cgroup/cgroup_common.c ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../test/mocks/sender_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/cmd/command_parser.c + volume_ut.cc) + +@@ -43,6 +44,6 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../src/utils/console + ) + +-target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) ++target_link_libraries(${EXE} ${GTEST_BOTH_LIBRARIES} ${GMOCK_LIBRARY} ${GMOCK_MAIN_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${ISULA_LIBUTILS_LIBRARY} -lcrypto -lyajl -lz) + add_test(NAME ${EXE} COMMAND ${EXE} --gtest_output=xml:${EXE}-Results.xml) + set_tests_properties(${EXE} PROPERTIES TIMEOUT 120) +-- +2.34.1 + diff --git a/0037-add-cgroup-v2-doc.patch b/0037-add-cgroup-v2-doc.patch new file mode 100644 index 0000000..bfd7530 --- /dev/null +++ b/0037-add-cgroup-v2-doc.patch @@ -0,0 +1,252 @@ +From 2cbce684b973bf4e250f41d750253b5b8abde32d Mon Sep 17 00:00:00 2001 +From: zhongtao +Date: Tue, 20 Feb 2024 19:22:11 +0800 +Subject: [PATCH 37/43] add cgroup v2 doc + +Signed-off-by: zhongtao +--- + docs/design/README_zh.md | 2 + + .../detailed/Container/cgroup_v2_design_zh.md | 193 ++++++++++++++++++ + docs/images/cgroup_v2_module.svg | 16 ++ + 3 files changed, 211 insertions(+) + create mode 100644 docs/design/detailed/Container/cgroup_v2_design_zh.md + create mode 100644 docs/images/cgroup_v2_module.svg + +diff --git a/docs/design/README_zh.md b/docs/design/README_zh.md +index f2c187a1..b7ec3ddb 100644 +--- a/docs/design/README_zh.md ++++ b/docs/design/README_zh.md +@@ -18,6 +18,8 @@ + + - 查看 restart 模块的设计文档： [restart_manager_design](./detailed/Container/restart_manager_design.md)。 + ++- 查看 cgroup v2 的设计文档： [cgroup_v2_design](./detailed/Container/cgroup_v2_design_zh.md)。 ++ + ## CRI + + - 查看 CRI的启动程序的重构文档： [cri_cni_refactor](./detailed/CRI/cri_cni_refactor_zh.md) 。 +diff --git a/docs/design/detailed/Container/cgroup_v2_design_zh.md b/docs/design/detailed/Container/cgroup_v2_design_zh.md +new file mode 100644 +index 00000000..e1ce81d0 +--- /dev/null ++++ b/docs/design/detailed/Container/cgroup_v2_design_zh.md +@@ -0,0 +1,193 @@ ++| Author | zhongtao | ++| ------ | --------------------- | ++| Date | 2024-02-19 | ++| Email | zhongtao17@huawei.com | ++# 方案目标 ++ ++cgroup是linux中用于限制进程组资源的功能。cgroup目前包括两个版本，cgroup v1和cgroup v2。cgroup v2的目标是取代cgroup v1，出于兼容性的考虑，cgroup v1并没有在内核中删除，并且大概率会长期存在。该需求的目的为使得iSulad支持cgroup v2. ++ ++## 与cgroup v1差异 ++无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。不过由于有部分cgroup v1支持的功能在cgroup v2中被去掉了或者实现方式有所变化，因此部分接口在cgroup v2中不可用或者含义发生变化。iSulad支持限制如下资源： ++ ++|资源|功能|和cgroup v1的差异| ++|---|---|---| ++|devices|限制对应的设备是否可以在容器中访问以及访问权限|devcies子系统不再使用往cgroup文件里写值的方式进行限制，而是采用ebpf的方式进行限制| ++|memory|限制容器的内存资源|不支持swappiness，不支持kmem相关参数，不支持oom_control| ++|cpu/cpuset|限制容器的cpu资源|不支持rt_*相关(实时线程)的限制| ++|blkio/io|限制容器的块设备io|不仅限制块设备的IO，也能限制buffer IO| ++|hugetlb|限制大页内存的使用|无差异| ++|pids|限制容器使用的pid|无差异| ++|files|限制容器使用的fd|无差异| ++|freeze|暂停容器|无差异| ++ ++## 使用方式 ++ ++使用的示例如下： ++ ++1. 以限制内存资源为例，假设我们需要限制单个容器最多使用10M内存，则可以在运行容器时加上-m 10m参数进行限制： ++ ++ ```sh ++ [root@openEuler iSulad]# isula run -tid -m 10m busybox sh ++ 000c0c6eb609179062b19a3d2de4d7c38a42c887f55e2a7759ed9df851277163 ++ ``` ++ ++ -m 10m表示限制容器内最多只能使用10m内存，可以通过isula stats命令查看资源的限制情况： ++ ++ ```sh ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 10.00 MiB 1.02 0.00 B / 0.00 B 1 ++ ``` ++ ++ 可以动态更新资源的限制： ++ ```sh ++ [root@openEuler iSulad]# isula update -m 20m 000c0c6eb6 ++ 000c0c6eb6 ++ [root@openEuler iSulad]# isula stats --no-stream 000c0c6eb6 ++ CONTAINER CPU % MEM USAGE / LIMIT MEM % BLOCK I / O PIDS ++ 000c0c6eb609 0.00 104.00 KiB / 20.00 MiB 0.51 0.00 B / 0.00 B 1 ++ ``` ++ ++2. 假设我们要将设备/dev/sda挂载到容器中成为/dev/sdx并限制为只读设备，则可以这么配置： ++ ++```sh ++ [root@openEuler iSulad]# isula run -ti --rm --device=/dev/sda:/dev/sdx:wm busybox fdisk /dev/sdx ++ fdisk: can't open '/dev/sdx' ++ [root@openEuler iSulad]# ++``` ++ ++挂载设备到容器的语法为`--device=$host:$container:rwm $host`指定设备在主机上的绝对路径，$container指定设备在容器内的绝对路径，r表示可读，w表示可写，m表示可以创建node 上述命令中rwm三个参数缺少r参数，也就是说允许写和创建node但是不允许读(即只读)。 ++ ++3. 使用cri的PodSandboxStats接口与ContainerStats接口获取容器的资源使用状况： ++ ++```sh ++[root@openEuler ~]# crictl statsp c3 ++ POD POD ID CPU % MEM ++ test-sandbox c32556d3bb139 0.00 196.6kB ++[root@openEuler ~]# crictl statsp --output json c3 ++...... ++ "linux": { ++ "cpu": { ++ "timestamp": "1708499622485777700", ++ "usageCoreNanoSeconds": { ++ "value": "180973" ++ }, ++ "usageNanoCores": null ++ }, ++ "memory": { ++ "timestamp": "1708499622485777700", ++ "workingSetBytes": { ++ "value": "196608" ++ }, ++ "availableBytes": { ++ "value": "0" ++ }, ++ "usageBytes": { ++ "value": "4386816" ++ }, ++ "rssBytes": { ++ "value": "176128" ++ }, ++ "pageFaults": { ++ "value": "1193" ++ }, ++ "majorPageFaults": { ++ "value": "6" ++ } ++ }, ++ "network": null, ++ "process": { ++ "timestamp": "1708499622485777700", ++ "processCount": { ++ "value": "2" ++ } ++ }, ++..... ++ ++[root@openEuler ~]# crictl stats 01 ++CONTAINER CPU % MEM DISK INODES ++01a726f61c5c3 0.01 3.801MB 16.4kB 8 ++[root@openEuler ~]# ++``` ++ ++# 总体设计 ++ ++在原有只支持cgroup v1的基础上，对cgroup模块进行了重构，重构后的架构图如下： ++ ++![cgroup_v2_module](../../../images/cgroup_v2_module.svg) ++ ++主要功能为以下两种： ++ ++1. iSulad在资源控制参数设置和更新过程中，负责参数的合法校验，用于拦截非法请求，真正的cgroup操作由容器运行时完成。 ++ ++2. iSulad在获得sandbox资源使用信息时，直接读取sandbox cgroup文件信息。 ++ ++# 接口描述 ++由于无论是cgroup v1还是cgroup v2，iSulad提供给用户使用的接口都是一致的。 ++无新增接口。 ++ ++```c ++int verify_container_settings(const oci_runtime_spec *container, const sysinfo_t *sysinfo); ++ ++int verify_host_config_settings(host_config *hostconfig, const sysinfo_t *sysinfo, bool update); ++ ++ ++typedef struct { ++ int (*get_cgroup_info)(cgroup_mem_info_t *meminfo, cgroup_cpu_info_t *cpuinfo, ++ cgroup_hugetlb_info_t *hugetlbinfo, cgroup_blkio_info_t *blkioinfo, ++ cgroup_cpuset_info_t *cpusetinfo, cgroup_pids_info_t *pidsinfo, ++ cgroup_files_info_t *filesinfo, bool quiet); ++ int (*get_cgroup_metrics)(const char *cgroup_path, cgroup_metrics_t *cgroup_metrics); ++ ++ int (*common_find_cgroup_mnt_and_root)(const char *subsystem, char **mountpoint, char **root); ++ ++ char *(*sysinfo_cgroup_controller_cpurt_mnt_path)(void); ++} cgroup_ops; ++ ++int cgroup_v2_ops_init(cgroup_ops *ops) ++{ ++ if (ops == NULL) { ++ return -1; ++ } ++ ops->get_cgroup_info = common_get_cgroup_info_v2; ++ ops->get_cgroup_metrics = common_get_cgroup_v2_metrics; ++ ops->common_find_cgroup_mnt_and_root = common_find_cgroup_v2_mnt_and_root; ++ return 0; ++} ++``` ++ ++# 详细设计 ++ ++```mermaid ++sequenceDiagram ++ participant isula ++ participant kubelet ++ participant isulad ++ participant runc ++ participant cgroup ++ ++ isula->>isulad: request ++ kubelet->>isulad:request ++ alt run/create/update ++ isulad->>isulad:verify request option ++ isulad->>runc:run/create/update request ++ runc ->> cgroup:write cgroup file ++ else stats ++ par container stats ++ isulad->>runc:container stats request ++ runc ->> cgroup:read cgroup file ++ runc ->> isulad:container stats info ++ and sandbox stats ++ isulad ->> cgroup: read cgroup file ++ end ++ end ++ isulad ->> isula:response ++ isulad ->> kubelet:response ++``` ++ ++# 使用限制 ++ ++1. 只支持cgroup 挂载点在/sys/fs/cgroup ++2. cgroup v1与cgrooup v2混用场景不支持 ++3. 该需求只涉及cgroup v2对runc容器运行时的支持 ++ +diff --git a/docs/images/cgroup_v2_module.svg b/docs/images/cgroup_v2_module.svg +new file mode 100644 +index 00000000..59e7939b +--- /dev/null ++++ b/docs/images/cgroup_v2_module.svg +@@ -0,0 +1,16 @@ ++ ++ ++ ++ ++ ++ ++ CLICRICRI moduleverify modulecgroup moduleisulad-> main.ccgroup_ops_initcgroup_v1 modulecgroup_v2 moduleruntime moduleruncisula modulecommon cgroup apicgroup fileread/writeread/write +\ No newline at end of file +-- +2.34.1 + diff --git a/0038-fix-run-ubuntu-container-bug-in-inspect.sh.patch b/0038-fix-run-ubuntu-container-bug-in-inspect.sh.patch new file mode 100644 index 0000000..f16356c --- /dev/null +++ b/0038-fix-run-ubuntu-container-bug-in-inspect.sh.patch @@ -0,0 +1,27 @@ +From 8e1fe0302bf1a871f66a296e456811e878b1fa3b Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 10:06:18 +0800 +Subject: [PATCH 38/43] fix run ubuntu container bug in inspect.sh + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index b4f4a785..86aed3d8 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -146,7 +146,8 @@ function test_inspect_spec() + + isula rm -f $containername + +- isula run -it -m 4m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ # use more than 10m memory limit, otherwise it might fail to run ++ isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0039-add-support-for-GetContainerEvents.patch b/0039-add-support-for-GetContainerEvents.patch new file mode 100644 index 0000000..6e7d03f --- /dev/null +++ b/0039-add-support-for-GetContainerEvents.patch @@ -0,0 +1,2601 @@ +From 745497bdc5c5192709ecc7b3edc91a5170f5b30e Mon Sep 17 00:00:00 2001 +From: jikai +Date: Fri, 29 Mar 2024 09:33:38 +0000 +Subject: [PATCH 39/43] add support for GetContainerEvents + +Signed-off-by: jikai +--- + src/daemon/CMakeLists.txt | 3 + + src/daemon/common/cri/v1/v1_cri_helpers.cc | 205 +++++++++++++++ + src/daemon/common/cri/v1/v1_cri_helpers.h | 5 + + .../{entry => common}/cri/v1/v1_naming.cc | 0 + .../{entry => common}/cri/v1/v1_naming.h | 0 + src/daemon/config/isulad_config.c | 1 + + .../entry/connect/grpc/cri/cri_service.cc | 3 +- + .../entry/connect/grpc/cri/cri_service.h | 1 + + .../cri/v1/cri_v1_runtime_runtime_service.cc | 147 ++++++++++- + .../cri/v1/cri_v1_runtime_runtime_service.h | 11 +- + src/daemon/entry/connect/grpc/grpc_service.cc | 6 +- + .../v1/v1_cri_container_manager_service.cc | 203 +-------------- + .../cri/v1/v1_cri_container_manager_service.h | 13 - + .../v1/v1_cri_pod_sandbox_manager_service.cc | 92 ++++++- + .../v1/v1_cri_pod_sandbox_manager_service.h | 10 +- + .../entry/cri/v1/v1_cri_runtime_service.h | 4 +- + .../cri/v1/v1_cri_runtime_service_impl.cc | 10 +- + .../cri/v1/v1_cri_runtime_service_impl.h | 7 +- + src/daemon/executor/container_cb/execution.c | 25 ++ + .../executor/container_cb/execution_create.c | 12 + + src/daemon/mailbox/CMakeLists.txt | 11 + + src/daemon/mailbox/mailbox.c | 167 +++++++++++++ + src/daemon/mailbox/mailbox.h | 82 ++++++ + src/daemon/mailbox/mailbox_message.c | 94 +++++++ + src/daemon/mailbox/mailbox_message.h | 50 ++++ + src/daemon/mailbox/message_queue.c | 234 ++++++++++++++++++ + src/daemon/mailbox/message_queue.h | 57 +++++ + src/daemon/mailbox/message_subscriber.c | 85 +++++++ + src/daemon/mailbox/message_subscriber.h | 41 +++ + src/daemon/modules/api/container_api.h | 5 + + .../modules/container/supervisor/supervisor.c | 18 ++ + src/daemon/sandbox/sandbox.cc | 9 + + src/utils/cutils/blocking_queue.c | 185 ++++++++++++++ + src/utils/cutils/blocking_queue.h | 66 +++++ + test/mocks/mailbox_mock.cc | 30 +++ + test/mocks/mailbox_mock.h | 30 +++ + test/sandbox/controller/shim/CMakeLists.txt | 1 + + test/sandbox/sandbox/CMakeLists.txt | 2 + + 38 files changed, 1681 insertions(+), 244 deletions(-) + rename src/daemon/{entry => common}/cri/v1/v1_naming.cc (100%) + rename src/daemon/{entry => common}/cri/v1/v1_naming.h (100%) + create mode 100644 src/daemon/mailbox/CMakeLists.txt + create mode 100644 src/daemon/mailbox/mailbox.c + create mode 100644 src/daemon/mailbox/mailbox.h + create mode 100644 src/daemon/mailbox/mailbox_message.c + create mode 100644 src/daemon/mailbox/mailbox_message.h + create mode 100644 src/daemon/mailbox/message_queue.c + create mode 100644 src/daemon/mailbox/message_queue.h + create mode 100644 src/daemon/mailbox/message_subscriber.c + create mode 100644 src/daemon/mailbox/message_subscriber.h + create mode 100644 src/utils/cutils/blocking_queue.c + create mode 100644 src/utils/cutils/blocking_queue.h + create mode 100644 test/mocks/mailbox_mock.cc + create mode 100644 test/mocks/mailbox_mock.h + +diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt +index d5280c88..29af3dca 100644 +--- a/src/daemon/CMakeLists.txt ++++ b/src/daemon/CMakeLists.txt +@@ -3,6 +3,7 @@ + aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} daemon_top_srcs) + add_subdirectory(executor) + add_subdirectory(entry) ++add_subdirectory(mailbox) + add_subdirectory(modules) + add_subdirectory(config) + add_subdirectory(common) +@@ -11,6 +12,7 @@ set(local_daemon_srcs + ${daemon_top_srcs} + ${EXECUTOR_SRCS} + ${ENTRY_SRCS} ++ ${MAILBOX_SRCS} + ${MODULES_SRCS} + ${CONFIG_SRCS} + ${DAEMON_COMMON_SRCS} +@@ -20,6 +22,7 @@ set(local_daemon_incs + ${CMAKE_CURRENT_SOURCE_DIR} + ${EXECUTOR_INCS} + ${ENTRY_INCS} ++ ${MAILBOX_INCS} + ${MODULES_INCS} + ${CONFIG_INCS} + ${DAEMON_COMMON_INCS} +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index c57301ce..a3488894 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -32,6 +32,7 @@ + #include "service_container_api.h" + #include "isulad_config.h" + #include "sha256.h" ++#include "v1_naming.h" + + namespace CRIHelpersV1 { + +@@ -458,4 +459,208 @@ void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecu + } + } + ++void PackContainerImageToStatus( ++ container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) ++{ ++ if (inspect->config == nullptr) { ++ return; ++ } ++ ++ if (inspect->config->image != nullptr) { ++ contStatus->mutable_image()->set_image(inspect->config->image); ++ } ++ ++ contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); ++} ++ ++void UpdateBaseStatusFromInspect( ++ container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, ++ std::unique_ptr &contStatus) ++{ ++ runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; ++ std::string reason; ++ std::string message; ++ int32_t exitCode { 0 }; ++ ++ if (inspect->state == nullptr) { ++ goto pack_status; ++ } ++ ++ if (inspect->state->running) { ++ // Container is running ++ state = runtime::v1::CONTAINER_RUNNING; ++ } else { ++ // Container is not running. ++ if (finishedAt != 0) { // Case 1 ++ state = runtime::v1::CONTAINER_EXITED; ++ if (inspect->state->exit_code == 0) { ++ reason = "Completed"; ++ } else { ++ reason = "Error"; ++ } ++ } else if (inspect->state->exit_code != 0) { // Case 2 ++ state = runtime::v1::CONTAINER_EXITED; ++ finishedAt = createdAt; ++ startedAt = createdAt; ++ reason = "ContainerCannotRun"; ++ } else { // Case 3 ++ state = runtime::v1::CONTAINER_CREATED; ++ } ++ if (inspect->state->error != nullptr) { ++ message = inspect->state->error; ++ } ++ exitCode = (int32_t)inspect->state->exit_code; ++ } ++ ++pack_status: ++ contStatus->set_exit_code(exitCode); ++ contStatus->set_state(state); ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ contStatus->set_reason(reason); ++ contStatus->set_message(message); ++} ++ ++void PackLabelsToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ if (inspect->config == nullptr || inspect->config->labels == nullptr) { ++ return; ++ } ++ CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); ++ CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); ++ for (size_t i = 0; i < inspect->config->labels->len; i++) { ++ if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { ++ contStatus->set_log_path(inspect->config->labels->values[i]); ++ break; ++ } ++ } ++} ++ ++void ConvertMountsToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ for (size_t i = 0; i < inspect->mounts_len; i++) { ++ runtime::v1::Mount *mount = contStatus->add_mounts(); ++ mount->set_host_path(inspect->mounts[i]->source); ++ mount->set_container_path(inspect->mounts[i]->destination); ++ mount->set_readonly(!inspect->mounts[i]->rw); ++ if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); ++ } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { ++ mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); ++ } ++ // Note: Can't set SeLinuxRelabel ++ } ++} ++ ++void ConvertResourcesToStatus(container_inspect *inspect, ++ std::unique_ptr &contStatus) ++{ ++ if (inspect->resources == nullptr) { ++ return; ++ } ++ runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); ++ if (inspect->resources->cpu_shares != 0) { ++ resources->set_cpu_shares(inspect->resources->cpu_shares); ++ } ++ if (inspect->resources->cpu_period != 0) { ++ resources->set_cpu_period(inspect->resources->cpu_period); ++ } ++ if (inspect->resources->cpu_quota != 0) { ++ resources->set_cpu_quota(inspect->resources->cpu_quota); ++ } ++ if (inspect->resources->memory != 0) { ++ resources->set_memory_limit_in_bytes(inspect->resources->memory); ++ } ++ if (inspect->resources->memory_swap != 0) { ++ resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); ++ } ++ for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { ++ runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); ++ hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); ++ hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); ++ } ++ if (inspect->resources->unified != nullptr) { ++ for (size_t i = 0; i < inspect->resources->unified->len; i++) { ++ auto &resUnified = *(resources->mutable_unified()); ++ resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; ++ } ++ } ++} ++ ++void ContainerStatusToGRPC(container_inspect *inspect, ++ std::unique_ptr &contStatus, ++ Errors &error) ++{ ++ if (inspect->id != nullptr) { ++ contStatus->set_id(inspect->id); ++ } ++ ++ int64_t createdAt {}; ++ int64_t startedAt {}; ++ int64_t finishedAt {}; ++ CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ contStatus->set_created_at(createdAt); ++ contStatus->set_started_at(startedAt); ++ contStatus->set_finished_at(finishedAt); ++ ++ PackContainerImageToStatus(inspect, contStatus, error); ++ UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); ++ PackLabelsToStatus(inspect, contStatus); ++ CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); ++ if (error.NotEmpty()) { ++ return; ++ } ++ ConvertMountsToStatus(inspect, contStatus); ++ ConvertResourcesToStatus(inspect, contStatus); ++} ++ ++std::unique_ptr GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++{ ++ if (m_cb == nullptr) { ++ error.SetError("Invalid input arguments: empty service executor"); ++ return nullptr; ++ } ++ ++ if (containerID.empty()) { ++ error.SetError("Empty container id"); ++ return nullptr; ++ } ++ ++ std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); ++ return nullptr; ++ } ++ ++ container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); ++ if (error.NotEmpty()) { ++ return nullptr; ++ } ++ if (inspect == nullptr) { ++ error.SetError("Get null inspect"); ++ return nullptr; ++ } ++ using ContainerStatusPtr = std::unique_ptr; ++ ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); ++ if (contStatus == nullptr) { ++ error.SetError("Out of memory"); ++ free_container_inspect(inspect); ++ return nullptr; ++ } ++ ++ ContainerStatusToGRPC(inspect, contStatus, error); ++ ++ free_container_inspect(inspect); ++ return contStatus; ++} ++ + } // v1 namespace CRIHelpers +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h +index b6e6aec6..1578c428 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.h ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.h +@@ -27,6 +27,8 @@ + #include "checkpoint_handler.h" + #include "constants.h" + #include "errors.h" ++#include "callback.h" ++#include "cstruct_wrapper.h" + + namespace CRIHelpersV1 { + +@@ -78,6 +80,9 @@ std::string CRISandboxerConvert(const std::string &runtime); + void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc, + Errors &error); + ++auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error) ++-> std::unique_ptr; ++ + }; // namespace CRIHelpers + + #endif // DAEMON_ENTRY_CRI_V1ALPHA_CRI_HELPERS_H +diff --git a/src/daemon/entry/cri/v1/v1_naming.cc b/src/daemon/common/cri/v1/v1_naming.cc +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.cc +rename to src/daemon/common/cri/v1/v1_naming.cc +diff --git a/src/daemon/entry/cri/v1/v1_naming.h b/src/daemon/common/cri/v1/v1_naming.h +similarity index 100% +rename from src/daemon/entry/cri/v1/v1_naming.h +rename to src/daemon/common/cri/v1/v1_naming.h +diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c +index 8179558e..778ff921 100644 +--- a/src/daemon/config/isulad_config.c ++++ b/src/daemon/config/isulad_config.c +@@ -1760,6 +1760,7 @@ int merge_json_confs_into_global(struct service_arguments *args) + args->json_confs->cri_sandboxers = tmp_json_confs->cri_sandboxers; + tmp_json_confs->cri_sandboxers = NULL; + args->json_confs->enable_cri_v1 = tmp_json_confs->enable_cri_v1; ++ args->json_confs->enable_pod_events = tmp_json_confs->enable_pod_events; + #endif + + args->json_confs->systemd_cgroup = tmp_json_confs->systemd_cgroup; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.cc b/src/daemon/entry/connect/grpc/cri/cri_service.cc +index c1986c44..d10a60b5 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.cc +@@ -89,8 +89,9 @@ int CRIService::Init(const isulad_daemon_configs *config) + + #ifdef ENABLE_CRI_API_V1 + m_enableCRIV1 = config->enable_cri_v1; ++ m_enablePodEvents = config->enable_pod_events; + if (m_enableCRIV1) { +- m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, err); ++ m_runtimeV1RuntimeService.Init(m_podSandboxImage, m_pluginManager, m_enablePodEvents, err); + if (err.NotEmpty()) { + ERROR("Init CRI v1 runtime service failed: %s", err.GetCMessage()); + return -1; +diff --git a/src/daemon/entry/connect/grpc/cri/cri_service.h b/src/daemon/entry/connect/grpc/cri/cri_service.h +index 77b2eb72..041c7c63 100644 +--- a/src/daemon/entry/connect/grpc/cri/cri_service.h ++++ b/src/daemon/entry/connect/grpc/cri/cri_service.h +@@ -56,6 +56,7 @@ private: + std::string m_podSandboxImage; + std::shared_ptr m_pluginManager; + bool m_enableCRIV1; ++ bool m_enablePodEvents; + }; + + } +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +index 76e393f3..bc5ab591 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.cc +@@ -22,11 +22,37 @@ + #include "callback.h" + #include "network_plugin.h" + #include "v1_cri_runtime_service_impl.h" ++#include "mailbox.h" ++#include "mailbox_message.h" + + using namespace CRIV1; + ++static void *cri_container_topic_handler(void *context, void *arg) ++{ ++ if (context == nullptr || arg == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ auto v1runtimeService = static_cast(context); ++ auto msg = static_cast(arg); ++ return v1runtimeService->GenerateCRIContainerEvent(msg->container_id, msg->sandbox_id, ++ static_cast(msg->type)); ++} ++ ++static void cri_container_topic_release(void *arg) ++{ ++ if (arg == nullptr) { ++ return; ++ } ++ ++ auto resp = static_cast(arg); ++ delete resp; ++} ++ + void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, +- std::shared_ptr networkPlugin, Errors &err) ++ std::shared_ptr networkPlugin, ++ bool enablePodEvents, Errors &err) + { + // Assembly implementation for CRIRuntimeServiceImpl + service_executor_t *cb = get_service_executor(); +@@ -36,7 +62,18 @@ void RuntimeV1RuntimeServiceImpl::Init(std::string &podSandboxImage, + return; + } + +- m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin)); ++ if (enablePodEvents) { ++ if (mailbox_register_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER, cri_container_topic_handler, ++ this, cri_container_topic_release, true) != 0) { ++ ERROR("Failed to register container topic handler"); ++ err.SetError("Failed to register container topic handler"); ++ return; ++ } ++ m_enablePodEvents = enablePodEvents; ++ } ++ ++ ++ m_rService = std::unique_ptr(new CRIRuntimeServiceImpl(podSandboxImage, cb, networkPlugin, m_enablePodEvents)); + } + + void RuntimeV1RuntimeServiceImpl::Wait() +@@ -45,6 +82,54 @@ void RuntimeV1RuntimeServiceImpl::Wait() + + void RuntimeV1RuntimeServiceImpl::Shutdown() + { ++ mailbox_unregister_topic_handler(MAILBOX_TOPIC_CRI_CONTAINER); ++} ++ ++auto RuntimeV1RuntimeServiceImpl::GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, ++ runtime::v1::ContainerEventType type) -> runtime::v1::ContainerEventResponse * ++{ ++ if (container_id == nullptr || sandbox_id == nullptr) { ++ ERROR("Invalid input arguments"); ++ return nullptr; ++ } ++ ++ if (type < runtime::v1::ContainerEventType::CONTAINER_CREATED_EVENT || ++ type > runtime::v1::ContainerEventType::CONTAINER_DELETED_EVENT) { ++ ERROR("Invalid container event type %d", type); ++ return nullptr; ++ } ++ ++ std::string containerID(container_id), sandboxID(sandbox_id); ++ Errors error; ++ runtime::v1::ContainerEventResponse *response = new (std::nothrow) runtime::v1::ContainerEventResponse(); ++ if (response == nullptr) { ++ ERROR("Out of memory"); ++ return nullptr; ++ } ++ ++ runtime::v1::PodSandboxStatusResponse *statusReply = new (std::nothrow) runtime::v1::PodSandboxStatusResponse(); ++ if (statusReply == nullptr) { ++ ERROR("Out of memory"); ++ delete response; ++ return nullptr; ++ } ++ ++ m_rService->PodSandboxStatus(sandboxID, statusReply, error); ++ if (!error.Empty()) { ++ WARN("Object: CRI, Type: Failed to status pod:%s due to %s", sandboxID.c_str(), ++ error.GetMessage().c_str()); ++ } else { ++ *(response->mutable_pod_sandbox_status()) = *(statusReply->mutable_status()); ++ for (auto &containerStatus : statusReply->containers_statuses()) { ++ *(response->add_containers_statuses()) = containerStatus; ++ } ++ } ++ ++ response->set_container_event_type((runtime::v1::ContainerEventType)type); ++ response->set_container_id(containerID); ++ response->set_created_at(util_get_now_time_nanos()); ++ ++ return response; + } + + grpc::Status RuntimeV1RuntimeServiceImpl::Version(grpc::ServerContext *context, +@@ -398,14 +483,12 @@ grpc::Status RuntimeV1RuntimeServiceImpl::PodSandboxStatus(grpc::ServerContext * + + INFO("Event: {Object: CRI, Type: Status Pod: %s}", request->pod_sandbox_id().c_str()); + +- std::unique_ptr podStatus; +- podStatus = m_rService->PodSandboxStatus(request->pod_sandbox_id(), error); +- if (!error.Empty() || !podStatus) { ++ m_rService->PodSandboxStatus(request->pod_sandbox_id(), reply, error); ++ if (!error.Empty()) { + ERROR("Object: CRI, Type: Failed to status pod:%s due to %s", request->pod_sandbox_id().c_str(), + error.GetMessage().c_str()); + return grpc::Status(grpc::StatusCode::UNKNOWN, error.GetMessage()); + } +- *(reply->mutable_status()) = *podStatus; + + INFO("Event: {Object: CRI, Type: Statused Pod: %s}", request->pod_sandbox_id().c_str()); + +@@ -657,3 +740,55 @@ RuntimeV1RuntimeServiceImpl::RuntimeConfig(grpc::ServerContext *context, + + return grpc::Status::OK; + } ++ ++grpc::Status RuntimeV1RuntimeServiceImpl::GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter *writer) ++{ ++ Errors error; ++ ++ if (context == nullptr || request == nullptr || writer == nullptr) { ++ ERROR("Invalid input arguments"); ++ return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "Invalid input arguments"); ++ } ++ ++ if (!m_enablePodEvents) { ++ ERROR("Pod events is not enabled"); ++ return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "Pod events is not enabled"); ++ } ++ ++ INFO("Event: {Object: CRI, Type: Getting Container Events}"); ++ ++ __isula_auto_subscriber auto sub = mailbox_subscribe(MAILBOX_TOPIC_CRI_CONTAINER); ++ if (sub == nullptr) { ++ ERROR("Object: CRI, Type: Failed to subscribe container events"); ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Failed to subscribe container events"); ++ } ++ ++ for (;;) { ++ __isula_auto_mailbox_message mailbox_message *msg = NULL; ++ int ret = message_subscriber_pop(sub, &msg); ++ if (ret == 0) { ++ if (msg == nullptr) { ++ // nullptr response indicates eventqueue being shutdown, not need to unscribe now ++ return grpc::Status(grpc::StatusCode::UNKNOWN, "Event queue is shutdown"); ++ } ++ auto *response = static_cast(msg->data); ++ if (!writer->Write(*response)) { ++ break; ++ } ++ } else if (ret != ETIMEDOUT) { ++ ERROR("Failed to pop message from subscriber"); ++ break; ++ } ++ if (context->IsCancelled()) { ++ INFO("Object: CRI, Type: GetContainerEvents is cancelled"); ++ break; ++ } ++ } ++ ++ mailbox_unsubscribe(MAILBOX_TOPIC_CRI_CONTAINER, sub); ++ INFO("Event: {Object: CRI, Type: Got Container Events}"); ++ ++ return grpc::Status::OK; ++} +diff --git a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +index 52cc6b99..842d1811 100644 +--- a/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h ++++ b/src/daemon/entry/connect/grpc/cri/v1/cri_v1_runtime_runtime_service.h +@@ -26,9 +26,13 @@ + // Implement of runtime RuntimeService + class RuntimeV1RuntimeServiceImpl : public runtime::v1::RuntimeService::Service { + public: +- void Init(std::string &podSandboxImage, std::shared_ptr networkPlugin, Errors &err); ++ void Init(std::string &podSandboxImage, std::shared_ptr networkPlugin, ++ bool enablePodEvents, Errors &err); + void Wait(); + void Shutdown(); ++ auto GenerateCRIContainerEvent(const char *container_id, const char *sandbox_id, runtime::v1::ContainerEventType type) ++ -> runtime::v1::ContainerEventResponse *; ++ + grpc::Status Version(grpc::ServerContext *context, const runtime::v1::VersionRequest *request, + runtime::v1::VersionResponse *reply) override; + +@@ -105,8 +109,13 @@ public: + const runtime::v1::RuntimeConfigRequest *request, + runtime::v1::RuntimeConfigResponse *reply) override; + ++ grpc::Status GetContainerEvents(grpc::ServerContext *context, ++ const runtime::v1::GetEventsRequest *request, ++ grpc::ServerWriter *writer) override; ++ + private: + std::unique_ptr m_rService; ++ bool m_enablePodEvents; + }; + + #endif // DAEMON_ENTRY_CONNECT_GRPC_CRI_V1_RUNTIME_RUNTIME_SERVICE_H +diff --git a/src/daemon/entry/connect/grpc/grpc_service.cc b/src/daemon/entry/connect/grpc/grpc_service.cc +index 61e284f3..1d8de922 100644 +--- a/src/daemon/entry/connect/grpc/grpc_service.cc ++++ b/src/daemon/entry/connect/grpc/grpc_service.cc +@@ -108,11 +108,11 @@ public: + + void Shutdown(void) + { +- m_server->Shutdown(); +- +- // call CRI to shutdown stream server ++ // call CRI to shutdown stream server, shutdown cri first to notify events thread to exit + m_criService.Shutdown(); + ++ m_server->Shutdown(); ++ + // Shutdown daemon, this operation should remove socket file. + for (const auto &address : m_socketPath) { + if (address.find(UNIX_SOCKET_PREFIX) == 0) { +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +index cac5c0ba..e86dafae 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc +@@ -1007,208 +1007,9 @@ cleanup: + return contStats; + } + +-void ContainerManagerService::PackContainerImageToStatus( +- container_inspect *inspect, std::unique_ptr &contStatus, Errors &error) ++std::unique_ptr ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) + { +- if (inspect->config == nullptr) { +- return; +- } +- +- if (inspect->config->image != nullptr) { +- contStatus->mutable_image()->set_image(inspect->config->image); +- } +- +- contStatus->set_image_ref(CRIHelpers::ToPullableImageID(inspect->config->image, inspect->config->image_ref)); +- return; +-} +- +-void ContainerManagerService::UpdateBaseStatusFromInspect( +- container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, int64_t &finishedAt, +- std::unique_ptr &contStatus) +-{ +- runtime::v1::ContainerState state { runtime::v1::CONTAINER_UNKNOWN }; +- std::string reason; +- std::string message; +- int32_t exitCode { 0 }; +- +- if (inspect->state == nullptr) { +- goto pack_status; +- } +- +- if (inspect->state->running) { +- // Container is running +- state = runtime::v1::CONTAINER_RUNNING; +- } else { +- // Container is not running. +- if (finishedAt != 0) { // Case 1 +- state = runtime::v1::CONTAINER_EXITED; +- if (inspect->state->exit_code == 0) { +- reason = "Completed"; +- } else { +- reason = "Error"; +- } +- } else if (inspect->state->exit_code != 0) { // Case 2 +- state = runtime::v1::CONTAINER_EXITED; +- finishedAt = createdAt; +- startedAt = createdAt; +- reason = "ContainerCannotRun"; +- } else { // Case 3 +- state = runtime::v1::CONTAINER_CREATED; +- } +- if (inspect->state->oom_killed) { +- reason = "OOMKilled"; +- } +- if (inspect->state->error != nullptr) { +- message = inspect->state->error; +- } +- exitCode = (int32_t)inspect->state->exit_code; +- } +- +-pack_status: +- contStatus->set_exit_code(exitCode); +- contStatus->set_state(state); +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- contStatus->set_reason(reason); +- contStatus->set_message(message); +-} +- +-void ContainerManagerService::PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- if (inspect->config == nullptr || inspect->config->labels == nullptr) { +- return; +- } +- CRIHelpers::ExtractLabels(inspect->config->labels, *contStatus->mutable_labels()); +- CRIHelpers::ExtractAnnotations(inspect->config->annotations, *contStatus->mutable_annotations()); +- for (size_t i = 0; i < inspect->config->labels->len; i++) { +- if (strcmp(inspect->config->labels->keys[i], CRIHelpers::Constants::CONTAINER_LOGPATH_LABEL_KEY.c_str()) == 0) { +- contStatus->set_log_path(inspect->config->labels->values[i]); +- break; +- } +- } +-} +- +-void ContainerManagerService::ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- for (size_t i = 0; i < inspect->mounts_len; i++) { +- runtime::v1::Mount *mount = contStatus->add_mounts(); +- mount->set_host_path(inspect->mounts[i]->source); +- mount->set_container_path(inspect->mounts[i]->destination); +- mount->set_readonly(!inspect->mounts[i]->rw); +- if (inspect->mounts[i]->propagation == nullptr || strcmp(inspect->mounts[i]->propagation, "rprivate") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_PRIVATE); +- } else if (strcmp(inspect->mounts[i]->propagation, "rslave") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_HOST_TO_CONTAINER); +- } else if (strcmp(inspect->mounts[i]->propagation, "rshared") == 0) { +- mount->set_propagation(runtime::v1::PROPAGATION_BIDIRECTIONAL); +- } +- // Note: Can't set SeLinuxRelabel +- } +-} +- +-void ContainerManagerService::ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus) +-{ +- if (inspect->resources == nullptr) { +- return; +- } +- runtime::v1::LinuxContainerResources *resources = contStatus->mutable_resources()->mutable_linux(); +- if (inspect->resources->cpu_shares != 0) { +- resources->set_cpu_shares(inspect->resources->cpu_shares); +- } +- if (inspect->resources->cpu_period != 0) { +- resources->set_cpu_period(inspect->resources->cpu_period); +- } +- if (inspect->resources->cpu_quota != 0) { +- resources->set_cpu_quota(inspect->resources->cpu_quota); +- } +- if (inspect->resources->memory != 0) { +- resources->set_memory_limit_in_bytes(inspect->resources->memory); +- } +- if (inspect->resources->memory_swap != 0) { +- resources->set_memory_swap_limit_in_bytes(inspect->resources->memory_swap); +- } +- for (size_t i = 0; i < inspect->resources->hugetlbs_len; i++) { +- runtime::v1::HugepageLimit *hugepage = resources->add_hugepage_limits(); +- hugepage->set_page_size(inspect->resources->hugetlbs[i]->page_size); +- hugepage->set_limit(inspect->resources->hugetlbs[i]->limit); +- } +- if (inspect->resources->unified != nullptr) { +- for (size_t i = 0; i < inspect->resources->unified->len; i++) { +- auto &resUnified = *(resources->mutable_unified()); +- resUnified[inspect->resources->unified->keys[i]] = inspect->resources->unified->values[i]; +- } +- } +-} +- +-void ContainerManagerService::ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr &contStatus, +- Errors &error) +-{ +- if (inspect->id != nullptr) { +- contStatus->set_id(inspect->id); +- } +- +- int64_t createdAt {}; +- int64_t startedAt {}; +- int64_t finishedAt {}; +- CRIHelpers::GetContainerTimeStamps(inspect, &createdAt, &startedAt, &finishedAt, error); +- if (error.NotEmpty()) { +- return; +- } +- contStatus->set_created_at(createdAt); +- contStatus->set_started_at(startedAt); +- contStatus->set_finished_at(finishedAt); +- +- PackContainerImageToStatus(inspect, contStatus, error); +- UpdateBaseStatusFromInspect(inspect, createdAt, startedAt, finishedAt, contStatus); +- PackLabelsToStatus(inspect, contStatus); +- CRINamingV1::ParseContainerName(contStatus->annotations(), contStatus->mutable_metadata(), error); +- if (error.NotEmpty()) { +- return; +- } +- ConvertMountsToStatus(inspect, contStatus); +- ConvertResourcesToStatus(inspect, contStatus); +-} +- +-std::unique_ptr +-ContainerManagerService::ContainerStatus(const std::string &containerID, Errors &error) +-{ +- if (containerID.empty()) { +- error.SetError("Empty container id"); +- return nullptr; +- } +- +- std::string realContainerID = CRIHelpers::GetRealContainerOrSandboxID(m_cb, containerID, false, error); +- if (error.NotEmpty()) { +- ERROR("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- error.Errorf("Failed to find container id %s: %s", containerID.c_str(), error.GetCMessage()); +- return nullptr; +- } +- +- container_inspect *inspect = CRIHelpers::InspectContainer(realContainerID, error, false); +- if (error.NotEmpty()) { +- return nullptr; +- } +- if (inspect == nullptr) { +- error.SetError("Get null inspect"); +- return nullptr; +- } +- using ContainerStatusPtr = std::unique_ptr; +- ContainerStatusPtr contStatus(new (std::nothrow) runtime::v1::ContainerStatus); +- if (contStatus == nullptr) { +- error.SetError("Out of memory"); +- free_container_inspect(inspect); +- return nullptr; +- } +- +- ContainerStatusToGRPC(inspect, contStatus, error); +- +- free_container_inspect(inspect); +- return contStatus; ++ return CRIHelpersV1::GetContainerStatus(m_cb, containerID, error); + } + + void ContainerManagerService::UpdateContainerResources(const std::string &containerID, +diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +index 4e772bda..50f5ed69 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.h +@@ -111,19 +111,6 @@ private: + std::unique_ptr &container); + void SetFsUsage(const imagetool_fs_info *fs_usage, int64_t timestamp, + std::unique_ptr &container); +- void ContainerStatusToGRPC(container_inspect *inspect, +- std::unique_ptr &contStatus, Errors &error); +- void PackContainerImageToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus, Errors &error); +- void UpdateBaseStatusFromInspect(container_inspect *inspect, int64_t &createdAt, int64_t &startedAt, +- int64_t &finishedAt, +- std::unique_ptr &contStatus); +- void PackLabelsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); +- void ConvertMountsToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); +- void ConvertResourcesToStatus(container_inspect *inspect, +- std::unique_ptr &contStatus); + void ExecSyncFromGRPC(const std::string &containerID, const google::protobuf::RepeatedPtrField &cmd, + int64_t timeout, container_exec_request **request, Errors &error); + auto ValidateExecRequest(const runtime::v1::ExecRequest &req, Errors &error) -> int; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +index f125e714..4291d8a0 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.cc +@@ -36,6 +36,7 @@ + #include "sandbox_manager.h" + #include "transform.h" + #include "isulad_config.h" ++#include "mailbox.h" + + namespace CRIV1 { + void PodSandboxManagerService::PrepareSandboxData(const runtime::v1::PodSandboxConfig &config, +@@ -302,6 +303,7 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + std::string jsonCheckpoint; + std::string network_setting_json; + runtime::v1::PodSandboxConfig copyConfig = config; ++ cri_container_message_t msg = { 0 }; + + // Step 1: Parepare sandbox name, runtime and networkMode + PrepareSandboxData(config, runtimeHandler, sandboxName, runtimeInfo, networkMode, error); +@@ -372,6 +374,11 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + goto cleanup_network; + } + ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + // Step 10: Save network settings json to disk + // Update network settings before start sandbox since sandbox container will use the sandbox key + if (namespace_is_cni(networkMode.c_str())) { +@@ -391,6 +398,9 @@ auto PodSandboxManagerService::RunPodSandbox(const runtime::v1::PodSandboxConfig + return response_id; + } + ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ + return sandbox->GetId(); + + cleanup_network: +@@ -700,6 +710,13 @@ void PodSandboxManagerService::RemovePodSandbox(const std::string &podSandboxID, + ERROR("Failed to delete sandbox %s: %s", podSandboxID.c_str(), error.GetCMessage()); + } + ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = sandbox->GetId().c_str(); ++ msg.sandbox_id = sandbox->GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto PodSandboxManagerService::SharesHostNetwork(const container_inspect *inspect) -> runtime::v1::NamespaceMode +@@ -800,10 +817,29 @@ void PodSandboxManagerService::SetSandboxStatusNetwork(std::shared_ptr +-PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Errors &error) ++void PodSandboxManagerService::GetContainerStatuses(const std::string &podSandboxID, ++ std::vector> &containerStatuses, ++ std::vector &errors) { ++ auto list_response_wrapper = GetContainerListResponse(podSandboxID, errors); ++ if (list_response_wrapper == nullptr) { ++ return; ++ } ++ ++ auto list_response = list_response_wrapper->get(); ++ // Remove all containers in the sandbox. ++ for (size_t i = 0; i < list_response->containers_len; i++) { ++ Errors stError; ++ containerStatuses.push_back(CRIHelpersV1::GetContainerStatus(m_cb, list_response->containers[i]->id, stError)); ++ if (stError.NotEmpty()) { ++ ERROR("Error get container status: %s: %s", list_response->containers[i]->id, stError.GetCMessage()); ++ errors.push_back(stError.GetMessage()); ++ } ++ } ++} ++ ++std::unique_ptr PodSandboxManagerService::GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) + { +- std::unique_ptr podStatus(new runtime::v1::PodSandboxStatus); ++ std::unique_ptr podStatus(new (std::nothrow) runtime::v1::PodSandboxStatus); + if (podStatus == nullptr) { + ERROR("Out of memory"); + error.SetError("Out of memory"); +@@ -831,6 +867,50 @@ PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, Erro + return podStatus; + } + ++void PodSandboxManagerService::PodSandboxStatus(const std::string &podSandboxID, ++ runtime::v1::PodSandboxStatusResponse *reply, Errors &error) ++{ ++ if (reply == nullptr) { ++ ERROR("Invalid NULL reply"); ++ error.SetError("Invalid NULL reply"); ++ return; ++ } ++ ++ ++ auto podStatus = GetPodSandboxStatus(podSandboxID, error); ++ if (error.NotEmpty()) { ++ ERROR("Failed to get pod sandbox status: %s", error.GetCMessage()); ++ return; ++ } ++ ++ auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(podSandboxID); ++ if (sandbox == nullptr) { ++ ERROR("Failed to find sandbox id %s", podSandboxID.c_str()); ++ error.Errorf("Failed to find sandbox id %s", podSandboxID.c_str()); ++ return; ++ } ++ ++ *(reply->mutable_status()) = *podStatus; ++ ++ ++ if (!m_enablePodEvents) { ++ return; ++ } ++ ++ std::vector> containerStatuses; ++ std::vector errors; ++ GetContainerStatuses(sandbox->GetId(), containerStatuses, errors); ++ if (errors.size() != 0) { ++ error.SetAggregate(errors); ++ return; ++ } ++ ++ for (auto &containerStatus : containerStatuses) { ++ *(reply->add_containers_statuses()) = *containerStatus; ++ } ++ return; ++} ++ + void PodSandboxManagerService::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, + Errors &error) +@@ -944,7 +1024,7 @@ void PodSandboxManagerService::GetPodSandboxNetworkMetrics(const std::string &ne + void PodSandboxManagerService::PackagePodSandboxStatsAttributes( + const std::string &id, std::unique_ptr &podStatsPtr, Errors &error) + { +- auto status = PodSandboxStatus(id, error); ++ auto status = GetPodSandboxStatus(id, error); + if (error.NotEmpty()) { + return; + } +@@ -1111,8 +1191,8 @@ auto PodSandboxManagerService::PodSandboxStats(const std::string &podSandboxID, + auto &config = sandbox->GetSandboxConfig(); + auto oldStatsRec = sandbox->GetStatsInfo(); + +- auto status = PodSandboxStatus(sandbox->GetId(), tmpErr); +- if (error.NotEmpty()) { ++ auto status = GetPodSandboxStatus(sandbox->GetId(), tmpErr); ++ if (tmpErr.NotEmpty()) { + ERROR("Failed to get podsandbox %s status: %s", sandbox->GetId().c_str(), tmpErr.GetCMessage()); + error.Errorf("Failed to get podsandbox %s status", sandbox->GetId().c_str()); + return nullptr; +diff --git a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +index c3d98b8c..3872c4c9 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_pod_sandbox_manager_service.h +@@ -38,10 +38,11 @@ namespace CRIV1 { + class PodSandboxManagerService { + public: + PodSandboxManagerService(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager) ++ std::shared_ptr pluginManager, bool enablePodEvents) + : m_podSandboxImage(podSandboxImage) + , m_cb(cb) + , m_pluginManager(pluginManager) ++ , m_enablePodEvents(enablePodEvents) + { + } + PodSandboxManagerService(const PodSandboxManagerService &) = delete; +@@ -55,8 +56,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error); + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error); + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error); +@@ -129,6 +129,9 @@ private: + std::vector &podSandboxIDs, Errors &error); + void ApplySandboxLinuxOptions(const runtime::v1::LinuxPodSandboxConfig &lc, host_config *hc, + container_config *custom_config, Errors &error); ++ auto GetPodSandboxStatus(const std::string &podSandboxID, Errors &error) -> std::unique_ptr; ++ void GetContainerStatuses(const std::string &podSandboxID, std::vector> &containerStatuses, ++ std::vector &errors); + + private: + std::string m_podSandboxImage; +@@ -136,6 +139,7 @@ private: + std::map m_networkReady; + service_executor_t *m_cb { nullptr }; + std::shared_ptr m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRI + +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +index 839f6724..4521e3df 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service.h +@@ -70,8 +70,8 @@ public: + + virtual void RemovePodSandbox(const std::string &podSandboxID, Errors &error) = 0; + +- virtual auto PodSandboxStatus(const std::string &podSandboxID, +- Errors &error) -> std::unique_ptr = 0; ++ virtual void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, ++ Errors &error) = 0; + + virtual void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error) = 0; +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +index aa5ae516..7b40e29d 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.cc +@@ -19,11 +19,12 @@ + + namespace CRIV1 { + CRIRuntimeServiceImpl::CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager) ++ std::shared_ptr pluginManager, bool enablePodEvents) + : m_runtimeVersioner(new RuntimeVersionerService(cb)) + , m_containerManager(new ContainerManagerService(cb)) +- , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager)) ++ , m_podSandboxManager(new PodSandboxManagerService(podSandboxImage, cb, pluginManager, enablePodEvents)) + , m_runtimeManager(new RuntimeManagerService(cb, pluginManager)) ++ , m_enablePodEvents(enablePodEvents) + { + } + +@@ -124,10 +125,9 @@ void CRIRuntimeServiceImpl::RemovePodSandbox(const std::string &podSandboxID, Er + m_podSandboxManager->RemovePodSandbox(podSandboxID, error); + } + +-auto CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, Errors &error) +--> std::unique_ptr ++void CRIRuntimeServiceImpl::PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) + { +- return m_podSandboxManager->PodSandboxStatus(podSandboxID, error); ++ m_podSandboxManager->PodSandboxStatus(podSandboxID, reply, error); + } + + void CRIRuntimeServiceImpl::ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, +diff --git a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +index 0a25749f..6ae59bfa 100644 +--- a/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h ++++ b/src/daemon/entry/cri/v1/v1_cri_runtime_service_impl.h +@@ -26,7 +26,8 @@ namespace CRIV1 { + class CRIRuntimeServiceImpl : public CRIRuntimeService { + public: + CRIRuntimeServiceImpl(const std::string &podSandboxImage, service_executor_t *cb, +- std::shared_ptr pluginManager); ++ std::shared_ptr pluginManager, ++ bool enablePodEvents); + CRIRuntimeServiceImpl(const CRIRuntimeServiceImpl &) = delete; + auto operator=(const CRIRuntimeServiceImpl &) -> CRIRuntimeServiceImpl & = delete; + virtual ~CRIRuntimeServiceImpl() = default; +@@ -72,8 +73,7 @@ public: + + void RemovePodSandbox(const std::string &podSandboxID, Errors &error) override; + +- auto PodSandboxStatus(const std::string &podSandboxID, Errors &error) +- -> std::unique_ptr override; ++ void PodSandboxStatus(const std::string &podSandboxID, runtime::v1::PodSandboxStatusResponse *reply, Errors &error) override; + + void ListPodSandbox(const runtime::v1::PodSandboxFilter &filter, + std::vector> &pods, Errors &error) override; +@@ -103,6 +103,7 @@ protected: + private: + std::string m_podSandboxImage; + std::shared_ptr m_pluginManager { nullptr }; ++ bool m_enablePodEvents; + }; + } // namespace CRIV1 + #endif // DAEMON_ENTRY_CRI_V1_CRI_RUNTIME_SERVICE_IMPL_H +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..e5c96628 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -62,6 +62,7 @@ + #include "event_type.h" + #include "utils_timestamp.h" + #include "utils_verify.h" ++#include "mailbox.h" + #ifdef ENABLE_NATIVE_NETWORK + #include "service_network_api.h" + +@@ -542,6 +543,9 @@ static int container_start_cb(const container_start_request *request, container_ + container_t *cont = NULL; + int sync_fd = -1; + pthread_t thread_id = 0; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -596,6 +600,15 @@ static int container_start_cb(const container_start_request *request, container_ + EVENT("Event: {Object: %s, Type: Running}", id); + (void)isulad_monitor_send_container_event(id, START, -1, 0, NULL, NULL); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_STARTED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + handle_start_io_thread_by_cc(cc, sync_fd, thread_id); + delete_daemon_fifos(fifopath, (const char **)fifos); +@@ -1009,6 +1022,9 @@ static int container_delete_cb(const container_delete_request *request, containe + char *name = NULL; + char *id = NULL; + container_t *cont = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + if (request == NULL || response == NULL) { +@@ -1063,6 +1079,15 @@ static int container_delete_cb(const container_delete_request *request, containe + + EVENT("Event: {Object: %s, Type: Deleted}", id); + ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ message.container_id = cont->common_config->id; ++ message.sandbox_id = cont->common_config->sandbox_info->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_DELETED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif ++ + pack_response: + pack_delete_response(*response, cc, id); + container_unref(cont); +diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c +index e00afb68..a9102226 100644 +--- a/src/daemon/executor/container_cb/execution_create.c ++++ b/src/daemon/executor/container_cb/execution_create.c +@@ -62,6 +62,7 @@ + #include "opt_log.h" + #include "runtime_api.h" + #include "id_name_manager.h" ++#include "mailbox.h" + + #ifdef ENABLE_CRI_API_V1 + static bool validate_sandbox_info(const container_sandbox_info *sandbox) +@@ -1389,6 +1390,9 @@ int container_create_cb(const container_create_request *request, container_creat + bool skip_id_name_manage = false; + bool skip_sandbox_key_manage = false; + __isula_auto_sysinfo_t sysinfo_t *sysinfo = NULL; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t message; ++#endif + + DAEMON_CLEAR_ERRMSG(); + +@@ -1572,6 +1576,14 @@ int container_create_cb(const container_create_request *request, container_creat + + EVENT("Event: {Object: %s, Type: Created %s}", id, name); + (void)isulad_monitor_send_container_event(id, CREATE, -1, 0, NULL, NULL); ++#ifdef ENABLE_CRI_API_V1 ++ if (is_container_in_sandbox(request->sandbox)) { ++ message.container_id = id; ++ message.sandbox_id = request->sandbox->id; ++ message.type = CRI_CONTAINER_MESSAGE_TYPE_CREATED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &message); ++ } ++#endif + goto pack_response; + + umount_channel: +diff --git a/src/daemon/mailbox/CMakeLists.txt b/src/daemon/mailbox/CMakeLists.txt +new file mode 100644 +index 00000000..984f9acb +--- /dev/null ++++ b/src/daemon/mailbox/CMakeLists.txt +@@ -0,0 +1,11 @@ ++# get current directory sources files ++aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} mailbox_top_srcs) ++ ++set(MAILBOX_SRCS ++ ${mailbox_top_srcs} ++ PARENT_SCOPE ++ ) ++set(MAILBOX_INCS ++ ${CMAKE_CURRENT_SOURCE_DIR} ++ PARENT_SCOPE ++ ) +\ No newline at end of file +diff --git a/src/daemon/mailbox/mailbox.c b/src/daemon/mailbox/mailbox.c +new file mode 100644 +index 00000000..732b91b9 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.c +@@ -0,0 +1,167 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#include "mailbox.h" ++ ++#include ++ ++#include "message_queue.h" ++#include "mailbox_message.h" ++#include "message_subscriber.h" ++ ++mailbox_topic_handler_t mailbox_topic_handlers[MAILBOX_TOPIC_MAX] = { 0 }; ++ ++static bool mailbox_topic_valid(mailbox_topic topic) { ++ return topic > MAILBOX_TOPIC_INVALID && topic < MAILBOX_TOPIC_MAX; ++} ++ ++static bool mailbox_should_publish(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return false; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ return false; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ return true; ++ } ++ ++ // for async queues, only publish if anyone subscribe ++ return message_queue_have_subscribers(mailbox_topic_handlers[topic].queue); ++} ++ ++// only register once when iSulad start, no need to free the queue ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t generator, void *context, ++ message_release_t release, bool async) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return -1; ++ } ++ ++ if (generator == NULL) { ++ ERROR("Invalid generator for topic %d", topic); ++ return -1; ++ } ++ ++ mailbox_topic_handlers[topic].generator = generator; ++ mailbox_topic_handlers[topic].context = context; ++ mailbox_topic_handlers[topic].release = release; ++ if (async) { ++ mailbox_topic_handlers[topic].queue = message_queue_create(release); ++ if (mailbox_topic_handlers[topic].queue == NULL) { ++ ERROR("Failed to create message queue for topic %d", topic); ++ return -1; ++ } ++ } ++ mailbox_topic_handlers[topic].registered = true; ++ return 0; ++} ++ ++// unregister only when iSulad shutdown, no need to free the queue ++void mailbox_unregister_topic_handler(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ message_queue_shutdown(mailbox_topic_handlers[topic].queue); ++ } ++ mailbox_topic_handlers[topic].registered = false; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (!mailbox_should_publish(topic)) { ++ return; ++ } ++ ++ message_generator_t generator = mailbox_topic_handlers[topic].generator; ++ void *context = mailbox_topic_handlers[topic].context; ++ message_release_t release = mailbox_topic_handlers[topic].release; ++ message_queue *queue = mailbox_topic_handlers[topic].queue; ++ ++ if (generator == NULL) { ++ ERROR("No handler for topic %d", topic); ++ return; ++ } ++ ++ void *middle = generator(context, data); ++ if (middle == NULL) { ++ return; ++ } ++ ++ if (queue != NULL) { ++ mailbox_message *msg = mailbox_message_create(middle, release); ++ if (msg == NULL) { ++ ERROR("Failed to create mailbox message"); ++ if (release) { ++ release(middle); ++ } ++ return; ++ } ++ if (message_queue_publish(queue, msg) != 0) { ++ ERROR("Failed to publish event"); ++ mailbox_message_unref(msg); ++ return; ++ } ++ } ++} ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return NULL; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return NULL; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_subscribe(mailbox_topic_handlers[topic].queue, ++ mailbox_topic_handlers[topic].release); ++ } ++ ++ // For sync queues, there is no need to subscribe, just return ++ return NULL; ++} ++ ++void mailbox_unsubscribe(mailbox_topic topic, message_subscriber *sub) ++{ ++ if (!mailbox_topic_valid(topic)) { ++ ERROR("Invalid topic %d", topic); ++ return; ++ } ++ ++ if (!mailbox_topic_handlers[topic].registered) { ++ ERROR("Handler for topic %d not registered", topic); ++ return; ++ } ++ ++ if (mailbox_topic_handlers[topic].queue != NULL) { ++ return message_queue_unsubscribe(mailbox_topic_handlers[topic].queue, sub); ++ } ++ ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox.h b/src/daemon/mailbox/mailbox.h +new file mode 100644 +index 00000000..1dc2e934 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox.h +@@ -0,0 +1,82 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide common event definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_H ++#define DAEMON_MAILBOX_MAILBOX_H ++ ++#include "daemon_arguments.h" ++#include "blocking_queue.h" ++#include "message_queue.h" ++#include "message_subscriber.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++typedef enum { ++ MAILBOX_TOPIC_INVALID = -1, ++ MAILBOX_TOPIC_CRI_CONTAINER, ++ MAILBOX_TOPIC_MAX ++} mailbox_topic; ++ ++// for async message, it generates a true message to publish ++// for sync message, it is the callback function to handle the data to publish ++typedef void *(*message_generator_t)(void *, void *); ++// release function of message generated by generator, if any ++typedef void (*message_release_t)(void *); ++ ++typedef struct { ++ // to generate a message ++ message_generator_t generator; ++ // context of handler ++ void *context; ++ // release function of message, if any ++ message_release_t release; ++ // message queue ++ message_queue *queue; ++ // if registered ++ bool registered; ++} mailbox_topic_handler_t; ++ ++typedef enum { ++ CRI_CONTAINER_MESSAGE_TYPE_INVALID = -1, ++ CRI_CONTAINER_MESSAGE_TYPE_CREATED, ++ CRI_CONTAINER_MESSAGE_TYPE_STARTED, ++ CRI_CONTAINER_MESSAGE_TYPE_STOPPED, ++ CRI_CONTAINER_MESSAGE_TYPE_DELETED, ++ CRI_CONTAINER_MESSAGE_TYPE_MAX ++} cri_container_message_type; ++ ++typedef struct { ++ const char *container_id; ++ const char *sandbox_id; ++ cri_container_message_type type; ++} cri_container_message_t; ++ ++int mailbox_register_topic_handler(mailbox_topic topic, message_generator_t handle, void *context, ++ message_release_t release, bool async); ++ ++void mailbox_unregister_topic_handler(mailbox_topic topic); ++ ++void mailbox_publish(mailbox_topic topic, void *data); ++ ++message_subscriber *mailbox_subscribe(mailbox_topic topic); ++ ++void mailbox_unsubscribe(mailbox_topic, message_subscriber *sub); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/mailbox_message.c b/src/daemon/mailbox/mailbox_message.c +new file mode 100644 +index 00000000..b16a1bdd +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.c +@@ -0,0 +1,94 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide mailbox message definition ++ ******************************************************************************/ ++ ++#include "mailbox_message.h" ++ ++#include ++ ++#include "utils.h" ++ ++// Once the create succeeds, the ownership is transferred to the mailbox_message. ++mailbox_message *mailbox_message_create(void *data, void (*destroy)(void *)) { ++ __isula_auto_free mailbox_message *msg = NULL; ++ msg = util_common_calloc_s(sizeof(mailbox_message)); ++ if (msg == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ msg->data = data; ++ msg->destroy = destroy; ++ msg->ref_count = 1; ++ ++ if (pthread_mutex_init(&msg->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(msg); ++} ++ ++int mailbox_message_ref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ ERROR("Invalid mailbox_message"); ++ return -1; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == INT_MAX) { ++ ERROR("Reference count overflow"); ++ return -1; ++ } ++ ++ dest->ref_count++; ++ ++ return 0; ++} ++ ++void mailbox_message_unref(mailbox_message *dest) { ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (dest == NULL) { ++ return; ++ } ++ ++ if (pthread_mutex_lock(&dest->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &dest->lock; ++ ++ if (dest->ref_count == 0) { ++ ERROR("Reference count underflow, should not reach here"); ++ return; ++ } ++ ++ dest->ref_count--; ++ if (dest->ref_count == 0) { ++ if (dest->destroy) { ++ dest->destroy(dest->data); ++ } ++ lock = NULL; ++ (void)pthread_mutex_unlock(&dest->lock); ++ (void)pthread_mutex_destroy(&dest->lock); ++ free(dest); ++ } ++ return; ++} +diff --git a/src/daemon/mailbox/mailbox_message.h b/src/daemon/mailbox/mailbox_message.h +new file mode 100644 +index 00000000..39e40b70 +--- /dev/null ++++ b/src/daemon/mailbox/mailbox_message.h +@@ -0,0 +1,50 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide ref counted ptr definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++#define DAEMON_MAILBOX_MAILBOX_MESSAGE_H ++ ++#include ++#include ++ ++#include ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++typedef struct mailbox_message { ++ void *data; ++ size_t ref_count; ++ pthread_mutex_t lock; ++ void (*destroy)(void *); ++} mailbox_message; ++ ++mailbox_message *mailbox_message_create(void *ptr, void (*destroy)(void *)); ++ ++int mailbox_message_ref(mailbox_message *p); ++ ++void mailbox_message_unref(mailbox_message *p); ++ ++// define auto free function callback for mailbox_message ++define_auto_cleanup_callback(mailbox_message_unref, mailbox_message); ++// define auto free macro for char * ++#define __isula_auto_mailbox_message auto_cleanup_tag(mailbox_message_unref) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +new file mode 100644 +index 00000000..7fe044f2 +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.c +@@ -0,0 +1,234 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#include "message_queue.h" ++ ++#include ++#include ++ ++#include "utils.h" ++ ++// default set subscriber timeout to 1000ms, maybe could be configured later ++const int64_t subscribe_timeout = 1000; ++ ++static void message_queue_subscriber_free(void *key, void *val) ++{ ++ return; ++} ++ ++static void *message_queue_thread(void *arg) ++{ ++ int ret = 0; ++ ++ ret = pthread_detach(pthread_self()); ++ if (ret != 0) { ++ CRIT("Set thread detach fail"); ++ return NULL; ++ } ++ ++ prctl(PR_SET_NAME, "Message Queue"); ++ ++ message_queue *mq = (message_queue *)arg; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ for (;;) { ++ void *data = NULL; ++ if (blocking_queue_pop(mq->messages, &data) != 0) { ++ ERROR("Fail to get message, message queue thread exit"); ++ break; ++ } ++ ++ __isula_auto_mailbox_message mailbox_message *msg = (mailbox_message *)data; ++ // an empty msg indicates shutdown ++ if (pthread_rwlock_rdlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ continue; ++ } ++ ++ bool should_shutdown = (msg == NULL); ++ map_itor *itor = map_itor_new(mq->subscribers); ++ if (itor == NULL) { ++ ERROR("Out of memory"); ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ } ++ break; ++ } ++ ++ for (; map_itor_valid(itor); map_itor_next(itor)) { ++ void *sub = map_itor_key(itor); ++ if (should_shutdown) { ++ message_subscriber_shutdown(sub); ++ } else { ++ if (message_subscriber_push(sub, msg) != 0) { ++ ERROR("Failed to push event to subscriber"); ++ } ++ } ++ } ++ map_itor_free(itor); ++ ++ if (pthread_rwlock_unlock(&mq->rwlock) != 0) { ++ ERROR("Failed to unlock rwlock"); ++ } ++ ++ // if msg is NULL, it is a shutdown signal ++ if (should_shutdown) { ++ break; ++ } ++ } ++ ++ return NULL; ++} ++ ++message_queue *message_queue_create(void (*release)(void *)) ++{ ++ __isula_auto_free message_queue *mq = NULL; ++ __isula_auto_blocking_queue blocking_queue *bq = NULL; ++ pthread_t message_queue_tid; ++ mq = util_common_calloc_s(sizeof(message_queue)); ++ if (mq == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (bq == NULL) { ++ ERROR("Failed to create events queue"); ++ return NULL; ++ } ++ ++ mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); ++ if (mq->subscribers == NULL) { ++ ERROR("Failed to create subscribers map"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_init(&mq->rwlock, NULL) != 0) { ++ ERROR("Failed to init rwlock"); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ if (pthread_create(&message_queue_tid, NULL, message_queue_thread, mq) != 0) { ++ ERROR("Failed to create message queue thread"); ++ pthread_rwlock_destroy(&mq->rwlock); ++ map_free(mq->subscribers); ++ return NULL; ++ } ++ ++ mq->messages = isula_transfer_ptr(bq); ++ return isula_transfer_ptr(mq); ++} ++ ++// message queue should be global value, it will be destroyed when daemon exit ++void message_queue_shutdown(message_queue *mq) ++{ ++ if (mq == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(mq->messages); ++ ++ // push a nullptr to notify the thread to exit ++ if (blocking_queue_push(mq->messages, NULL) != 0) { ++ ERROR("Failed to push nullptr to message queue"); ++ } ++} ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)) ++{ ++ __isula_auto_subscriber message_subscriber *sub = NULL; ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ int val = 0; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return NULL; ++ } ++ ++ sub = message_subscriber_create(subscribe_timeout, release); ++ if (sub == NULL) { ++ ERROR("Failed to create subscriber"); ++ return NULL; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return NULL; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_insert(mq->subscribers, sub, (void *)&val) == false) { ++ ERROR("Failed to insert subscriber"); ++ return NULL; ++ } ++ ++ return isula_transfer_ptr(sub); ++} ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return; ++ } ++ lock = &mq->rwlock; ++ ++ if (map_remove(mq->subscribers, sub) == false) { ++ ERROR("Failed to remove subscriber"); ++ return; ++ } ++ ++ return; ++} ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg) ++{ ++ if (mq == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(mq->messages, msg) != 0) { ++ ERROR("Failed to push message"); ++ return -1; ++ } ++ return 0; ++} ++ ++bool message_queue_have_subscribers(message_queue *mq) ++{ ++ __isula_auto_prw_unlock pthread_rwlock_t *lock = NULL; ++ if (mq == NULL) { ++ ERROR("Invalid argument"); ++ return false; ++ } ++ ++ if (pthread_rwlock_wrlock(&mq->rwlock) != 0) { ++ ERROR("Failed to lock rwlock"); ++ return false; ++ } ++ lock = &mq->rwlock; ++ ++ return map_size(mq->subscribers) > 0; ++} +diff --git a/src/daemon/mailbox/message_queue.h b/src/daemon/mailbox/message_queue.h +new file mode 100644 +index 00000000..7905840f +--- /dev/null ++++ b/src/daemon/mailbox/message_queue.h +@@ -0,0 +1,57 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_MESSAGE_MESSAGE_QUEUE_H ++#define DAEMON_MESSAGE_MESSAGE_QUEUE_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++#include "map.h" ++#include "message_subscriber.h" ++ ++typedef struct message_queue { ++ blocking_queue *messages; ++ ++ // lock for set of subscribers ++ pthread_rwlock_t rwlock; ++ ++ map_t *subscribers; ++ ++ int64_t sub_timeout; ++} message_queue; ++ ++message_queue *message_queue_create(void (*release)(void *)); ++ ++void message_queue_shutdown(message_queue *mq); ++ ++message_subscriber *message_queue_subscribe(message_queue *mq, void (*release)(void *)); ++ ++void message_queue_unsubscribe(message_queue *mq, message_subscriber *sub); ++ ++int message_queue_publish(message_queue *mq, mailbox_message *msg); ++ ++bool message_queue_have_subscribers(message_queue *mq); ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/src/daemon/mailbox/message_subscriber.c b/src/daemon/mailbox/message_subscriber.c +new file mode 100644 +index 00000000..8ef3cb58 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.c +@@ -0,0 +1,85 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#include "message_subscriber.h" ++ ++#include ++ ++#include "utils.h" ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)) ++{ ++ message_subscriber *sub = (message_subscriber *)util_common_calloc_s(sizeof(message_subscriber)); ++ if (sub == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ sub->queue = blocking_queue_create(timeout, release); ++ if (sub->queue == NULL) { ++ ERROR("Failed to create blocking queue"); ++ free(sub); ++ return NULL; ++ } ++ return sub; ++} ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg) ++{ ++ if (sub == NULL || msg == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ ++ if (mailbox_message_ref(msg) != 0) { ++ ERROR("Failed to get message"); ++ return -1; ++ } ++ ++ if (blocking_queue_push(sub->queue, msg) != 0) { ++ ERROR("Failed to push message to queue"); ++ mailbox_message_unref(msg); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg) ++{ ++ if (sub == NULL) { ++ ERROR("Invalid argument"); ++ return -1; ++ } ++ return blocking_queue_pop(sub->queue, (void **)msg); ++} ++ ++void message_subscriber_shutdown(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(sub->queue); ++ (void)blocking_queue_push(sub->queue, NULL); ++} ++ ++void message_subscriber_destroy(message_subscriber *sub) ++{ ++ if (sub == NULL) { ++ return; ++ } ++ blocking_queue_destroy(sub->queue); ++ free(sub); ++} +diff --git a/src/daemon/mailbox/message_subscriber.h b/src/daemon/mailbox/message_subscriber.h +new file mode 100644 +index 00000000..de4574d9 +--- /dev/null ++++ b/src/daemon/mailbox/message_subscriber.h +@@ -0,0 +1,41 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide message subscriber definition ++ ******************************************************************************/ ++ ++#ifndef SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++#define SRC_DAEMON_MAILBOX_MESSAGE_SUBSCRIBER_H ++ ++#include "blocking_queue.h" ++#include "mailbox_message.h" ++ ++typedef struct { ++ blocking_queue *queue; ++} message_subscriber; ++ ++message_subscriber *message_subscriber_create(int64_t timeout, void (*release)(void *)); ++ ++void message_subscriber_shutdown(message_subscriber *sub); ++ ++void message_subscriber_destroy(message_subscriber *sub); ++ ++int message_subscriber_push(message_subscriber *sub, mailbox_message *msg); ++ ++int message_subscriber_pop(message_subscriber *sub, mailbox_message **msg); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(message_subscriber_destroy, message_subscriber); ++// define auto free macro for blocking queue ++#define __isula_auto_subscriber auto_cleanup_tag(message_subscriber_destroy) ++ ++#endif +diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h +index 830fd696..55c59980 100644 +--- a/src/daemon/modules/api/container_api.h ++++ b/src/daemon/modules/api/container_api.h +@@ -289,6 +289,11 @@ static inline bool is_sandbox_container(container_sandbox_info *sandbox) + { + return sandbox != NULL && sandbox->is_sandbox_container; + } ++ ++static inline bool is_container_in_sandbox(container_sandbox_info *sandbox) ++{ ++ return sandbox != NULL && !sandbox->is_sandbox_container; ++} + #endif + + #if defined(__cplusplus) || defined(c_plusplus) +diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c +index 1b7da383..83d46268 100644 +--- a/src/daemon/modules/container/supervisor/supervisor.c ++++ b/src/daemon/modules/container/supervisor/supervisor.c +@@ -38,6 +38,7 @@ + #include "container_api.h" + #include "event_type.h" + #include "utils_file.h" ++#include "mailbox.h" + #ifdef ENABLE_CRI_API_V1 + #include "sandbox_ops.h" + #endif +@@ -51,6 +52,7 @@ struct supervisor_handler_data { + int fd; + int exit_code; + char *name; ++ char *sandbox_name; + char *runtime; + bool is_sandbox_container; + pid_ppid_info_t pid_info; +@@ -152,6 +154,9 @@ static void supervisor_handler_data_free(struct supervisor_handler_data *data) + free(data->name); + data->name = NULL; + ++ free(data->sandbox_name); ++ data->sandbox_name = NULL; ++ + free(data->runtime); + data->runtime = NULL; + +@@ -172,6 +177,9 @@ static void *clean_resources_thread(void *arg) + pid_t pid = data->pid_info.pid; + int retry_count = 0; + int max_retry = 10; ++#ifdef ENABLE_CRI_API_V1 ++ cri_container_message_t msg; ++#endif + + ret = pthread_detach(pthread_self()); + if (ret != 0) { +@@ -218,6 +226,13 @@ retry: + (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL); + + #ifdef ENABLE_CRI_API_V1 ++ if (data->sandbox_name) { ++ msg.container_id = name; ++ msg.sandbox_id = data->sandbox_name; ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } ++ + if (data->is_sandbox_container) { + if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) { + ERROR("Failed to handle sandbox %s exit", name); +@@ -329,6 +344,9 @@ int container_supervisor_add_exit_monitor(int fd, const char *exit_fifo, const p + data->runtime = util_strdup_s(cont->runtime); + #ifdef ENABLE_CRI_API_V1 + data->is_sandbox_container = is_sandbox_container(cont->common_config->sandbox_info); ++ if (is_container_in_sandbox(cont->common_config->sandbox_info)) { ++ data->sandbox_name = util_strdup_s(cont->common_config->sandbox_info->id); ++ } + #endif + data->pid_info.pid = pid_info->pid; + data->pid_info.start_time = pid_info->start_time; +diff --git a/src/daemon/sandbox/sandbox.cc b/src/daemon/sandbox/sandbox.cc +index 7b6496ed..c70116c1 100644 +--- a/src/daemon/sandbox/sandbox.cc ++++ b/src/daemon/sandbox/sandbox.cc +@@ -37,6 +37,7 @@ + #include "cxxutils.h" + #include "controller_manager.h" + #include "utils_timestamp.h" ++#include "mailbox.h" + + #define SANDBOX_READY_STATE_STR "SANDBOX_READY" + #define SANDBOX_NOTREADY_STATE_STR "SANDBOX_NOTREADY" +@@ -527,6 +528,14 @@ void Sandbox::OnSandboxExit(const ControllerExitInfo &exitInfo) + if (!SaveState(error)) { + ERROR("Failed to save sandbox state, %s", m_id.c_str()); + } ++ ++ if (error.Empty()) { ++ cri_container_message_t msg = { 0 }; ++ msg.container_id = GetId().c_str(); ++ msg.sandbox_id = GetId().c_str(); ++ msg.type = CRI_CONTAINER_MESSAGE_TYPE_STOPPED; ++ mailbox_publish(MAILBOX_TOPIC_CRI_CONTAINER, &msg); ++ } + } + + auto Sandbox::UpdateStatus(Errors &error) -> bool +diff --git a/src/utils/cutils/blocking_queue.c b/src/utils/cutils/blocking_queue.c +new file mode 100644 +index 00000000..7c9c5f50 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.c +@@ -0,0 +1,185 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#include "blocking_queue.h" ++ ++#include ++#include ++#include ++ ++#include "utils.h" ++#include "utils_timestamp.h" ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)) ++{ ++ __isula_auto_free blocking_queue *queue = NULL; ++ __isula_auto_free blocking_node *node = NULL; ++ queue = (blocking_queue *)util_common_calloc_s(sizeof(blocking_queue)); ++ if (queue == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (node == NULL) { ++ ERROR("Out of memory"); ++ return NULL; ++ } ++ ++ if (pthread_mutex_init(&queue->lock, NULL) != 0) { ++ ERROR("Failed to init mutex"); ++ return NULL; ++ } ++ ++ if (pthread_cond_init(&queue->not_empty, NULL) != 0) { ++ ERROR("Failed to init cond"); ++ (void)pthread_mutex_destroy(&queue->lock); ++ return NULL; ++ } ++ ++ queue->head = node; ++ queue->tail = node; ++ node = NULL; ++ queue->release = release; ++ ++ if (timeout >= 0) { ++ queue->timeout.tv_sec = timeout / (Time_Second / Time_Milli); ++ queue->timeout.tv_nsec = (timeout % (Time_Second / Time_Milli) ) * Time_Milli; ++ } else { ++ queue->timeout.tv_sec = -1; ++ } ++ ++ return isula_transfer_ptr(queue); ++} ++ ++int blocking_queue_push(blocking_queue *queue, void *data) ++{ ++ __isula_auto_free blocking_node *new_node = NULL; ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ new_node = (blocking_node *)util_common_calloc_s(sizeof(blocking_node)); ++ if (new_node == NULL) { ++ ERROR("Out of memory"); ++ return -1; ++ } ++ new_node->data = data; ++ new_node->next = NULL; ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ queue->tail->next = new_node; ++ queue->tail = new_node; ++ new_node = NULL; ++ ++ if (pthread_cond_broadcast(&queue->not_empty) != 0) { ++ ERROR("Failed to broadcast cond"); ++ } ++ ++ return 0; ++} ++ ++int blocking_queue_pop(blocking_queue *queue, void **data) { ++ if (queue == NULL || data == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return -1; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return -1; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next == NULL) { ++ if (queue->timeout.tv_sec >= 0) { ++ int ret = pthread_cond_timedwait(&queue->not_empty, &queue->lock, &queue->timeout); ++ if (ret != 0) { ++ if (ret != ETIMEDOUT) { ++ ERROR("Failed to wait cond"); ++ } ++ return ret; ++ } ++ } else { ++ int ret = pthread_cond_wait(&queue->not_empty, &queue->lock); ++ if (ret != 0) { ++ ERROR("Failed to wait cond"); ++ return ret; ++ } ++ } ++ } ++ ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ *data = new_head->data; ++ queue->head = new_head; ++ ++ free(old_head); ++ return 0; ++} ++ ++void blocking_queue_clear(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ __isula_auto_pm_unlock pthread_mutex_t *lock = NULL; ++ // clear all nodes in queue ++ if (queue == NULL) { ++ ERROR("Invalid NULL arguments"); ++ return; ++ } ++ ++ if (pthread_mutex_lock(&queue->lock) != 0) { ++ ERROR("Failed to lock mutex"); ++ return; ++ } ++ lock = &queue->lock; ++ ++ while (queue->head->next != NULL) { ++ blocking_node *old_head = queue->head; ++ blocking_node *new_head = old_head->next; ++ if (queue->release) { ++ queue->release(old_head->data); ++ } ++ free(old_head); ++ queue->head = new_head; ++ } ++} ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue) ++{ ++ if (queue == NULL) { ++ return; ++ } ++ ++ blocking_queue_clear(queue); ++ ++ (void)pthread_mutex_destroy(&queue->lock); ++ ++ (void)pthread_cond_destroy(&queue->not_empty); ++ ++ free(queue); ++} +diff --git a/src/utils/cutils/blocking_queue.h b/src/utils/cutils/blocking_queue.h +new file mode 100644 +index 00000000..1c52a9d3 +--- /dev/null ++++ b/src/utils/cutils/blocking_queue.h +@@ -0,0 +1,66 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-03-25 ++ * Description: provide blocking queue definition ++ ******************************************************************************/ ++ ++#ifndef DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++#define DAEMON_UTILS_CUTILS_BLOCKING_QUEUE_H ++ ++#include ++#include ++#include ++ ++#include "utils_timestamp.h" ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#define BLOCKING_QUEUE_NO_TIMEOUT -1 ++ ++typedef struct blocking_node { ++ void *data; ++ struct blocking_node *next; ++} blocking_node; ++ ++typedef struct blocking_queue { ++ blocking_node *head; ++ blocking_node *tail; ++ pthread_mutex_t lock; ++ struct timespec timeout; ++ pthread_cond_t not_empty; ++ void (*release)(void *); ++} blocking_queue; ++ ++// create blocking queue with timeout(ms), if timeout < 0, then with no timeout ++blocking_queue *blocking_queue_create(int64_t timeout, void (*release)(void *)); ++ ++int blocking_queue_push(blocking_queue *queue, void *data); ++ ++int blocking_queue_pop(blocking_queue *queue, void **data); ++ ++void blocking_queue_clear(blocking_queue *queue); ++ ++// ensure there is no other thread executing enqueue or dequeue operation ++void blocking_queue_destroy(blocking_queue *queue); ++ ++// define auto free function callback for blocking queue ++define_auto_cleanup_callback(blocking_queue_destroy, blocking_queue); ++// define auto free macro for blocking queue ++#define __isula_auto_blocking_queue auto_cleanup_tag(blocking_queue_destroy) ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif +diff --git a/test/mocks/mailbox_mock.cc b/test/mocks/mailbox_mock.cc +new file mode 100644 +index 00000000..601b804e +--- /dev/null ++++ b/test/mocks/mailbox_mock.cc +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#include "mailbox_mock.h" ++ ++MockMailbox *g_mailbox_mock = nullptr; ++ ++void Mailbox_SetMock(MockMailbox* mock) ++{ ++ g_mailbox_mock = mock; ++} ++ ++void mailbox_publish(mailbox_topic topic, void *data) ++{ ++ if (g_mailbox_mock != nullptr) { ++ g_mailbox_mock->MailboxPublish(topic, data); ++ } ++} +diff --git a/test/mocks/mailbox_mock.h b/test/mocks/mailbox_mock.h +new file mode 100644 +index 00000000..ce48f0fc +--- /dev/null ++++ b/test/mocks/mailbox_mock.h +@@ -0,0 +1,30 @@ ++/****************************************************************************** ++ * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved. ++ * iSulad licensed under the Mulan PSL v2. ++ * You can use this software according to the terms and conditions of the Mulan PSL v2. ++ * You may obtain a copy of Mulan PSL v2 at: ++ * http://license.coscl.org.cn/MulanPSL2 ++ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++ * PURPOSE. ++ * See the Mulan PSL v2 for more details. ++ * Author: jikai ++ * Create: 2024-04-02 ++ * Description: mailbox mock ++ ******************************************************************************/ ++ ++#ifndef _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++#define _ISULAD_TEST_MOCKS_MAILBOX_MOCK_H ++ ++#include ++#include "mailbox.h" ++ ++class MockMailbox { ++public: ++ virtual ~MockMailbox() = default; ++ MOCK_METHOD2(MailboxPublish, void(mailbox_topic topic, void *data)); ++}; ++ ++void Mailbox_SetMock(MockMailbox* mock); ++ ++#endif +diff --git a/test/sandbox/controller/shim/CMakeLists.txt b/test/sandbox/controller/shim/CMakeLists.txt +index 6423bb80..26a66e51 100644 +--- a/test/sandbox/controller/shim/CMakeLists.txt ++++ b/test/sandbox/controller/shim/CMakeLists.txt +@@ -7,6 +7,7 @@ add_executable(${EXE} + ${CMAKE_BINARY_DIR}/grpc/src/api/services/cri/gogo.pb.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_helpers.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_cri_security_context.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/v1/v1_naming.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/checkpoint_handler.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_constants.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/daemon/common/cri/cri_helpers.cc +diff --git a/test/sandbox/sandbox/CMakeLists.txt b/test/sandbox/sandbox/CMakeLists.txt +index 138d4d8d..2a35388f 100644 +--- a/test/sandbox/sandbox/CMakeLists.txt ++++ b/test/sandbox/sandbox/CMakeLists.txt +@@ -23,6 +23,7 @@ add_executable(${EXE} + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/grpc_sandboxer_client_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/controller_stub_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/shim_controller_mock.cc ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../test/mocks/mailbox_mock.cc + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/common/err_msg.c + ${CMAKE_CURRENT_SOURCE_DIR}/sandbox_ut.cc) + +@@ -33,6 +34,7 @@ target_include_directories(${EXE} PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/config + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/entry/cri + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/executor ++ ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/mailbox + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/shim + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/daemon/sandbox/controller/sandboxer +-- +2.34.1 + diff --git a/0040-fix-cpurt-init-bug-for-systemd-cgroup.patch b/0040-fix-cpurt-init-bug-for-systemd-cgroup.patch new file mode 100644 index 0000000..ce92d43 --- /dev/null +++ b/0040-fix-cpurt-init-bug-for-systemd-cgroup.patch @@ -0,0 +1,74 @@ +From fe11b34a3c2843ea2198b310160b182d63aeb63b Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 11:22:09 +0800 +Subject: [PATCH 40/43] fix cpurt init bug for systemd-cgroup + +Signed-off-by: jikai +--- + src/daemon/common/cgroup/cgroup.c | 13 +++++++------ + src/daemon/executor/container_cb/execution.c | 13 +++++++------ + 2 files changed, 14 insertions(+), 12 deletions(-) + +diff --git a/src/daemon/common/cgroup/cgroup.c b/src/daemon/common/cgroup/cgroup.c +index d3f1445a..007dbb70 100644 +--- a/src/daemon/common/cgroup/cgroup.c ++++ b/src/daemon/common/cgroup/cgroup.c +@@ -146,17 +146,18 @@ char *common_convert_cgroup_path(const char *cgroup_path) + return NULL; + } + +- // for cgroup fs cgroup path, return directly +- if (!util_has_suffix(cgroup_path, ".slice")) { +- return util_strdup_s(cgroup_path); +- } +- + // for systemd cgroup, cgroup_path should have the form slice:prefix:id, + // convert it to a true path, such as from test-a.slice:isulad:id + // to test.slice/test-a.slice/isulad-id.scope + arr = util_string_split_n(cgroup_path, ':', 3); + if (arr == NULL || util_array_len((const char **)arr) != 3) { +- ERROR("Invalid systemd cgroup parent"); ++ // not a systemd cgroup, return cgroup path directly ++ return util_strdup_s(cgroup_path); ++ } ++ ++ // for cgroup fs cgroup path, return directly ++ if (!util_has_suffix(arr[0], ".slice")) { ++ ERROR("Invalid systemd cgroup path: %s", cgroup_path); + return NULL; + } + +diff --git a/src/daemon/executor/container_cb/execution.c b/src/daemon/executor/container_cb/execution.c +index 88c6b354..4bf3621d 100644 +--- a/src/daemon/executor/container_cb/execution.c ++++ b/src/daemon/executor/container_cb/execution.c +@@ -435,11 +435,12 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + } + + if (conf_get_systemd_cgroup()) { +- // currently it is the same as docker, yet it is unclear that +- // if systemd cgroup is used and cgroup parent is set to a slice rather than system.slice +- // should iSulad set cpu.rt_runtime_us and cpu.rt_period_us for the parent path? +- // in fact, even if system.slice is used, +- // cpu.rt_runtime_us and cpu.rt_period_us might still needed to be set manually ++ __isula_auto_free char *converted_cgroup = common_convert_cgroup_path(cgroups_path); ++ if (converted_cgroup == NULL) { ++ ERROR("Failed to convert cgroup path"); ++ return -1; ++ } ++ + __isula_auto_free char *init_cgroup = common_get_init_cgroup_path("cpu"); + if (init_cgroup == NULL) { + ERROR("Failed to get init cgroup"); +@@ -451,7 +452,7 @@ static int cpurt_controller_init(const char *id, const host_config *host_spec) + ERROR("Failed to get own cgroup"); + return -1; + } +- char *new_cgroups_path = util_path_join(init_cgroup, cgroups_path); ++ char *new_cgroups_path = util_path_join(init_cgroup, converted_cgroup); + if (new_cgroups_path == NULL) { + ERROR("Failed to join path"); + return -1; +-- +2.34.1 + diff --git a/0041-fix-message-queue-concurrent-bug.patch b/0041-fix-message-queue-concurrent-bug.patch new file mode 100644 index 0000000..12c9876 --- /dev/null +++ b/0041-fix-message-queue-concurrent-bug.patch @@ -0,0 +1,41 @@ +From f90a145d9d29682295aebf2bcd30865ee5f6491f Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 07:53:54 +0000 +Subject: [PATCH 41/43] fix message queue concurrent bug + +Signed-off-by: jikai +--- + src/daemon/mailbox/message_queue.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/src/daemon/mailbox/message_queue.c b/src/daemon/mailbox/message_queue.c +index 7fe044f2..7e53301e 100644 +--- a/src/daemon/mailbox/message_queue.c ++++ b/src/daemon/mailbox/message_queue.c +@@ -106,11 +106,12 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- bq = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); +- if (bq == NULL) { ++ mq->messages = blocking_queue_create(BLOCKING_QUEUE_NO_TIMEOUT, release); ++ if (mq->messages == NULL) { + ERROR("Failed to create events queue"); + return NULL; + } ++ bq = mq->messages; + + mq->subscribers = map_new(MAP_PTR_INT, MAP_DEFAULT_CMP_FUNC, message_queue_subscriber_free); + if (mq->subscribers == NULL) { +@@ -131,7 +132,7 @@ message_queue *message_queue_create(void (*release)(void *)) + return NULL; + } + +- mq->messages = isula_transfer_ptr(bq); ++ bq = NULL; + return isula_transfer_ptr(mq); + } + +-- +2.34.1 + diff --git a/0042-specify-runtime-as-runc-for-oom-test-CI.patch b/0042-specify-runtime-as-runc-for-oom-test-CI.patch new file mode 100644 index 0000000..9db5fe1 --- /dev/null +++ b/0042-specify-runtime-as-runc-for-oom-test-CI.patch @@ -0,0 +1,26 @@ +From 7af700c4021ef9961aaac37ffa5767bd4f3dd184 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 08:00:37 +0000 +Subject: [PATCH 42/43] specify runtime as runc for oom test CI + +Signed-off-by: jikai +--- + CI/test_cases/container_cases/inspect.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/CI/test_cases/container_cases/inspect.sh b/CI/test_cases/container_cases/inspect.sh +index 86aed3d8..5d976281 100755 +--- a/CI/test_cases/container_cases/inspect.sh ++++ b/CI/test_cases/container_cases/inspect.sh +@@ -147,7 +147,7 @@ function test_inspect_spec() + isula rm -f $containername + + # use more than 10m memory limit, otherwise it might fail to run +- isula run -it -m 10m --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' ++ isula run -it -m 10m --runtime runc --name $containername $ubuntu_image perl -e 'for ($i = 0; $i < 100000000; $i++) { $a .= " " x 1024 }' + + isula inspect -f "{{json .State.OOMKilled}} {{.Name}}" $containername 2>&1 | sed -n '1p' | grep "true" + [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check container with image: ${ubuntu_image}" && ((ret++)) +-- +2.34.1 + diff --git a/0043-set-oomkilled-in-cri.patch b/0043-set-oomkilled-in-cri.patch new file mode 100644 index 0000000..486579e --- /dev/null +++ b/0043-set-oomkilled-in-cri.patch @@ -0,0 +1,27 @@ +From 5393ce7d02bb73ce4760edefa959dfb4846f1958 Mon Sep 17 00:00:00 2001 +From: jikai +Date: Tue, 2 Apr 2024 11:19:06 +0000 +Subject: [PATCH 43/43] set oomkilled in cri + +Signed-off-by: jikai +--- + src/daemon/common/cri/v1/v1_cri_helpers.cc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc +index a3488894..ea5c8bb5 100644 +--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc ++++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc +@@ -506,6 +506,9 @@ void UpdateBaseStatusFromInspect( + } else { // Case 3 + state = runtime::v1::CONTAINER_CREATED; + } ++ if (inspect->state->oom_killed == true) { ++ reason = "OOMKilled"; ++ } + if (inspect->state->error != nullptr) { + message = inspect->state->error; + } +-- +2.34.1 + diff --git a/iSulad.spec b/iSulad.spec index 62d8e8b..5ce391a 100644 --- a/iSulad.spec +++ b/iSulad.spec @@ -1,5 +1,5 @@ %global _version 2.1.5 -%global _release 3 +%global _release 4 %global is_systemd 1 %global enable_criv1 1 %global enable_shimv2 1 @@ -49,6 +49,15 @@ Patch0031: 0031-cgroup-v2-does-not-support-isulad-setting-cpu_rt-opt.patch Patch0032: 0032-add-test-that-isulad-cannot-set-cpu_rt-parameters-wh.patch Patch0033: 0033-fix-sandbox-container-bool-value-uninitialized.patch Patch0034: 0034-bugfix-for-cpurt.sh.patch +Patch0035: 0035-monitor-cgroup-oom-killed-event-and-update-to-cri-of.patch +Patch0036: 0036-add-ci-cases-for-oomkilled-monitor.patch +Patch0037: 0037-add-cgroup-v2-doc.patch +Patch0038: 0038-fix-run-ubuntu-container-bug-in-inspect.sh.patch +Patch0039: 0039-add-support-for-GetContainerEvents.patch +Patch0040: 0040-fix-cpurt-init-bug-for-systemd-cgroup.patch +Patch0041: 0041-fix-message-queue-concurrent-bug.patch +Patch0042: 0042-specify-runtime-as-runc-for-oom-test-CI.patch +Patch0043: 0043-set-oomkilled-in-cri.patch %ifarch x86_64 aarch64 Provides: libhttpclient.so()(64bit) @@ -303,6 +312,12 @@ fi %endif %changelog +* Tue Apr 02 2024 jikai - 2.1.5-4 +- Type: update +- ID: NA +- SUG: NA +- DESC: upgrade from upstream + * Sat Mar 30 2024 zhongtao - 2.1.5-3 - Type: update - ID: NA