packages/iSulad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
iSulad/0026-network-support-mutlnetworks.patch
gaohuatao 147f0afca6 update from openeuler 
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
2020-11-12 19:32:18 +08:00

976 lines
40 KiB
Diff
Raw Blame History

From aa35a1a1621d911cf9b76eba232814775ea6b4d9 Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Mon, 2 Nov 2020 11:15:34 +0800
Subject: [PATCH 26/28] network: support mutlnetworks
1. support mutlnetworks
2. support dualstack for default network
Signed-off-by: haozi007 <liuhao27@huawei.com>
---
src/api/services/cri/api.proto | 6 +
src/daemon/entry/cri/cni_network_plugin.cc | 221 +++++++++++++++++++--
src/daemon/entry/cri/cni_network_plugin.h | 14 +-
src/daemon/entry/cri/cri_helpers.cc | 4 +-
src/daemon/entry/cri/cri_helpers.h | 2 +-
src/daemon/entry/cri/cri_runtime_service.h | 13 +-
src/daemon/entry/cri/cri_sandbox.cc | 202 +++++++++----------
src/daemon/entry/cri/network_plugin.cc | 120 +++++++----
src/daemon/entry/cri/network_plugin.h | 10 +-
9 files changed, 414 insertions(+), 178 deletions(-)
diff --git a/src/api/services/cri/api.proto b/src/api/services/cri/api.proto
index 8aba0d3..67e5527 100644
--- a/src/api/services/cri/api.proto
+++ b/src/api/services/cri/api.proto
@@ -399,10 +399,16 @@ message PodSandboxStatusRequest {
bool verbose = 2;
}
+// PodIP represents an ip of a Pod
+message PodIP {
+ // an ip is a string representation of an IPV4 or an IPV6
+ string ip = 1;
+}
// PodSandboxNetworkStatus is the status of the network for a PodSandbox.
message PodSandboxNetworkStatus {
// IP address of the PodSandbox.
string ip = 1;
+ repeated PodIP additional_ips = 2;
}
// Namespace contains paths to the namespaces.
diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc
index f15eba3..9cb5722 100644
--- a/src/daemon/entry/cri/cni_network_plugin.cc
+++ b/src/daemon/entry/cri/cni_network_plugin.cc
@@ -80,7 +80,7 @@ auto CNINetwork::GetPaths(Errors &err) -> char **
{
char **paths = CRIHelpers::StringVectorToCharArray(m_path);
if (paths == nullptr) {
- err.SetError("Get char ** path failed");
+ err.SetError("Get cni network paths failed");
}
return paths;
}
@@ -124,6 +124,26 @@ void CniNetworkPlugin::SetDefaultNetwork(std::unique_ptr<CNINetwork> network, st
}
}
+void CniNetworkPlugin::UpdateMutlNetworks(std::vector<std::unique_ptr<CNINetwork>> &multNets,
+ std::vector<std::string> &binDirs, Errors &err)
+{
+ if (multNets.size() == 0) {
+ return;
+ }
+ WLockNetworkMap(err);
+ if (err.NotEmpty()) {
+ return;
+ }
+
+ m_mutlNetworks.clear();
+ for (auto iter = multNets.begin(); iter != multNets.end(); ++iter) {
+ (*iter)->SetPaths(binDirs);
+ m_mutlNetworks[(*iter)->GetName()] = std::move(*iter);
+ }
+
+ UnlockNetworkMap(err);
+}
+
CniNetworkPlugin::CniNetworkPlugin(std::vector<std::string> &binDirs, const std::string &confDir,
const std::string &podCidr)
: m_confDir(confDir)
@@ -139,6 +159,7 @@ CniNetworkPlugin::~CniNetworkPlugin()
if (m_syncThread.joinable()) {
m_syncThread.join();
}
+ m_mutlNetworks.clear();
}
void CniNetworkPlugin::PlatformInit(Errors &error)
@@ -259,7 +280,9 @@ out:
void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vector<std::string> &binDirs, Errors &err)
{
std::vector<std::string> files;
- bool found = false;
+ std::vector<std::unique_ptr<CNINetwork>> mutlNets;
+ char *default_net_name = nullptr;
+ std::string message = { "" };
if (GetCNIConfFiles(confDir, files, err) != 0) {
goto free_out;
@@ -279,17 +302,33 @@ void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vec
n_list = nullptr;
continue;
}
+ DEBUG("parse cni network: %s", n_list->name);
- SetDefaultNetwork(std::unique_ptr<CNINetwork>(new (std::nothrow) CNINetwork(n_list->name, n_list)), binDirs,
- err);
- found = true;
- break;
+ if (default_net_name == nullptr) {
+ SetDefaultNetwork(std::unique_ptr<CNINetwork>(new (std::nothrow) CNINetwork(n_list->name, n_list)), binDirs, err);
+ default_net_name = util_strdup_s(n_list->name);
+ message += default_net_name;
+ continue;
+ }
+ if (strcmp(default_net_name, n_list->name) == 0) {
+ WARN("Use same name of default net: %s", default_net_name);
+ continue;
+ }
+ mutlNets.push_back(std::unique_ptr<CNINetwork>(new (std::nothrow) CNINetwork(n_list->name, n_list)));
+ message += ", " + std::string(n_list->name);
}
- if (!found) {
+ if (default_net_name == nullptr) {
err.Errorf("No valid networks found in %s", confDir.c_str());
+ goto free_out;
+ }
+ UpdateMutlNetworks(mutlNets, binDirs, err);
+ if (err.NotEmpty()) {
+ goto free_out;
}
+ INFO("Loaded cni plugins successfully, [ %s ]", message.c_str());
free_out:
+ free(default_net_name);
return;
}
@@ -350,6 +389,71 @@ void CniNetworkPlugin::Status(Errors &err)
CheckInitialized(err);
}
+
+bool CniNetworkPlugin::SetupMultNetworks(const std::string &ns, const std::string &defaultInterface,
+ const std::string &name,
+ const std::string &netnsPath, const std::string &podSandboxID,
+ const std::map<std::string, std::string> &annotations,
+ const std::map<std::string, std::string> &options, Errors &err)
+{
+ bool ret = false;
+ int defaultIdx = -1;
+ size_t len = 0;
+ cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, err);
+ if (err.NotEmpty()) {
+ ERROR("Couldn't get network plane from pod annotations: %s", err.GetCMessage());
+ err.Errorf("Couldn't get network plane from pod annotations: %s", err.GetCMessage());
+ goto cleanup;
+ }
+
+ for (size_t i = 0; i < len; i++) {
+ if (networks[i] == nullptr || networks[i]->name == nullptr || networks[i]->interface == nullptr) {
+ continue;
+ }
+ struct result *preResult = nullptr;
+ auto netIter = m_mutlNetworks.find(networks[i]->name);
+ if (netIter == m_mutlNetworks.end()) {
+ err.Errorf("Cannot found user defined net: %s", networks[i]->name);
+ break;
+ }
+ if (defaultInterface == networks[i]->interface) {
+ defaultIdx = i;
+ continue;
+ }
+ AddToNetwork((netIter->second).get(), name, ns, networks[i]->interface, podSandboxID, netnsPath, annotations, options,
+ &preResult, err);
+ free_result(preResult);
+ if (err.NotEmpty()) {
+ ERROR("Do setup user defined net: %s, failed: %s", networks[i]->name, err.GetCMessage());
+ break;
+ }
+ INFO("Setup user defained net: %s success", networks[i]->name);
+ }
+
+ // mask default network pod, if user defined net use same interface
+ if (defaultIdx >= 0) {
+ auto netIter = m_mutlNetworks.find(networks[defaultIdx]->name);
+ if (netIter == m_mutlNetworks.end()) {
+ err.Errorf("Cannot found user defined net: %s", networks[defaultIdx]->name);
+ goto cleanup;
+ }
+
+ struct result *preResult = nullptr;
+ AddToNetwork((netIter->second).get(), name, ns, networks[defaultIdx]->interface, podSandboxID, netnsPath, annotations,
+ options, &preResult, err);
+ free_result(preResult);
+ if (err.NotEmpty()) {
+ ERROR("Do setup user defined net: %s, failed: %s", networks[defaultIdx]->name, err.GetCMessage());
+ goto cleanup;
+ }
+ INFO("Setup default net: %s success", networks[defaultIdx]->name);
+ ret = true;
+ }
+cleanup:
+ free_cri_pod_network(networks, len);
+ return ret;
+}
+
void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName,
const std::string &id, const std::map<std::string, std::string> &annotations,
const std::map<std::string, std::string> &options, Errors &err)
@@ -381,17 +485,83 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name,
return;
}
- AddToNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err);
+ bool setedDefaultNet = SetupMultNetworks(ns, interfaceName, name, netnsPath, id, annotations, options, err);
+ if (err.NotEmpty()) {
+ goto unlock;
+ }
+
+ if (setedDefaultNet) {
+ goto unlock;
+ }
+ AddToNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err);
free_result(preResult);
- preResult = nullptr;
if (err.NotEmpty()) {
ERROR("Error while adding to cni network: %s", err.GetCMessage());
}
+unlock:
UnlockNetworkMap(err);
}
+bool CniNetworkPlugin::TearDownMultNetworks(const std::string &ns, const std::string &defaultInterface,
+ const std::string &name,
+ const std::string &netnsPath, const std::string &podSandboxID, const std::map<std::string, std::string> &annotations,
+ Errors &err)
+{
+ bool ret = false;
+ int defaultIdx = -1;
+ size_t len = 0;
+ cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, err);
+ if (err.NotEmpty()) {
+ ERROR("Couldn't get network plane from pod annotations: %s", err.GetCMessage());
+ err.Errorf("Couldn't get network plane from pod annotations: %s", err.GetCMessage());
+ goto cleanup;
+ }
+
+ for (size_t i = 0; i < len; i++) {
+ if (networks[i] == nullptr || networks[i]->name == nullptr || networks[i]->interface == nullptr) {
+ continue;
+ }
+ auto netIter = m_mutlNetworks.find(networks[i]->name);
+ if (netIter == m_mutlNetworks.end()) {
+ WARN("Cannot found user defined net: %s", networks[i]->name);
+ continue;
+ }
+ if (defaultInterface == networks[i]->interface) {
+ defaultIdx = i;
+ continue;
+ }
+ DeleteFromNetwork((netIter->second).get(), name, ns, networks[i]->interface, podSandboxID, netnsPath, annotations, err);
+ if (err.NotEmpty()) {
+ ERROR("Do teardown user defined net: %s, failed: %s", networks[i]->name, err.GetCMessage());
+ break;
+ }
+ INFO("Teardown user defained net: %s success", networks[i]->name);
+ }
+
+ // mask default network pod, if user defined net use same interface
+ if (defaultIdx >= 0) {
+ auto netIter = m_mutlNetworks.find(networks[defaultIdx]->name);
+ if (netIter == m_mutlNetworks.end()) {
+ err.Errorf("Cannot found user defined net: %s", networks[defaultIdx]->name);
+ goto cleanup;
+ }
+
+ DeleteFromNetwork((netIter->second).get(), name, ns, networks[defaultIdx]->interface, podSandboxID, netnsPath,
+ annotations, err);
+ if (err.NotEmpty()) {
+ ERROR("Do teardown user defined net: %s, failed: %s", networks[defaultIdx]->name, err.GetCMessage());
+ goto cleanup;
+ }
+ INFO("Teardown default net: %s success", networks[defaultIdx]->name);
+ ret = true;
+ }
+cleanup:
+ free_cri_pod_network(networks, len);
+ return ret;
+}
+
void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName,
const std::string &id, const std::map<std::string, std::string> &annotations,
Errors &err)
@@ -400,6 +570,7 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam
if (err.NotEmpty()) {
return;
}
+ Errors tmpErr;
std::string netnsPath = m_criImpl->GetNetNS(id, err);
if (err.NotEmpty()) {
@@ -413,8 +584,21 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam
return;
}
- DeleteFromNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, err);
+ bool defaultNetDone = TearDownMultNetworks(ns, interfaceName, name, netnsPath, id, annotations, err);
+ if (defaultNetDone) {
+ goto unlock;
+ }
+ if (err.NotEmpty()) {
+ WARN("Teardown user defined networks failed: %s", err.GetCMessage());
+ }
+
+ DeleteFromNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, tmpErr);
+ if (tmpErr.NotEmpty()) {
+ WARN("Teardown default network failed: %s", tmpErr.GetCMessage());
+ err.AppendError(tmpErr.GetMessage());
+ }
+unlock:
UnlockNetworkMap(err);
}
@@ -464,7 +648,7 @@ void CniNetworkPlugin::GetPodNetworkStatus(const std::string & /*ns*/, const std
PodNetworkStatus &status, Errors &err)
{
std::string netnsPath;
- std::string ip;
+ std::vector<std::string> ips;
Errors tmpErr;
if (podSandboxID.empty()) {
@@ -482,15 +666,15 @@ void CniNetworkPlugin::GetPodNetworkStatus(const std::string & /*ns*/, const std
podSandboxID.c_str());
goto out;
}
- ip = GetPodIP(m_nsenterPath, netnsPath, interfaceName, err);
+ GetPodIP(m_nsenterPath, netnsPath, interfaceName, ips, err);
if (err.NotEmpty()) {
ERROR("GetPodIP failed: %s", err.GetCMessage());
goto out;
}
- status.SetIP(ip);
+ status.SetIPs(ips);
out:
- INFO("get_pod_network_status: %s", podSandboxID.c_str());
+ INFO("Get pod: %s network status success", podSandboxID.c_str());
}
void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &podName, const std::string &podNamespace,
@@ -761,7 +945,8 @@ void CniNetworkPlugin::RLockNetworkMap(Errors &error)
{
int ret = pthread_rwlock_rdlock(&m_netsLock);
if (ret != 0) {
- error.Errorf("Get read lock failed: %s", strerror(ret));
+ error.Errorf("Failed to get read lock");
+ ERROR("Get read lock failed: %s", strerror(ret));
}
}
@@ -769,7 +954,8 @@ void CniNetworkPlugin::WLockNetworkMap(Errors &error)
{
int ret = pthread_rwlock_wrlock(&m_netsLock);
if (ret != 0) {
- error.Errorf("Get write lock failed: %s", strerror(ret));
+ error.Errorf("Failed to get write lock");
+ ERROR("Get write lock failed: %s", strerror(ret));
}
}
@@ -777,7 +963,8 @@ void CniNetworkPlugin::UnlockNetworkMap(Errors &error)
{
int ret = pthread_rwlock_unlock(&m_netsLock);
if (ret != 0) {
- error.Errorf("Unlock failed: %s", strerror(ret));
+ error.Errorf("Failed to unlock");
+ ERROR("Unlock failed: %s", strerror(ret));
}
}
diff --git a/src/daemon/entry/cri/cni_network_plugin.h b/src/daemon/entry/cri/cni_network_plugin.h
index 02c95fb..c59c200 100644
--- a/src/daemon/entry/cri/cni_network_plugin.h
+++ b/src/daemon/entry/cri/cni_network_plugin.h
@@ -146,6 +146,9 @@ private:
void RLockNetworkMap(Errors &error);
void WLockNetworkMap(Errors &error);
void UnlockNetworkMap(Errors &error);
+
+ void UpdateMutlNetworks(std::vector<std::unique_ptr<CNINetwork>> &multNets, std::vector<std::string> &binDirs,
+ Errors &err);
void SetDefaultNetwork(std::unique_ptr<CNINetwork> network, std::vector<std::string> &binDirs, Errors &err);
void SetPodCidr(const std::string &podCidr);
static auto GetCNIConfFiles(const std::string &pluginDir, std::vector<std::string> &vect_files, Errors &err) -> int;
@@ -155,10 +158,19 @@ private:
void ResetCNINetwork(std::map<std::string, std::unique_ptr<CNINetwork>> &newNets, Errors &err);
void UpdateDefaultNetwork();
+ bool SetupMultNetworks(const std::string &ns, const std::string &defaultInterface, const std::string &name,
+ const std::string &netnsPath, const std::string &podSandboxID, const std::map<std::string, std::string> &annotations,
+ const std::map<std::string, std::string> &options, Errors &err);
+
+ bool TearDownMultNetworks(const std::string &ns, const std::string &defaultInterface, const std::string &name,
+ const std::string &netnsPath, const std::string &podSandboxID, const std::map<std::string, std::string> &annotations,
+ Errors &err);
+
NoopNetworkPlugin m_noop;
std::unique_ptr<CNINetwork> m_loNetwork { nullptr };
-
std::unique_ptr<CNINetwork> m_defaultNetwork { nullptr };
+ std::map<std::string, std::unique_ptr<CNINetwork>> m_mutlNetworks;
+
CRIRuntimeServiceImpl *m_criImpl { nullptr };
std::string m_nsenterPath;
std::string m_confDir;
diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc
index ee633b7..34d32e5 100644
--- a/src/daemon/entry/cri/cri_helpers.cc
+++ b/src/daemon/entry/cri/cri_helpers.cc
@@ -389,7 +389,7 @@ auto sha256(const char *val) -> std::string
return outputBuffer;
}
-auto GetNetworkPlaneFromPodAnno(const google::protobuf::Map<std::string, std::string> &annotations, size_t *len,
+auto GetNetworkPlaneFromPodAnno(const std::map<std::string, std::string> &annotations, size_t *len,
Errors &error) -> cri_pod_network_element **
{
auto iter = annotations.find(CRIHelpers::Constants::POD_NETWORK_ANNOTATION_KEY);
@@ -399,7 +399,7 @@ auto GetNetworkPlaneFromPodAnno(const google::protobuf::Map<std::string, std::st
parser_error err = nullptr;
result = cri_pod_network_parse_data(iter->second.c_str(), nullptr, &err, len);
if (result == nullptr) {
- error.Errorf("parse pod network json failed: %s", err);
+ error.Errorf("parse pod network json: %s failed: %s", iter->second.c_str(), err);
}
free(err);
}
diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h
index 3ea9ba6..b9fb153 100644
--- a/src/daemon/entry/cri/cri_helpers.h
+++ b/src/daemon/entry/cri/cri_helpers.h
@@ -94,7 +94,7 @@ auto IsImageNotFoundError(const std::string &err) -> bool;
auto sha256(const char *val) -> std::string;
-auto GetNetworkPlaneFromPodAnno(const google::protobuf::Map<std::string, std::string> &annotations,
+auto GetNetworkPlaneFromPodAnno(const std::map<std::string, std::string> &annotations,
size_t *len, Errors &error) -> cri_pod_network_element **;
auto CheckpointToSandbox(const std::string &id,
diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h
index 66837e9..1a0f601 100644
--- a/src/daemon/entry/cri/cri_runtime_service.h
+++ b/src/daemon/entry/cri/cri_runtime_service.h
@@ -210,9 +210,12 @@ private:
void ConstructPodSandboxCheckpoint(const runtime::v1alpha2::PodSandboxConfig &config,
cri::PodSandboxCheckpoint &checkpoint);
- auto GetIP(const std::string &podSandboxID, container_inspect *inspect, const std::string &networkInterface,
- Errors &error) -> std::string;
- auto GetIPFromPlugin(container_inspect *inspect, const std::string &networkInterface, Errors &error) -> std::string;
+ void GetIPs(const std::string &podSandboxID, container_inspect *inspect, const std::string &networkInterface,
+ std::vector<std::string> &ips, Errors &error);
+ void GetFormatIPsForMultNet(container_inspect *inspect, const std::string &defaultInterface,
+ const runtime::v1alpha2::PodSandboxMetadata &metadata, std::vector<std::string> &result, Errors &error);
+ auto GetIPsFromPlugin(container_inspect *inspect, const std::string &networkInterface,
+ Errors &error) -> std::vector<std::string>;
auto GetNetworkReady(const std::string &podSandboxID, Errors &error) -> bool;
void SetNetworkReady(const std::string &podSandboxID, bool ready, Errors &error);
void ClearNetworkReady(const std::string &podSandboxID);
@@ -247,9 +250,6 @@ private:
void SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id,
const std::string &jsonCheckpoint, Errors &error);
- void SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id,
- container_inspect *inspect_data, std::map<std::string, std::string> &stdAnnos,
- std::map<std::string, std::string> &options, Errors &error);
void StartSandboxContainer(const std::string &response_id, Errors &error);
auto CreateSandboxContainer(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &image,
std::string &jsonCheckpoint, const std::string &runtimeHandler,
@@ -273,6 +273,7 @@ private:
auto ClearCniNetwork(const std::string &realSandboxID, bool hostNetwork, const std::string &ns,
const std::string &name, std::vector<std::string> &errlist,
std::map<std::string, std::string> &stdAnnos, Errors &error) -> int;
+
auto RemoveAllContainersInSandbox(const std::string &realSandboxID, std::vector<std::string> &errors) -> int;
auto DoRemovePodSandbox(const std::string &realSandboxID, std::vector<std::string> &errors) -> int;
static void MergeSecurityContextToHostConfig(const runtime::v1alpha2::PodSandboxConfig &c, host_config *hc,
diff --git a/src/daemon/entry/cri/cri_sandbox.cc b/src/daemon/entry/cri/cri_sandbox.cc
index 6db9616..b44c86c 100644
--- a/src/daemon/entry/cri/cri_sandbox.cc
+++ b/src/daemon/entry/cri/cri_sandbox.cc
@@ -458,44 +458,6 @@ void CRIRuntimeServiceImpl::StartSandboxContainer(const std::string &response_id
free_container_start_response(start_response);
}
-void CRIRuntimeServiceImpl::SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config,
- const std::string &response_id,
- container_inspect *inspect_data,
- std::map<std::string, std::string> &stdAnnos,
- std::map<std::string, std::string> &options, Errors &error)
-{
- google::protobuf::Map<std::string, std::string> annotations;
- CRIHelpers::ExtractAnnotations(inspect_data->config->annotations, annotations);
-
- size_t len = 0;
- cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, error);
- if (error.NotEmpty()) {
- ERROR("Couldn't get network plane from pod annotations: %s", error.GetCMessage());
- error.Errorf("Couldn't get network plane from pod annotations: %s", error.GetCMessage());
- goto cleanup;
- }
- for (size_t i = 0; i < len; i++) {
- if ((networks[i] != nullptr) && (networks[i]->name != nullptr) && (networks[i]->interface != nullptr) &&
- strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) {
- INFO("SetupPod net: %s", networks[i]->name);
- m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), networks[i]->interface,
- response_id, stdAnnos, options, error);
- if (error.Empty()) {
- continue;
- }
- Errors tmpErr;
- StopContainerHelper(response_id, tmpErr);
- if (tmpErr.NotEmpty()) {
- WARN("Failed to stop sandbox container %s for pod %s: %s", response_id.c_str(), networks[i]->name,
- tmpErr.GetCMessage());
- }
- goto cleanup;
- }
- }
-cleanup:
- free_cri_pod_network(networks, len);
-}
-
void CRIRuntimeServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config,
const std::string &response_id, const std::string &jsonCheckpoint,
Errors &error)
@@ -685,45 +647,6 @@ cleanup:
return ret;
}
-auto CRIRuntimeServiceImpl::TearDownPodCniNetwork(const std::string &realSandboxID, std::vector<std::string> &errlist,
- std::map<std::string, std::string> &stdAnnos, const std::string &ns,
- const std::string &name, Errors &error) -> int
-{
- int ret = 0;
- cri_pod_network_element **networks = nullptr;
- container_inspect *inspect_data = InspectContainer(realSandboxID, error);
- if (inspect_data == nullptr) {
- return -1;
- }
-
- google::protobuf::Map<std::string, std::string> annotations;
- CRIHelpers::ExtractAnnotations(inspect_data->config->annotations, annotations);
- size_t len = 0;
-
- networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, error);
- if (error.NotEmpty()) {
- ERROR("Couldn't get network plane from pod annotations: %s", error.GetCMessage());
- error.Errorf("Couldn't get network plane from pod annotations: %s", error.GetCMessage());
- ret = -1;
- goto cleanup;
- }
- for (size_t i = 0; i < len; i++) {
- if ((networks[i] != nullptr) && (networks[i]->name != nullptr) && (networks[i]->interface != nullptr) &&
- strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) {
- Errors tmpErr;
- m_pluginManager->TearDownPod(ns, name, networks[i]->interface, inspect_data->id, stdAnnos, tmpErr);
- if (tmpErr.NotEmpty()) {
- WARN("TearDownPod cni network failed: %s", tmpErr.GetCMessage());
- errlist.push_back(tmpErr.GetMessage());
- }
- }
- }
-cleanup:
- free_cri_pod_network(networks, len);
- free_container_inspect(inspect_data);
- return ret;
-}
-
auto CRIRuntimeServiceImpl::ClearCniNetwork(const std::string &realSandboxID, bool hostNetwork, const std::string &ns,
const std::string &name, std::vector<std::string> &errlist,
std::map<std::string, std::string> &stdAnnos, Errors &
@@ -966,8 +889,19 @@ void CRIRuntimeServiceImpl::SetSandboxStatusNetwork(container_inspect *inspect,
std::unique_ptr<runtime::v1alpha2::PodSandboxStatus> &podStatus,
Errors &error)
{
- std::string interfaceIP = GetIP(podSandboxID, inspect, Network::DEFAULT_NETWORK_INTERFACE_NAME, error);
- podStatus->mutable_network()->set_ip(interfaceIP);
+ std::vector<std::string> ips;
+ size_t i;
+
+ GetIPs(podSandboxID, inspect, Network::DEFAULT_NETWORK_INTERFACE_NAME, ips, error);
+ if (ips.size() == 0) {
+ return;
+ }
+ podStatus->mutable_network()->set_ip(ips[0]);
+
+ for (i = 1; i < ips.size(); i++) {
+ auto tPoint = podStatus->mutable_network()->add_additional_ips();
+ tPoint->set_ip(ips[i]);
+ }
}
void CRIRuntimeServiceImpl::PodSandboxStatusToGRPC(container_inspect *inspect, const std::string &podSandboxID,
@@ -1019,66 +953,126 @@ void CRIRuntimeServiceImpl::PodSandboxStatusToGRPC(container_inspect *inspect, c
}
}
-auto CRIRuntimeServiceImpl::GetIPFromPlugin(container_inspect *inspect, const std::string &networkInterface,
- Errors &error) -> std::string
+void CRIRuntimeServiceImpl::GetFormatIPsForMultNet(container_inspect *inspect, const std::string &defaultInterface,
+ const runtime::v1alpha2::PodSandboxMetadata &metadata, std::vector<std::string> &result, Errors &error)
{
- if (inspect == nullptr || inspect->id == nullptr || inspect->name == nullptr) {
- error.SetError("Empty arguments");
- return "";
+ size_t len = 0;
+ cri_pod_network_element **elems { nullptr };
+ parser_error jerr { nullptr };
+
+ if (inspect->config == nullptr || inspect->config->annotations == nullptr) {
+ return;
+ }
+
+ for (size_t i = 0; i < inspect->config->annotations->len; i++) {
+ if (strcmp(inspect->config->annotations->keys[i], CRIHelpers::Constants::POD_NETWORK_ANNOTATION_KEY.c_str()) != 0) {
+ continue;
+ }
+ elems = cri_pod_network_parse_data(inspect->config->annotations->values[i], nullptr, &jerr, &len);
+ if (elems == nullptr) {
+ ERROR("parse mutlnetwork config failed: %s", jerr);
+ error.SetError("parse mutlnetwork config failed");
+ goto out;
+ }
+ break;
+ }
+
+ for (size_t i = 0; i < len; i++) {
+ if (elems[i]->interface == nullptr || strcmp(elems[i]->interface, defaultInterface.c_str()) == 0) {
+ continue;
+ }
+ Network::PodNetworkStatus status;
+ m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), elems[i]->interface, inspect->id, status,
+ error);
+ if (error.NotEmpty()) {
+ goto out;
+ }
+ // add a sentry to make ips of mutlnetwork store from position 2
+ if (result.size() < 2) {
+ result.push_back("");
+ }
+
+ result.push_back(std::string(elems[i]->name) + "@" + std::string(elems[i]->interface) + "@[" + CXXUtils::StringsJoin(
+ status.GetIPs(), ", ") + "]");
+ }
+out:
+ for (size_t i = 0; i < len; i++) {
+ free_cri_pod_network_element(elems[i]);
+ elems[i] = nullptr;
}
+ free(elems);
+ free(jerr);
+}
+auto CRIRuntimeServiceImpl::GetIPsFromPlugin(container_inspect *inspect, const std::string &networkInterface,
+ Errors &error) -> std::vector<std::string>
+{
+ std::vector<std::string> ret;
runtime::v1alpha2::PodSandboxMetadata metadata;
+ std::string defaultInterface = networkInterface;
+
+ if (inspect == nullptr || inspect->id == nullptr || inspect->name == nullptr) {
+ error.SetError("Empty arguments");
+ return ret;
+ }
CRINaming::ParseSandboxName(inspect->name, metadata, error);
if (error.NotEmpty()) {
- return "";
+ return ret;
}
- std::string cid = inspect->id;
- Network::PodNetworkStatus status;
- if (networkInterface.empty()) {
- m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(),
- Network::DEFAULT_NETWORK_INTERFACE_NAME, cid, status, error);
- } else {
- m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), networkInterface, cid, status,
- error);
+ if (defaultInterface.empty()) {
+ defaultInterface = Network::DEFAULT_NETWORK_INTERFACE_NAME;
}
+
+ // step 1: get ips of default network
+ Network::PodNetworkStatus status;
+ m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), defaultInterface, inspect->id, status,
+ error);
if (error.NotEmpty()) {
- return "";
+ return ret;
+ }
+ for (auto &iter : status.GetIPs()) {
+ ret.push_back(iter);
}
- return status.GetIP();
+ // step 2: get ips of mutl networks
+ GetFormatIPsForMultNet(inspect, defaultInterface, metadata, ret, error);
+
+ return ret;
}
-auto CRIRuntimeServiceImpl::GetIP(const std::string &podSandboxID, container_inspect *inspect,
- const std::string &networkInterface, Errors &error) -> std::string
+void CRIRuntimeServiceImpl::GetIPs(const std::string &podSandboxID, container_inspect *inspect,
+ const std::string &networkInterface, std::vector<std::string> &ips, Errors &error)
{
if (inspect == nullptr || inspect->network_settings == nullptr) {
- return "";
+ return;
}
if (SharesHostNetwork(inspect) != 0) {
// For sandboxes using host network, the shim is not responsible for reporting the IP.
- return "";
+ return;
}
bool ready = GetNetworkReady(podSandboxID, error);
if (error.Empty() && !ready) {
WARN("Network %s do not ready", podSandboxID.c_str());
- return "";
+ return;
}
error.Clear();
- auto ip = GetIPFromPlugin(inspect, networkInterface, error);
+ auto tmpIPs = GetIPsFromPlugin(inspect, networkInterface, error);
if (error.Empty()) {
- return ip;
+ for (const auto &iter : tmpIPs) {
+ ips.push_back(iter);
+ }
+ return;
}
if (inspect->network_settings->ip_address != nullptr) {
WARN("Use container inspect ip info: %s", error.GetCMessage());
error.Clear();
- return inspect->network_settings->ip_address;
+ ips.push_back(inspect->network_settings->ip_address);
}
WARN("Failed to read pod IP from plugin/docker: %s", error.GetCMessage());
- return "";
}
std::unique_ptr<runtime::v1alpha2::PodSandboxStatus>
diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc
index 0cab31a..311ebb6 100644
--- a/src/daemon/entry/cri/network_plugin.cc
+++ b/src/daemon/entry/cri/network_plugin.cc
@@ -65,25 +65,58 @@ static void runGetIP(void *cmdArgs)
execvp(tmpArgs[0], args);
}
-static std::string GetOnePodIP(std::string nsenterPath, std::string netnsPath, std::string interfaceName,
- std::string addrType, Errors &error)
+static std::string ParseIPFromLine(const char *line, const char *stdout_str)
{
- char *stderr_str { nullptr };
- char *stdout_str { nullptr };
- char *strErr { nullptr };
- char **lines { nullptr };
+ char *cIP { nullptr };
char **fields { nullptr };
+ char *strErr { nullptr };
struct ipnet *ipnet_val {
nullptr
};
+ std::string ret;
+
+ fields = util_string_split(line, ' ');
+ if (fields == nullptr) {
+ ERROR("Out of memory");
+ goto out;
+ }
+ if (util_array_len((const char **)fields) < 4) {
+ ERROR("Unexpected address output %s ", line);
+ goto out;
+ }
+
+ if (parse_cidr(fields[3], &ipnet_val, &strErr) != 0) {
+ ERROR("CNI failed to parse ip from output %s due to %s", stdout_str, strErr);
+ goto out;
+ }
+ cIP = ip_to_string(ipnet_val->ip, ipnet_val->ip_len);
+ if (cIP == nullptr) {
+ ERROR("Out of memory");
+ goto out;
+ }
+
+ ret = cIP;
+out:
+ free(cIP);
+ free(strErr);
+ free_ipnet_type(ipnet_val);
+ util_free_array(fields);
+ return ret;
+}
+
+static void GetOnePodIP(std::string nsenterPath, std::string netnsPath, std::string interfaceName,
+ std::string addrType, std::vector<std::string> &ips, Errors &error)
+{
+ char *stderr_str { nullptr };
+ char *stdout_str { nullptr };
+ char **lines { nullptr };
char **args { nullptr };
- std::string result { "" };
- char *cIP { nullptr };
+ size_t i;
args = (char **)util_common_calloc_s(sizeof(char *) * 5);
if (args == nullptr) {
error.SetError("Out of memory");
- return result;
+ return;
}
args[0] = util_strdup_s(nsenterPath.c_str());
@@ -102,52 +135,55 @@ static std::string GetOnePodIP(std::string nsenterPath, std::string netnsPath, s
error.SetError("Out of memory");
goto free_out;
}
- if (util_array_len((const char **)lines) < 1) {
- error.Errorf("Unexpected command output %s", stdout_str);
- goto free_out;
- }
- fields = util_string_split(lines[0], ' ');
- if (fields == nullptr) {
- error.SetError("Out of memory");
- goto free_out;
- }
- if (util_array_len((const char **)fields) < 4) {
- error.Errorf("Unexpected address output %s ", lines[0]);
+ if (util_array_len((const char **)lines) == 0) {
+ error.Errorf("Unexpected command output %s", stdout_str);
goto free_out;
}
- if (parse_cidr(fields[3], &ipnet_val, &strErr) != 0) {
- error.Errorf("CNI failed to parse ip from output %s due to %s", stdout_str, strErr);
- goto free_out;
- }
- cIP = ip_to_string(ipnet_val->ip, ipnet_val->ip_len);
- if (cIP == nullptr) {
- error.SetError("Out of memory");
- goto free_out;
+ for (i = 0; i < util_array_len((const char **)lines); i++) {
+ // ip string min length must bigger than 4
+ if (lines[i] == nullptr || strlen(lines[i]) < 4) {
+ continue;
+ }
+ std::string tIP = ParseIPFromLine(lines[i], stdout_str);
+ if (tIP.empty()) {
+ error.Errorf("parse %s to ip failed", lines[i]);
+ break;
+ }
+ ips.push_back(tIP);
}
- result = cIP;
- free(cIP);
free_out:
- free_ipnet_type(ipnet_val);
free(stdout_str);
free(stderr_str);
util_free_array(args);
util_free_array(lines);
- util_free_array(fields);
- return result;
}
-std::string GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName,
- Errors &error)
+void GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName,
+ std::vector<std::string> &getIPs, Errors &error)
{
- std::string ip = GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-4", error);
+ Errors tmpErr;
+
+ GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-4", getIPs, tmpErr);
+ if (tmpErr.NotEmpty()) {
+ WARN("Get ipv4 failed: %s", tmpErr.GetCMessage());
+ }
+
+ GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-6", getIPs, error);
if (error.NotEmpty()) {
- return GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-6", error);
+ WARN("Get ipv6 failed: %s", tmpErr.GetCMessage());
}
- return ip;
+ if (getIPs.size() > 0) {
+ error.Clear();
+ return;
+ }
+
+ if (tmpErr.NotEmpty()) {
+ error.AppendError(tmpErr.GetMessage());
+ }
}
void InitNetworkPlugin(std::vector<std::shared_ptr<NetworkPlugin>> *plugins, std::string networkPluginName,
@@ -290,14 +326,14 @@ void PodNetworkStatus::SetAPIVersion(const std::string &version)
m_apiVersion = version;
}
-const std::string &PodNetworkStatus::GetIP() const
+const std::vector<std::string> &PodNetworkStatus::GetIPs() const
{
- return m_ip;
+ return m_ips;
}
-void PodNetworkStatus::SetIP(const std::string &ip)
+void PodNetworkStatus::SetIPs(std::vector<std::string> &ips)
{
- m_ip = ip;
+ m_ips = ips;
}
void PluginManager::Lock(const std::string &fullPodName, Errors &error)
diff --git a/src/daemon/entry/cri/network_plugin.h b/src/daemon/entry/cri/network_plugin.h
index 5a46eb8..24afd71 100644
--- a/src/daemon/entry/cri/network_plugin.h
+++ b/src/daemon/entry/cri/network_plugin.h
@@ -82,13 +82,13 @@ public:
void SetKind(const std::string &kind);
const std::string &GetAPIVersion() const;
void SetAPIVersion(const std::string &version);
- const std::string &GetIP() const;
- void SetIP(const std::string &ip);
+ const std::vector<std::string> &GetIPs() const;
+ void SetIPs(std::vector<std::string> &ips);
private:
std::string m_kind;
std::string m_apiVersion;
- std::string m_ip;
+ std::vector<std::string> m_ips;
};
class NetworkPlugin {
@@ -227,8 +227,8 @@ void InitNetworkPlugin(std::vector<std::shared_ptr<NetworkPlugin>> *plugins, std
void ProbeNetworkPlugins(const std::string &pluginDir, const std::string &binDir,
std::vector<std::shared_ptr<NetworkPlugin>> *plugins);
-std::string GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName,
- Errors &error);
+void GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName,
+ std::vector<std::string> &getIPs, Errors &error);
const std::string &GetInterfaceName();
} // namespace Network
--
2.20.1
Reference in New Issue View Git Blame Copy Permalink