From aa35a1a1621d911cf9b76eba232814775ea6b4d9 Mon Sep 17 00:00:00 2001 From: haozi007 Date: Mon, 2 Nov 2020 11:15:34 +0800 Subject: [PATCH 26/28] network: support mutlnetworks 1. support mutlnetworks 2. support dualstack for default network Signed-off-by: haozi007 --- src/api/services/cri/api.proto | 6 + src/daemon/entry/cri/cni_network_plugin.cc | 221 +++++++++++++++++++-- src/daemon/entry/cri/cni_network_plugin.h | 14 +- src/daemon/entry/cri/cri_helpers.cc | 4 +- src/daemon/entry/cri/cri_helpers.h | 2 +- src/daemon/entry/cri/cri_runtime_service.h | 13 +- src/daemon/entry/cri/cri_sandbox.cc | 202 +++++++++---------- src/daemon/entry/cri/network_plugin.cc | 120 +++++++---- src/daemon/entry/cri/network_plugin.h | 10 +- 9 files changed, 414 insertions(+), 178 deletions(-) diff --git a/src/api/services/cri/api.proto b/src/api/services/cri/api.proto index 8aba0d3..67e5527 100644 --- a/src/api/services/cri/api.proto +++ b/src/api/services/cri/api.proto @@ -399,10 +399,16 @@ message PodSandboxStatusRequest { bool verbose = 2; } +// PodIP represents an ip of a Pod +message PodIP { + // an ip is a string representation of an IPV4 or an IPV6 + string ip = 1; +} // PodSandboxNetworkStatus is the status of the network for a PodSandbox. message PodSandboxNetworkStatus { // IP address of the PodSandbox. string ip = 1; + repeated PodIP additional_ips = 2; } // Namespace contains paths to the namespaces. diff --git a/src/daemon/entry/cri/cni_network_plugin.cc b/src/daemon/entry/cri/cni_network_plugin.cc index f15eba3..9cb5722 100644 --- a/src/daemon/entry/cri/cni_network_plugin.cc +++ b/src/daemon/entry/cri/cni_network_plugin.cc @@ -80,7 +80,7 @@ auto CNINetwork::GetPaths(Errors &err) -> char ** { char **paths = CRIHelpers::StringVectorToCharArray(m_path); if (paths == nullptr) { - err.SetError("Get char ** path failed"); + err.SetError("Get cni network paths failed"); } return paths; } @@ -124,6 +124,26 @@ void CniNetworkPlugin::SetDefaultNetwork(std::unique_ptr network, st } } +void CniNetworkPlugin::UpdateMutlNetworks(std::vector> &multNets, + std::vector &binDirs, Errors &err) +{ + if (multNets.size() == 0) { + return; + } + WLockNetworkMap(err); + if (err.NotEmpty()) { + return; + } + + m_mutlNetworks.clear(); + for (auto iter = multNets.begin(); iter != multNets.end(); ++iter) { + (*iter)->SetPaths(binDirs); + m_mutlNetworks[(*iter)->GetName()] = std::move(*iter); + } + + UnlockNetworkMap(err); +} + CniNetworkPlugin::CniNetworkPlugin(std::vector &binDirs, const std::string &confDir, const std::string &podCidr) : m_confDir(confDir) @@ -139,6 +159,7 @@ CniNetworkPlugin::~CniNetworkPlugin() if (m_syncThread.joinable()) { m_syncThread.join(); } + m_mutlNetworks.clear(); } void CniNetworkPlugin::PlatformInit(Errors &error) @@ -259,7 +280,9 @@ out: void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vector &binDirs, Errors &err) { std::vector files; - bool found = false; + std::vector> mutlNets; + char *default_net_name = nullptr; + std::string message = { "" }; if (GetCNIConfFiles(confDir, files, err) != 0) { goto free_out; @@ -279,17 +302,33 @@ void CniNetworkPlugin::GetDefaultCNINetwork(const std::string &confDir, std::vec n_list = nullptr; continue; } + DEBUG("parse cni network: %s", n_list->name); - SetDefaultNetwork(std::unique_ptr(new (std::nothrow) CNINetwork(n_list->name, n_list)), binDirs, - err); - found = true; - break; + if (default_net_name == nullptr) { + SetDefaultNetwork(std::unique_ptr(new (std::nothrow) CNINetwork(n_list->name, n_list)), binDirs, err); + default_net_name = util_strdup_s(n_list->name); + message += default_net_name; + continue; + } + if (strcmp(default_net_name, n_list->name) == 0) { + WARN("Use same name of default net: %s", default_net_name); + continue; + } + mutlNets.push_back(std::unique_ptr(new (std::nothrow) CNINetwork(n_list->name, n_list))); + message += ", " + std::string(n_list->name); } - if (!found) { + if (default_net_name == nullptr) { err.Errorf("No valid networks found in %s", confDir.c_str()); + goto free_out; + } + UpdateMutlNetworks(mutlNets, binDirs, err); + if (err.NotEmpty()) { + goto free_out; } + INFO("Loaded cni plugins successfully, [ %s ]", message.c_str()); free_out: + free(default_net_name); return; } @@ -350,6 +389,71 @@ void CniNetworkPlugin::Status(Errors &err) CheckInitialized(err); } + +bool CniNetworkPlugin::SetupMultNetworks(const std::string &ns, const std::string &defaultInterface, + const std::string &name, + const std::string &netnsPath, const std::string &podSandboxID, + const std::map &annotations, + const std::map &options, Errors &err) +{ + bool ret = false; + int defaultIdx = -1; + size_t len = 0; + cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, err); + if (err.NotEmpty()) { + ERROR("Couldn't get network plane from pod annotations: %s", err.GetCMessage()); + err.Errorf("Couldn't get network plane from pod annotations: %s", err.GetCMessage()); + goto cleanup; + } + + for (size_t i = 0; i < len; i++) { + if (networks[i] == nullptr || networks[i]->name == nullptr || networks[i]->interface == nullptr) { + continue; + } + struct result *preResult = nullptr; + auto netIter = m_mutlNetworks.find(networks[i]->name); + if (netIter == m_mutlNetworks.end()) { + err.Errorf("Cannot found user defined net: %s", networks[i]->name); + break; + } + if (defaultInterface == networks[i]->interface) { + defaultIdx = i; + continue; + } + AddToNetwork((netIter->second).get(), name, ns, networks[i]->interface, podSandboxID, netnsPath, annotations, options, + &preResult, err); + free_result(preResult); + if (err.NotEmpty()) { + ERROR("Do setup user defined net: %s, failed: %s", networks[i]->name, err.GetCMessage()); + break; + } + INFO("Setup user defained net: %s success", networks[i]->name); + } + + // mask default network pod, if user defined net use same interface + if (defaultIdx >= 0) { + auto netIter = m_mutlNetworks.find(networks[defaultIdx]->name); + if (netIter == m_mutlNetworks.end()) { + err.Errorf("Cannot found user defined net: %s", networks[defaultIdx]->name); + goto cleanup; + } + + struct result *preResult = nullptr; + AddToNetwork((netIter->second).get(), name, ns, networks[defaultIdx]->interface, podSandboxID, netnsPath, annotations, + options, &preResult, err); + free_result(preResult); + if (err.NotEmpty()) { + ERROR("Do setup user defined net: %s, failed: %s", networks[defaultIdx]->name, err.GetCMessage()); + goto cleanup; + } + INFO("Setup default net: %s success", networks[defaultIdx]->name); + ret = true; + } +cleanup: + free_cri_pod_network(networks, len); + return ret; +} + void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &id, const std::map &annotations, const std::map &options, Errors &err) @@ -381,17 +485,83 @@ void CniNetworkPlugin::SetUpPod(const std::string &ns, const std::string &name, return; } - AddToNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err); + bool setedDefaultNet = SetupMultNetworks(ns, interfaceName, name, netnsPath, id, annotations, options, err); + if (err.NotEmpty()) { + goto unlock; + } + + if (setedDefaultNet) { + goto unlock; + } + AddToNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, options, &preResult, err); free_result(preResult); - preResult = nullptr; if (err.NotEmpty()) { ERROR("Error while adding to cni network: %s", err.GetCMessage()); } +unlock: UnlockNetworkMap(err); } +bool CniNetworkPlugin::TearDownMultNetworks(const std::string &ns, const std::string &defaultInterface, + const std::string &name, + const std::string &netnsPath, const std::string &podSandboxID, const std::map &annotations, + Errors &err) +{ + bool ret = false; + int defaultIdx = -1; + size_t len = 0; + cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, err); + if (err.NotEmpty()) { + ERROR("Couldn't get network plane from pod annotations: %s", err.GetCMessage()); + err.Errorf("Couldn't get network plane from pod annotations: %s", err.GetCMessage()); + goto cleanup; + } + + for (size_t i = 0; i < len; i++) { + if (networks[i] == nullptr || networks[i]->name == nullptr || networks[i]->interface == nullptr) { + continue; + } + auto netIter = m_mutlNetworks.find(networks[i]->name); + if (netIter == m_mutlNetworks.end()) { + WARN("Cannot found user defined net: %s", networks[i]->name); + continue; + } + if (defaultInterface == networks[i]->interface) { + defaultIdx = i; + continue; + } + DeleteFromNetwork((netIter->second).get(), name, ns, networks[i]->interface, podSandboxID, netnsPath, annotations, err); + if (err.NotEmpty()) { + ERROR("Do teardown user defined net: %s, failed: %s", networks[i]->name, err.GetCMessage()); + break; + } + INFO("Teardown user defained net: %s success", networks[i]->name); + } + + // mask default network pod, if user defined net use same interface + if (defaultIdx >= 0) { + auto netIter = m_mutlNetworks.find(networks[defaultIdx]->name); + if (netIter == m_mutlNetworks.end()) { + err.Errorf("Cannot found user defined net: %s", networks[defaultIdx]->name); + goto cleanup; + } + + DeleteFromNetwork((netIter->second).get(), name, ns, networks[defaultIdx]->interface, podSandboxID, netnsPath, + annotations, err); + if (err.NotEmpty()) { + ERROR("Do teardown user defined net: %s, failed: %s", networks[defaultIdx]->name, err.GetCMessage()); + goto cleanup; + } + INFO("Teardown default net: %s success", networks[defaultIdx]->name); + ret = true; + } +cleanup: + free_cri_pod_network(networks, len); + return ret; +} + void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &name, const std::string &interfaceName, const std::string &id, const std::map &annotations, Errors &err) @@ -400,6 +570,7 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam if (err.NotEmpty()) { return; } + Errors tmpErr; std::string netnsPath = m_criImpl->GetNetNS(id, err); if (err.NotEmpty()) { @@ -413,8 +584,21 @@ void CniNetworkPlugin::TearDownPod(const std::string &ns, const std::string &nam return; } - DeleteFromNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, err); + bool defaultNetDone = TearDownMultNetworks(ns, interfaceName, name, netnsPath, id, annotations, err); + if (defaultNetDone) { + goto unlock; + } + if (err.NotEmpty()) { + WARN("Teardown user defined networks failed: %s", err.GetCMessage()); + } + + DeleteFromNetwork(m_defaultNetwork.get(), name, ns, interfaceName, id, netnsPath, annotations, tmpErr); + if (tmpErr.NotEmpty()) { + WARN("Teardown default network failed: %s", tmpErr.GetCMessage()); + err.AppendError(tmpErr.GetMessage()); + } +unlock: UnlockNetworkMap(err); } @@ -464,7 +648,7 @@ void CniNetworkPlugin::GetPodNetworkStatus(const std::string & /*ns*/, const std PodNetworkStatus &status, Errors &err) { std::string netnsPath; - std::string ip; + std::vector ips; Errors tmpErr; if (podSandboxID.empty()) { @@ -482,15 +666,15 @@ void CniNetworkPlugin::GetPodNetworkStatus(const std::string & /*ns*/, const std podSandboxID.c_str()); goto out; } - ip = GetPodIP(m_nsenterPath, netnsPath, interfaceName, err); + GetPodIP(m_nsenterPath, netnsPath, interfaceName, ips, err); if (err.NotEmpty()) { ERROR("GetPodIP failed: %s", err.GetCMessage()); goto out; } - status.SetIP(ip); + status.SetIPs(ips); out: - INFO("get_pod_network_status: %s", podSandboxID.c_str()); + INFO("Get pod: %s network status success", podSandboxID.c_str()); } void CniNetworkPlugin::AddToNetwork(CNINetwork *snetwork, const std::string &podName, const std::string &podNamespace, @@ -761,7 +945,8 @@ void CniNetworkPlugin::RLockNetworkMap(Errors &error) { int ret = pthread_rwlock_rdlock(&m_netsLock); if (ret != 0) { - error.Errorf("Get read lock failed: %s", strerror(ret)); + error.Errorf("Failed to get read lock"); + ERROR("Get read lock failed: %s", strerror(ret)); } } @@ -769,7 +954,8 @@ void CniNetworkPlugin::WLockNetworkMap(Errors &error) { int ret = pthread_rwlock_wrlock(&m_netsLock); if (ret != 0) { - error.Errorf("Get write lock failed: %s", strerror(ret)); + error.Errorf("Failed to get write lock"); + ERROR("Get write lock failed: %s", strerror(ret)); } } @@ -777,7 +963,8 @@ void CniNetworkPlugin::UnlockNetworkMap(Errors &error) { int ret = pthread_rwlock_unlock(&m_netsLock); if (ret != 0) { - error.Errorf("Unlock failed: %s", strerror(ret)); + error.Errorf("Failed to unlock"); + ERROR("Unlock failed: %s", strerror(ret)); } } diff --git a/src/daemon/entry/cri/cni_network_plugin.h b/src/daemon/entry/cri/cni_network_plugin.h index 02c95fb..c59c200 100644 --- a/src/daemon/entry/cri/cni_network_plugin.h +++ b/src/daemon/entry/cri/cni_network_plugin.h @@ -146,6 +146,9 @@ private: void RLockNetworkMap(Errors &error); void WLockNetworkMap(Errors &error); void UnlockNetworkMap(Errors &error); + + void UpdateMutlNetworks(std::vector> &multNets, std::vector &binDirs, + Errors &err); void SetDefaultNetwork(std::unique_ptr network, std::vector &binDirs, Errors &err); void SetPodCidr(const std::string &podCidr); static auto GetCNIConfFiles(const std::string &pluginDir, std::vector &vect_files, Errors &err) -> int; @@ -155,10 +158,19 @@ private: void ResetCNINetwork(std::map> &newNets, Errors &err); void UpdateDefaultNetwork(); + bool SetupMultNetworks(const std::string &ns, const std::string &defaultInterface, const std::string &name, + const std::string &netnsPath, const std::string &podSandboxID, const std::map &annotations, + const std::map &options, Errors &err); + + bool TearDownMultNetworks(const std::string &ns, const std::string &defaultInterface, const std::string &name, + const std::string &netnsPath, const std::string &podSandboxID, const std::map &annotations, + Errors &err); + NoopNetworkPlugin m_noop; std::unique_ptr m_loNetwork { nullptr }; - std::unique_ptr m_defaultNetwork { nullptr }; + std::map> m_mutlNetworks; + CRIRuntimeServiceImpl *m_criImpl { nullptr }; std::string m_nsenterPath; std::string m_confDir; diff --git a/src/daemon/entry/cri/cri_helpers.cc b/src/daemon/entry/cri/cri_helpers.cc index ee633b7..34d32e5 100644 --- a/src/daemon/entry/cri/cri_helpers.cc +++ b/src/daemon/entry/cri/cri_helpers.cc @@ -389,7 +389,7 @@ auto sha256(const char *val) -> std::string return outputBuffer; } -auto GetNetworkPlaneFromPodAnno(const google::protobuf::Map &annotations, size_t *len, +auto GetNetworkPlaneFromPodAnno(const std::map &annotations, size_t *len, Errors &error) -> cri_pod_network_element ** { auto iter = annotations.find(CRIHelpers::Constants::POD_NETWORK_ANNOTATION_KEY); @@ -399,7 +399,7 @@ auto GetNetworkPlaneFromPodAnno(const google::protobuf::Mapsecond.c_str(), nullptr, &err, len); if (result == nullptr) { - error.Errorf("parse pod network json failed: %s", err); + error.Errorf("parse pod network json: %s failed: %s", iter->second.c_str(), err); } free(err); } diff --git a/src/daemon/entry/cri/cri_helpers.h b/src/daemon/entry/cri/cri_helpers.h index 3ea9ba6..b9fb153 100644 --- a/src/daemon/entry/cri/cri_helpers.h +++ b/src/daemon/entry/cri/cri_helpers.h @@ -94,7 +94,7 @@ auto IsImageNotFoundError(const std::string &err) -> bool; auto sha256(const char *val) -> std::string; -auto GetNetworkPlaneFromPodAnno(const google::protobuf::Map &annotations, +auto GetNetworkPlaneFromPodAnno(const std::map &annotations, size_t *len, Errors &error) -> cri_pod_network_element **; auto CheckpointToSandbox(const std::string &id, diff --git a/src/daemon/entry/cri/cri_runtime_service.h b/src/daemon/entry/cri/cri_runtime_service.h index 66837e9..1a0f601 100644 --- a/src/daemon/entry/cri/cri_runtime_service.h +++ b/src/daemon/entry/cri/cri_runtime_service.h @@ -210,9 +210,12 @@ private: void ConstructPodSandboxCheckpoint(const runtime::v1alpha2::PodSandboxConfig &config, cri::PodSandboxCheckpoint &checkpoint); - auto GetIP(const std::string &podSandboxID, container_inspect *inspect, const std::string &networkInterface, - Errors &error) -> std::string; - auto GetIPFromPlugin(container_inspect *inspect, const std::string &networkInterface, Errors &error) -> std::string; + void GetIPs(const std::string &podSandboxID, container_inspect *inspect, const std::string &networkInterface, + std::vector &ips, Errors &error); + void GetFormatIPsForMultNet(container_inspect *inspect, const std::string &defaultInterface, + const runtime::v1alpha2::PodSandboxMetadata &metadata, std::vector &result, Errors &error); + auto GetIPsFromPlugin(container_inspect *inspect, const std::string &networkInterface, + Errors &error) -> std::vector; auto GetNetworkReady(const std::string &podSandboxID, Errors &error) -> bool; void SetNetworkReady(const std::string &podSandboxID, bool ready, Errors &error); void ClearNetworkReady(const std::string &podSandboxID); @@ -247,9 +250,6 @@ private: void SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id, const std::string &jsonCheckpoint, Errors &error); - void SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id, - container_inspect *inspect_data, std::map &stdAnnos, - std::map &options, Errors &error); void StartSandboxContainer(const std::string &response_id, Errors &error); auto CreateSandboxContainer(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &image, std::string &jsonCheckpoint, const std::string &runtimeHandler, @@ -273,6 +273,7 @@ private: auto ClearCniNetwork(const std::string &realSandboxID, bool hostNetwork, const std::string &ns, const std::string &name, std::vector &errlist, std::map &stdAnnos, Errors &error) -> int; + auto RemoveAllContainersInSandbox(const std::string &realSandboxID, std::vector &errors) -> int; auto DoRemovePodSandbox(const std::string &realSandboxID, std::vector &errors) -> int; static void MergeSecurityContextToHostConfig(const runtime::v1alpha2::PodSandboxConfig &c, host_config *hc, diff --git a/src/daemon/entry/cri/cri_sandbox.cc b/src/daemon/entry/cri/cri_sandbox.cc index 6db9616..b44c86c 100644 --- a/src/daemon/entry/cri/cri_sandbox.cc +++ b/src/daemon/entry/cri/cri_sandbox.cc @@ -458,44 +458,6 @@ void CRIRuntimeServiceImpl::StartSandboxContainer(const std::string &response_id free_container_start_response(start_response); } -void CRIRuntimeServiceImpl::SetupUserDefinedNetworkPlane(const runtime::v1alpha2::PodSandboxConfig &config, - const std::string &response_id, - container_inspect *inspect_data, - std::map &stdAnnos, - std::map &options, Errors &error) -{ - google::protobuf::Map annotations; - CRIHelpers::ExtractAnnotations(inspect_data->config->annotations, annotations); - - size_t len = 0; - cri_pod_network_element **networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, error); - if (error.NotEmpty()) { - ERROR("Couldn't get network plane from pod annotations: %s", error.GetCMessage()); - error.Errorf("Couldn't get network plane from pod annotations: %s", error.GetCMessage()); - goto cleanup; - } - for (size_t i = 0; i < len; i++) { - if ((networks[i] != nullptr) && (networks[i]->name != nullptr) && (networks[i]->interface != nullptr) && - strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) { - INFO("SetupPod net: %s", networks[i]->name); - m_pluginManager->SetUpPod(config.metadata().namespace_(), config.metadata().name(), networks[i]->interface, - response_id, stdAnnos, options, error); - if (error.Empty()) { - continue; - } - Errors tmpErr; - StopContainerHelper(response_id, tmpErr); - if (tmpErr.NotEmpty()) { - WARN("Failed to stop sandbox container %s for pod %s: %s", response_id.c_str(), networks[i]->name, - tmpErr.GetCMessage()); - } - goto cleanup; - } - } -cleanup: - free_cri_pod_network(networks, len); -} - void CRIRuntimeServiceImpl::SetupSandboxNetwork(const runtime::v1alpha2::PodSandboxConfig &config, const std::string &response_id, const std::string &jsonCheckpoint, Errors &error) @@ -685,45 +647,6 @@ cleanup: return ret; } -auto CRIRuntimeServiceImpl::TearDownPodCniNetwork(const std::string &realSandboxID, std::vector &errlist, - std::map &stdAnnos, const std::string &ns, - const std::string &name, Errors &error) -> int -{ - int ret = 0; - cri_pod_network_element **networks = nullptr; - container_inspect *inspect_data = InspectContainer(realSandboxID, error); - if (inspect_data == nullptr) { - return -1; - } - - google::protobuf::Map annotations; - CRIHelpers::ExtractAnnotations(inspect_data->config->annotations, annotations); - size_t len = 0; - - networks = CRIHelpers::GetNetworkPlaneFromPodAnno(annotations, &len, error); - if (error.NotEmpty()) { - ERROR("Couldn't get network plane from pod annotations: %s", error.GetCMessage()); - error.Errorf("Couldn't get network plane from pod annotations: %s", error.GetCMessage()); - ret = -1; - goto cleanup; - } - for (size_t i = 0; i < len; i++) { - if ((networks[i] != nullptr) && (networks[i]->name != nullptr) && (networks[i]->interface != nullptr) && - strcmp(networks[i]->name, Network::DEFAULT_NETWORK_PLANE_NAME.c_str()) != 0) { - Errors tmpErr; - m_pluginManager->TearDownPod(ns, name, networks[i]->interface, inspect_data->id, stdAnnos, tmpErr); - if (tmpErr.NotEmpty()) { - WARN("TearDownPod cni network failed: %s", tmpErr.GetCMessage()); - errlist.push_back(tmpErr.GetMessage()); - } - } - } -cleanup: - free_cri_pod_network(networks, len); - free_container_inspect(inspect_data); - return ret; -} - auto CRIRuntimeServiceImpl::ClearCniNetwork(const std::string &realSandboxID, bool hostNetwork, const std::string &ns, const std::string &name, std::vector &errlist, std::map &stdAnnos, Errors & @@ -966,8 +889,19 @@ void CRIRuntimeServiceImpl::SetSandboxStatusNetwork(container_inspect *inspect, std::unique_ptr &podStatus, Errors &error) { - std::string interfaceIP = GetIP(podSandboxID, inspect, Network::DEFAULT_NETWORK_INTERFACE_NAME, error); - podStatus->mutable_network()->set_ip(interfaceIP); + std::vector ips; + size_t i; + + GetIPs(podSandboxID, inspect, Network::DEFAULT_NETWORK_INTERFACE_NAME, ips, error); + if (ips.size() == 0) { + return; + } + podStatus->mutable_network()->set_ip(ips[0]); + + for (i = 1; i < ips.size(); i++) { + auto tPoint = podStatus->mutable_network()->add_additional_ips(); + tPoint->set_ip(ips[i]); + } } void CRIRuntimeServiceImpl::PodSandboxStatusToGRPC(container_inspect *inspect, const std::string &podSandboxID, @@ -1019,66 +953,126 @@ void CRIRuntimeServiceImpl::PodSandboxStatusToGRPC(container_inspect *inspect, c } } -auto CRIRuntimeServiceImpl::GetIPFromPlugin(container_inspect *inspect, const std::string &networkInterface, - Errors &error) -> std::string +void CRIRuntimeServiceImpl::GetFormatIPsForMultNet(container_inspect *inspect, const std::string &defaultInterface, + const runtime::v1alpha2::PodSandboxMetadata &metadata, std::vector &result, Errors &error) { - if (inspect == nullptr || inspect->id == nullptr || inspect->name == nullptr) { - error.SetError("Empty arguments"); - return ""; + size_t len = 0; + cri_pod_network_element **elems { nullptr }; + parser_error jerr { nullptr }; + + if (inspect->config == nullptr || inspect->config->annotations == nullptr) { + return; + } + + for (size_t i = 0; i < inspect->config->annotations->len; i++) { + if (strcmp(inspect->config->annotations->keys[i], CRIHelpers::Constants::POD_NETWORK_ANNOTATION_KEY.c_str()) != 0) { + continue; + } + elems = cri_pod_network_parse_data(inspect->config->annotations->values[i], nullptr, &jerr, &len); + if (elems == nullptr) { + ERROR("parse mutlnetwork config failed: %s", jerr); + error.SetError("parse mutlnetwork config failed"); + goto out; + } + break; + } + + for (size_t i = 0; i < len; i++) { + if (elems[i]->interface == nullptr || strcmp(elems[i]->interface, defaultInterface.c_str()) == 0) { + continue; + } + Network::PodNetworkStatus status; + m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), elems[i]->interface, inspect->id, status, + error); + if (error.NotEmpty()) { + goto out; + } + // add a sentry to make ips of mutlnetwork store from position 2 + if (result.size() < 2) { + result.push_back(""); + } + + result.push_back(std::string(elems[i]->name) + "@" + std::string(elems[i]->interface) + "@[" + CXXUtils::StringsJoin( + status.GetIPs(), ", ") + "]"); + } +out: + for (size_t i = 0; i < len; i++) { + free_cri_pod_network_element(elems[i]); + elems[i] = nullptr; } + free(elems); + free(jerr); +} +auto CRIRuntimeServiceImpl::GetIPsFromPlugin(container_inspect *inspect, const std::string &networkInterface, + Errors &error) -> std::vector +{ + std::vector ret; runtime::v1alpha2::PodSandboxMetadata metadata; + std::string defaultInterface = networkInterface; + + if (inspect == nullptr || inspect->id == nullptr || inspect->name == nullptr) { + error.SetError("Empty arguments"); + return ret; + } CRINaming::ParseSandboxName(inspect->name, metadata, error); if (error.NotEmpty()) { - return ""; + return ret; } - std::string cid = inspect->id; - Network::PodNetworkStatus status; - if (networkInterface.empty()) { - m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), - Network::DEFAULT_NETWORK_INTERFACE_NAME, cid, status, error); - } else { - m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), networkInterface, cid, status, - error); + if (defaultInterface.empty()) { + defaultInterface = Network::DEFAULT_NETWORK_INTERFACE_NAME; } + + // step 1: get ips of default network + Network::PodNetworkStatus status; + m_pluginManager->GetPodNetworkStatus(metadata.namespace_(), metadata.name(), defaultInterface, inspect->id, status, + error); if (error.NotEmpty()) { - return ""; + return ret; + } + for (auto &iter : status.GetIPs()) { + ret.push_back(iter); } - return status.GetIP(); + // step 2: get ips of mutl networks + GetFormatIPsForMultNet(inspect, defaultInterface, metadata, ret, error); + + return ret; } -auto CRIRuntimeServiceImpl::GetIP(const std::string &podSandboxID, container_inspect *inspect, - const std::string &networkInterface, Errors &error) -> std::string +void CRIRuntimeServiceImpl::GetIPs(const std::string &podSandboxID, container_inspect *inspect, + const std::string &networkInterface, std::vector &ips, Errors &error) { if (inspect == nullptr || inspect->network_settings == nullptr) { - return ""; + return; } if (SharesHostNetwork(inspect) != 0) { // For sandboxes using host network, the shim is not responsible for reporting the IP. - return ""; + return; } bool ready = GetNetworkReady(podSandboxID, error); if (error.Empty() && !ready) { WARN("Network %s do not ready", podSandboxID.c_str()); - return ""; + return; } error.Clear(); - auto ip = GetIPFromPlugin(inspect, networkInterface, error); + auto tmpIPs = GetIPsFromPlugin(inspect, networkInterface, error); if (error.Empty()) { - return ip; + for (const auto &iter : tmpIPs) { + ips.push_back(iter); + } + return; } if (inspect->network_settings->ip_address != nullptr) { WARN("Use container inspect ip info: %s", error.GetCMessage()); error.Clear(); - return inspect->network_settings->ip_address; + ips.push_back(inspect->network_settings->ip_address); } WARN("Failed to read pod IP from plugin/docker: %s", error.GetCMessage()); - return ""; } std::unique_ptr diff --git a/src/daemon/entry/cri/network_plugin.cc b/src/daemon/entry/cri/network_plugin.cc index 0cab31a..311ebb6 100644 --- a/src/daemon/entry/cri/network_plugin.cc +++ b/src/daemon/entry/cri/network_plugin.cc @@ -65,25 +65,58 @@ static void runGetIP(void *cmdArgs) execvp(tmpArgs[0], args); } -static std::string GetOnePodIP(std::string nsenterPath, std::string netnsPath, std::string interfaceName, - std::string addrType, Errors &error) +static std::string ParseIPFromLine(const char *line, const char *stdout_str) { - char *stderr_str { nullptr }; - char *stdout_str { nullptr }; - char *strErr { nullptr }; - char **lines { nullptr }; + char *cIP { nullptr }; char **fields { nullptr }; + char *strErr { nullptr }; struct ipnet *ipnet_val { nullptr }; + std::string ret; + + fields = util_string_split(line, ' '); + if (fields == nullptr) { + ERROR("Out of memory"); + goto out; + } + if (util_array_len((const char **)fields) < 4) { + ERROR("Unexpected address output %s ", line); + goto out; + } + + if (parse_cidr(fields[3], &ipnet_val, &strErr) != 0) { + ERROR("CNI failed to parse ip from output %s due to %s", stdout_str, strErr); + goto out; + } + cIP = ip_to_string(ipnet_val->ip, ipnet_val->ip_len); + if (cIP == nullptr) { + ERROR("Out of memory"); + goto out; + } + + ret = cIP; +out: + free(cIP); + free(strErr); + free_ipnet_type(ipnet_val); + util_free_array(fields); + return ret; +} + +static void GetOnePodIP(std::string nsenterPath, std::string netnsPath, std::string interfaceName, + std::string addrType, std::vector &ips, Errors &error) +{ + char *stderr_str { nullptr }; + char *stdout_str { nullptr }; + char **lines { nullptr }; char **args { nullptr }; - std::string result { "" }; - char *cIP { nullptr }; + size_t i; args = (char **)util_common_calloc_s(sizeof(char *) * 5); if (args == nullptr) { error.SetError("Out of memory"); - return result; + return; } args[0] = util_strdup_s(nsenterPath.c_str()); @@ -102,52 +135,55 @@ static std::string GetOnePodIP(std::string nsenterPath, std::string netnsPath, s error.SetError("Out of memory"); goto free_out; } - if (util_array_len((const char **)lines) < 1) { - error.Errorf("Unexpected command output %s", stdout_str); - goto free_out; - } - fields = util_string_split(lines[0], ' '); - if (fields == nullptr) { - error.SetError("Out of memory"); - goto free_out; - } - if (util_array_len((const char **)fields) < 4) { - error.Errorf("Unexpected address output %s ", lines[0]); + if (util_array_len((const char **)lines) == 0) { + error.Errorf("Unexpected command output %s", stdout_str); goto free_out; } - if (parse_cidr(fields[3], &ipnet_val, &strErr) != 0) { - error.Errorf("CNI failed to parse ip from output %s due to %s", stdout_str, strErr); - goto free_out; - } - cIP = ip_to_string(ipnet_val->ip, ipnet_val->ip_len); - if (cIP == nullptr) { - error.SetError("Out of memory"); - goto free_out; + for (i = 0; i < util_array_len((const char **)lines); i++) { + // ip string min length must bigger than 4 + if (lines[i] == nullptr || strlen(lines[i]) < 4) { + continue; + } + std::string tIP = ParseIPFromLine(lines[i], stdout_str); + if (tIP.empty()) { + error.Errorf("parse %s to ip failed", lines[i]); + break; + } + ips.push_back(tIP); } - result = cIP; - free(cIP); free_out: - free_ipnet_type(ipnet_val); free(stdout_str); free(stderr_str); util_free_array(args); util_free_array(lines); - util_free_array(fields); - return result; } -std::string GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName, - Errors &error) +void GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName, + std::vector &getIPs, Errors &error) { - std::string ip = GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-4", error); + Errors tmpErr; + + GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-4", getIPs, tmpErr); + if (tmpErr.NotEmpty()) { + WARN("Get ipv4 failed: %s", tmpErr.GetCMessage()); + } + + GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-6", getIPs, error); if (error.NotEmpty()) { - return GetOnePodIP(nsenterPath, netnsPath, interfaceName, "-6", error); + WARN("Get ipv6 failed: %s", tmpErr.GetCMessage()); } - return ip; + if (getIPs.size() > 0) { + error.Clear(); + return; + } + + if (tmpErr.NotEmpty()) { + error.AppendError(tmpErr.GetMessage()); + } } void InitNetworkPlugin(std::vector> *plugins, std::string networkPluginName, @@ -290,14 +326,14 @@ void PodNetworkStatus::SetAPIVersion(const std::string &version) m_apiVersion = version; } -const std::string &PodNetworkStatus::GetIP() const +const std::vector &PodNetworkStatus::GetIPs() const { - return m_ip; + return m_ips; } -void PodNetworkStatus::SetIP(const std::string &ip) +void PodNetworkStatus::SetIPs(std::vector &ips) { - m_ip = ip; + m_ips = ips; } void PluginManager::Lock(const std::string &fullPodName, Errors &error) diff --git a/src/daemon/entry/cri/network_plugin.h b/src/daemon/entry/cri/network_plugin.h index 5a46eb8..24afd71 100644 --- a/src/daemon/entry/cri/network_plugin.h +++ b/src/daemon/entry/cri/network_plugin.h @@ -82,13 +82,13 @@ public: void SetKind(const std::string &kind); const std::string &GetAPIVersion() const; void SetAPIVersion(const std::string &version); - const std::string &GetIP() const; - void SetIP(const std::string &ip); + const std::vector &GetIPs() const; + void SetIPs(std::vector &ips); private: std::string m_kind; std::string m_apiVersion; - std::string m_ip; + std::vector m_ips; }; class NetworkPlugin { @@ -227,8 +227,8 @@ void InitNetworkPlugin(std::vector> *plugins, std void ProbeNetworkPlugins(const std::string &pluginDir, const std::string &binDir, std::vector> *plugins); -std::string GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName, - Errors &error); +void GetPodIP(const std::string &nsenterPath, const std::string &netnsPath, const std::string &interfaceName, + std::vector &getIPs, Errors &error); const std::string &GetInterfaceName(); } // namespace Network -- 2.20.1