From e83ba81129646c3242ed96496cf2d503f977a2e0 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Tue, 7 Mar 2023 13:51:57 -0500 Subject: [PATCH] libnet/d/overlay: document some encryption code The overlay-network encryption code is woefully under-documented, which is especially problematic as it operates on under-documented kernel interfaces. Document what I have puzzled out of the implementation for the benefit of the next poor soul to touch this code. Signed-off-by: Cory Snider --- .../libnetwork/drivers/overlay/encryption.go | 46 +++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go index be09c70d7a..2084351624 100644 --- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go +++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go @@ -19,8 +19,31 @@ import ( "github.com/vishvananda/netlink" ) +/* +Encrypted overlay networks use IPsec in transport mode to encrypt and +authenticate the VXLAN UDP datagrams. This driver implements a bespoke control +plane which negotiates the security parameters for each peer-to-peer tunnel. +IPsec Terminology + - ESP: IPSec Encapsulating Security Payload + - SPI: Security Parameter Index + - ICV: Integrity Check Value + - SA: Security Association https://en.wikipedia.org/wiki/IPsec#Security_association +Developer documentation for Linux IPsec is rather sparse online. The following +slide deck provides a decent overview. +https://libreswan.org/wiki/images/e/e0/Netdev-0x12-ipsec-flow.pdf +The Linux IPsec stack is part of XFRM, the netlink packet transformation +interface. +https://man7.org/linux/man-pages/man8/ip-xfrm.8.html +*/ + const ( - r = 0xD0C4E3 + // Value used to mark outgoing packets which should have our IPsec + // processing applied. It is also used as a label to identify XFRM + // states (Security Associations) and policies (Security Policies) + // programmed by us so we know which ones we can clean up without + // disrupting other VPN connections on the system. + mark = 0xD0C4E3 + pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8) ) @@ -30,7 +53,9 @@ const ( bidir ) -var spMark = netlink.XfrmMark{Value: uint32(r), Mask: 0xffffffff} +// Mark value for matching packets which should have our IPsec security policy +// applied. +var spMark = netlink.XfrmMark{Value: mark, Mask: 0xffffffff} type key struct { value []byte @@ -49,6 +74,9 @@ type spi struct { reverse int } +// Security Parameter Indices for the IPsec flows between local node and a +// remote peer, which identify the Security Associations (XFRM states) to be +// applied when encrypting and decrypting packets. func (s *spi) String() string { return fmt.Sprintf("SPI(FWD: 0x%x, REV: 0x%x)", uint32(s.forward), uint32(s.reverse)) } @@ -200,7 +228,7 @@ func removeEncryption(localIP, remoteIP net.IP, em *encrMap) error { func programMangle(vni uint32, add bool) (err error) { var ( - m = strconv.FormatUint(uint64(r), 10) + m = strconv.FormatUint(mark, 10) chain = "OUTPUT" rule = append(matchVXLAN(overlayutils.VXLANUDPPort(), vni), "-j", "MARK", "--set-mark", m) a = "-A" @@ -239,10 +267,12 @@ func programInput(vni uint32, add bool) (err error) { msg = "remove" } + // Accept incoming VXLAN datagrams for the VNI which were subjected to IPSec processing. if err := iptables.ProgramRule(iptables.Filter, chain, action, accept); err != nil { logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err) } + // Drop incoming VXLAN datagrams for the VNI which were received in cleartext. if err := iptables.ProgramRule(iptables.Filter, chain, action, block); err != nil { logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err) } @@ -268,7 +298,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f Proto: netlink.XFRM_PROTO_ESP, Spi: spi.reverse, Mode: netlink.XFRM_MODE_TRANSPORT, - Reqid: r, + Reqid: mark, } if add { rSA.Aead = buildAeadAlgo(k, spi.reverse) @@ -294,7 +324,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f Proto: netlink.XFRM_PROTO_ESP, Spi: spi.forward, Mode: netlink.XFRM_MODE_TRANSPORT, - Reqid: r, + Reqid: mark, } if add { fSA.Aead = buildAeadAlgo(k, spi.forward) @@ -343,7 +373,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error { Proto: netlink.XFRM_PROTO_ESP, Mode: netlink.XFRM_MODE_TRANSPORT, Spi: fSA.Spi, - Reqid: r, + Reqid: mark, }, }, } @@ -557,7 +587,7 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr Proto: netlink.XFRM_PROTO_ESP, Mode: netlink.XFRM_MODE_TRANSPORT, Spi: fSA2.Spi, - Reqid: r, + Reqid: mark, }, }, } @@ -624,7 +654,7 @@ func clearEncryptionStates() { } } for _, sa := range saList { - if sa.Reqid == r { + if sa.Reqid == mark { if err := nlh.XfrmStateDel(&sa); err != nil { logrus.Warnf("Failed to delete stale SA %s: %v", sa, err) continue -- 2.33.0