144 lines
5.3 KiB
Diff
144 lines
5.3 KiB
Diff
From e83ba81129646c3242ed96496cf2d503f977a2e0 Mon Sep 17 00:00:00 2001
|
|
From: Cory Snider <csnider@mirantis.com>
|
|
Date: Tue, 7 Mar 2023 13:51:57 -0500
|
|
Subject: [PATCH] libnet/d/overlay: document some encryption code
|
|
|
|
The overlay-network encryption code is woefully under-documented, which
|
|
is especially problematic as it operates on under-documented kernel
|
|
interfaces. Document what I have puzzled out of the implementation for
|
|
the benefit of the next poor soul to touch this code.
|
|
|
|
Signed-off-by: Cory Snider <csnider@mirantis.com>
|
|
---
|
|
.../libnetwork/drivers/overlay/encryption.go | 46 +++++++++++++++----
|
|
1 file changed, 38 insertions(+), 8 deletions(-)
|
|
|
|
diff --git a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go
|
|
index be09c70d7a..2084351624 100644
|
|
--- a/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go
|
|
+++ b/components/engine/vendor/github.com/docker/libnetwork/drivers/overlay/encryption.go
|
|
@@ -19,8 +19,31 @@ import (
|
|
"github.com/vishvananda/netlink"
|
|
)
|
|
|
|
+/*
|
|
+Encrypted overlay networks use IPsec in transport mode to encrypt and
|
|
+authenticate the VXLAN UDP datagrams. This driver implements a bespoke control
|
|
+plane which negotiates the security parameters for each peer-to-peer tunnel.
|
|
+IPsec Terminology
|
|
+ - ESP: IPSec Encapsulating Security Payload
|
|
+ - SPI: Security Parameter Index
|
|
+ - ICV: Integrity Check Value
|
|
+ - SA: Security Association https://en.wikipedia.org/wiki/IPsec#Security_association
|
|
+Developer documentation for Linux IPsec is rather sparse online. The following
|
|
+slide deck provides a decent overview.
|
|
+https://libreswan.org/wiki/images/e/e0/Netdev-0x12-ipsec-flow.pdf
|
|
+The Linux IPsec stack is part of XFRM, the netlink packet transformation
|
|
+interface.
|
|
+https://man7.org/linux/man-pages/man8/ip-xfrm.8.html
|
|
+*/
|
|
+
|
|
const (
|
|
- r = 0xD0C4E3
|
|
+ // Value used to mark outgoing packets which should have our IPsec
|
|
+ // processing applied. It is also used as a label to identify XFRM
|
|
+ // states (Security Associations) and policies (Security Policies)
|
|
+ // programmed by us so we know which ones we can clean up without
|
|
+ // disrupting other VPN connections on the system.
|
|
+ mark = 0xD0C4E3
|
|
+
|
|
pktExpansion = 26 // SPI(4) + SeqN(4) + IV(8) + PadLength(1) + NextHeader(1) + ICV(8)
|
|
)
|
|
|
|
@@ -30,7 +53,9 @@ const (
|
|
bidir
|
|
)
|
|
|
|
-var spMark = netlink.XfrmMark{Value: uint32(r), Mask: 0xffffffff}
|
|
+// Mark value for matching packets which should have our IPsec security policy
|
|
+// applied.
|
|
+var spMark = netlink.XfrmMark{Value: mark, Mask: 0xffffffff}
|
|
|
|
type key struct {
|
|
value []byte
|
|
@@ -49,6 +74,9 @@ type spi struct {
|
|
reverse int
|
|
}
|
|
|
|
+// Security Parameter Indices for the IPsec flows between local node and a
|
|
+// remote peer, which identify the Security Associations (XFRM states) to be
|
|
+// applied when encrypting and decrypting packets.
|
|
func (s *spi) String() string {
|
|
return fmt.Sprintf("SPI(FWD: 0x%x, REV: 0x%x)", uint32(s.forward), uint32(s.reverse))
|
|
}
|
|
@@ -200,7 +228,7 @@ func removeEncryption(localIP, remoteIP net.IP, em *encrMap) error {
|
|
|
|
func programMangle(vni uint32, add bool) (err error) {
|
|
var (
|
|
- m = strconv.FormatUint(uint64(r), 10)
|
|
+ m = strconv.FormatUint(mark, 10)
|
|
chain = "OUTPUT"
|
|
rule = append(matchVXLAN(overlayutils.VXLANUDPPort(), vni), "-j", "MARK", "--set-mark", m)
|
|
a = "-A"
|
|
@@ -239,10 +267,12 @@ func programInput(vni uint32, add bool) (err error) {
|
|
msg = "remove"
|
|
}
|
|
|
|
+ // Accept incoming VXLAN datagrams for the VNI which were subjected to IPSec processing.
|
|
if err := iptables.ProgramRule(iptables.Filter, chain, action, accept); err != nil {
|
|
logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err)
|
|
}
|
|
|
|
+ // Drop incoming VXLAN datagrams for the VNI which were received in cleartext.
|
|
if err := iptables.ProgramRule(iptables.Filter, chain, action, block); err != nil {
|
|
logrus.Errorf("could not %s input rule: %v. Please do it manually.", msg, err)
|
|
}
|
|
@@ -268,7 +298,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
|
|
Proto: netlink.XFRM_PROTO_ESP,
|
|
Spi: spi.reverse,
|
|
Mode: netlink.XFRM_MODE_TRANSPORT,
|
|
- Reqid: r,
|
|
+ Reqid: mark,
|
|
}
|
|
if add {
|
|
rSA.Aead = buildAeadAlgo(k, spi.reverse)
|
|
@@ -294,7 +324,7 @@ func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (f
|
|
Proto: netlink.XFRM_PROTO_ESP,
|
|
Spi: spi.forward,
|
|
Mode: netlink.XFRM_MODE_TRANSPORT,
|
|
- Reqid: r,
|
|
+ Reqid: mark,
|
|
}
|
|
if add {
|
|
fSA.Aead = buildAeadAlgo(k, spi.forward)
|
|
@@ -343,7 +373,7 @@ func programSP(fSA *netlink.XfrmState, rSA *netlink.XfrmState, add bool) error {
|
|
Proto: netlink.XFRM_PROTO_ESP,
|
|
Mode: netlink.XFRM_MODE_TRANSPORT,
|
|
Spi: fSA.Spi,
|
|
- Reqid: r,
|
|
+ Reqid: mark,
|
|
},
|
|
},
|
|
}
|
|
@@ -557,7 +587,7 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr
|
|
Proto: netlink.XFRM_PROTO_ESP,
|
|
Mode: netlink.XFRM_MODE_TRANSPORT,
|
|
Spi: fSA2.Spi,
|
|
- Reqid: r,
|
|
+ Reqid: mark,
|
|
},
|
|
},
|
|
}
|
|
@@ -624,7 +654,7 @@ func clearEncryptionStates() {
|
|
}
|
|
}
|
|
for _, sa := range saList {
|
|
- if sa.Reqid == r {
|
|
+ if sa.Reqid == mark {
|
|
if err := nlh.XfrmStateDel(&sa); err != nil {
|
|
logrus.Warnf("Failed to delete stale SA %s: %v", sa, err)
|
|
continue
|
|
--
|
|
2.33.0
|
|
|