From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001 From: "Xinle.Guo" Date: Fri, 14 Jan 2022 17:07:37 +0800 Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to kata container Because stratovirt supports both microVM and standardVM architectures, adapts two sandbox types for kata container. Besides basic features, it also support virtio-fs, hotplug VFIO, support more devices. Signed-off-by: Xinle.Guo --- src/runtime/pkg/katautils/config.go | 1 + src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++--- 2 files changed, 438 insertions(+), 45 deletions(-) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index e523ed3..b04cdee 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, EnableIOThreads: h.EnableIOThreads, + PCIeRootPort: h.PCIeRootPort, DisableVhostNet: h.DisableVhostNet, EnableVhostUserStore: h.EnableVhostUserStore, VhostUserStorePath: h.vhostUserStorePath(), diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go index 4fcfb94..ffe8965 100644 --- a/src/runtime/virtcontainers/stratovirt.go +++ b/src/runtime/virtcontainers/stratovirt.go @@ -14,14 +14,14 @@ import ( "time" govmmQemu "github.com/kata-containers/govmm/qemu" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" "go.opentelemetry.io/otel" otelLabel "go.opentelemetry.io/otel/label" otelTrace "go.opentelemetry.io/otel/trace" @@ -44,6 +44,11 @@ const ( const ( WaitSandboxTimeoutSecs = 15 MachineTypeMicrovm = "microvm" + MachineTypeQ35 = "q35" + MachineTypeVirt = "virt" + RootPortPrefix = "pcie" + Q35PFlashCode = "/usr/share/edk2/ovmf/OVMF_CODE.fd" + VirtPFlashCode = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw" MmioBus VirtioDriver = "mmio" PciBus VirtioDriver = "pci" ) @@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver { } } +type rootPortDevice struct { + id string + port string + bus string + slot int + plugged bool + addedDev string +} + +func (r rootPortDevice) isVaild() bool { + if r.id == "" || r.port == "" { + return false + } + return true +} + +func (r rootPortDevice) getParams(config *vmConfig) []string { + if !r.isVaild() { + return nil + } + + var params []string + var devParams []Param + devParams = append(devParams, Param{"id", r.id}) + devParams = append(devParams, Param{"port", r.port}) + if r.bus == "" { + r.bus = "pcie.0" + } + devParams = append(devParams, Param{"bus", r.bus}) + devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)}) + + driver := "pcie-root-port" + params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) + return params +} + type blkDevice struct { id string filePath string @@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string { return params } +type virtioFs struct { + driver VirtioDriver + backend string + charID string + charDev string + tag string + deviceID string + bus string + addr string +} + +var virtiofsDriver = map[VirtioDriver]string{ + MmioBus: "vhost-user-fs-device", + PciBus: "vhost-user-fs-pci", +} + +func (v virtioFs) isVaild() bool { + if v.charID == "" || v.charDev == "" || v.deviceID == "" { + return false + } + return true +} + +func (v virtioFs) getParams(config *vmConfig) []string { + if !v.isVaild() { + return nil + } + + var params []string + var charParams []Param + var fsParams []Param + + charParams = append(charParams, Param{"id", v.charID}) + charParams = append(charParams, Param{"path", config.fsSockPath}) + + v.driver = v.driver.getDriver(config) + driver := virtiofsDriver[v.driver] + fsParams = append(fsParams, Param{"chardev", v.charDev}) + fsParams = append(fsParams, Param{"tag", v.tag}) + fsParams = append(fsParams, Param{"id", v.deviceID}) + if v.bus != "" { + fsParams = append(fsParams, Param{"bus", v.bus}) + fsParams = append(fsParams, Param{"addr", v.addr}) + } + + params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ","))) + params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ","))) + return params +} + type vhostVsock struct { driver VirtioDriver id string @@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) { for _, d := range c.devices { *params = append(*params, d.getParams(c)...) } + + if c.machineType == MachineTypeMicrovm { + return + } + // Add flag to unplug devices from their root port faster. + *params = append(*params, "-global", "pcie-root-port.fast-unplug=1") } func (c *vmConfig) appendPFlash(params *[]string) { @@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) { type State struct { mmioBlkSlots [mmioBlkCount]bool mmioNetSlots [mmioNetCount]bool + // The list of RootPorts that can be hot-added. + rootPort []rootPortDevice pid int virtiofsPid int } @@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str return strings.Join(params, " "), nil } +func (s *stratovirt) getPFlash(machineType string) ([]string, error) { + if s.config.PFlash != nil { + return s.config.PFlash, nil + } + + var PFlash []string + switch machineType { + case MachineTypeQ35: + PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode)) + case MachineTypeVirt: + PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode)) + case MachineTypeMicrovm: + return nil, nil + default: + return nil, fmt.Errorf("failed to match machine type %s", machineType) + } + + return PFlash, nil +} + func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket { socketPath := filepath.Join(vmPath, apiSocket) @@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev { } } + // Create root port for all devices that need to be hot-added. + if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 { + devices = s.appendRootPort(ctx, devices) + } + + return devices +} + +func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev { + number := s.config.PCIeRootPort + + for i := uint32(1); i < number+1; i++ { + addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i)) + if err != nil { + return devices + } + + rp := rootPortDevice{ + id: fmt.Sprintf("%s.%d", RootPortPrefix, i), + port: fmt.Sprintf("%d", i), + bus: defaultBridgeBus, + slot: int(addr), + addedDev: "", + } + s.state.rootPort = append(s.state.rootPort, rp) + devices = append(devices, rp) + } + return devices } @@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end return devices } +func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev { + if s.config.SharedFS != config.VirtioFS { + return devices + } + + var bus string + var addr uint32 + var err error + name := "virtio_fs" + + if s.vmConfig.machineType != MachineTypeMicrovm { + bus = "pcie.0" + addr, err = s.vmConfig.rootBus.AddDevice(ctx, name) + if err != nil { + return devices + } + } + + devices = append(devices, virtioFs{ + backend: "socket", + // Virtio-fs must be bound to unique charDev, it uses the same name. + charID: name, + charDev: name, + tag: volume.MountTag, + deviceID: "virtio-fs0", + bus: bus, + addr: fmt.Sprintf("%d", addr), + }) + + return devices +} + func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error { span, _ := s.trace(s.ctx, "setStratoVirtUp") defer span.End() @@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) return err } - var PFlash []string + PFlash, err := s.getPFlash(machineType) + if err != nil { + return err + } vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id) qmpSocket := s.createQMPSocket(vmPath) @@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error { return nil } -func (s *stratovirt) hypervisorConfig() HypervisorConfig { - return s.config +// Virtio fs daemon is a shared file system that lets VM access a directory +// tree on the host. +func (s *stratovirt) setupVirtioFs() error { + if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS { + return nil + } + + if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) { + return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon) + } + + args := []string{ + "-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"), + "-source", getSharePath(s.id)} + if len(s.config.VirtioFSExtraArgs) != 0 { + args = append(args, s.config.VirtioFSExtraArgs...) + } + + cmd := exec.Command(s.config.VirtioFSDaemon, args...) + s.Logger().Info("Virtiofsd start with cmd: ", cmd) + + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to strat virtiofsd: %v", cmd) + } + + s.state.virtiofsPid = cmd.Process.Pid + return nil } // Get StratoVirt binary path. @@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) { return path, nil } +func (s *stratovirt) hypervisorConfig() HypervisorConfig { + return s.config +} + func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { var span otelTrace.Span span, _ = s.trace(ctx, "createSandbox") @@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { } }() + if err = s.setupVirtioFs(); err != nil { + return err + } + var params []string s.createBaseParams(s.vmConfig, ¶ms) @@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v) case config.BlockDrive: s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices) + case types.Volume: + s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v) default: s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type") } return nil } -func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { +func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) { Name = filepath.Base(strings.ToLower(Name)) - if strings.HasPrefix(Name, "eth") { idxStr := strings.TrimPrefix(Name, "eth") if idxStr == Name { - return 0, fmt.Errorf("Could not parse idx from Name %q", Name) + return 0, fmt.Errorf("could not parse idx from name %q", Name) } idx, err := strconv.Atoi(idxStr) if err != nil { - return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) + return 0, fmt.Errorf("could not convert to int from str %q", idxStr) } if !isPut && s.state.mmioNetSlots[idx] { - return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) + return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr) } s.state.mmioNetSlots[idx] = !isPut @@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { } else if strings.HasPrefix(Name, "vd") { charStr := strings.TrimPrefix(Name, "vd") if charStr == Name { - return 0, fmt.Errorf("Could not parse idx from Name %q", Name) + return 0, fmt.Errorf("could not parse idx from name %q", Name) } char := []rune(charStr) idx := int(char[0] - 'a') if !isPut && s.state.mmioBlkSlots[idx] { - return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) + return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr) } s.state.mmioBlkSlots[idx] = !isPut return idx, nil } - return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) + return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name) +} + +func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) { + rootports := &s.state.rootPort + if len(*rootports) == 0 { + return "", 0, fmt.Errorf("failed to get available address from bridges") + } + + for i, rootport := range *rootports { + if !isPut && !rootport.plugged && rootport.addedDev == "" { + (*rootports)[i].plugged = true + (*rootports)[i].addedDev = Name + return rootport.id, rootport.slot, nil + } else if isPut && rootport.plugged && rootport.addedDev == Name { + (*rootports)[i].plugged = false + (*rootports)[i].addedDev = "" + return rootport.id, rootport.slot, nil + } + } + + return "", 0, fmt.Errorf("no more bridge slots available") +} + +func (s *stratovirt) getDevSlot(Name string) (string, int, error) { + if s.config.HypervisorMachineType == MachineTypeMicrovm { + slot, err := s.setupMmioSlot(Name, false) + if err != nil { + return "", 0, err + } + + return "", slot, nil + } + + bus, slot, err := s.setupPciSlot(Name, false) + if err != nil { + return "pcie.0", 0, err + } + + return bus, slot, nil +} + +func (s *stratovirt) delDevSlot(Name string) error { + if s.vmConfig.machineType == MachineTypeMicrovm { + if _, err := s.setupMmioSlot(Name, true); err != nil { + return err + } + + return nil + } + + if _, _, err := s.setupPciSlot(Name, true); err != nil { + return err + } + + return nil } -func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) { - err = s.qmpSetup() +func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error { + err := s.qmpSetup() if err != nil { return err } @@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera return fmt.Errorf("Endpoint is not supported") } + defer func() { + if err != nil { + if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } + } + }() + switch op { case addDevice: var ( @@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera VhostFdNames = append(VhostFdNames, fdName) } - slot, err := s.getDevSlot(endpoint.Name(), false) + bus, slot, err := s.getDevSlot(endpoint.Name()) if err != nil { - return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) + return fmt.Errorf("could not get unused slot for %q", endpoint.Name()) } if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { - s.getDevSlot(endpoint.Name(), true) return err } } else { if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { - s.getDevSlot(endpoint.Name(), true) return err } } - if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { - s.getDevSlot(endpoint.Name(), true) + + // The slot of net device that hotplugged to the root port + // must be zero. + devAddr := "0x0.0x0" + if s.vmConfig.machineType == MachineTypeMicrovm { + devAddr = fmt.Sprintf("%d", slot) + } else { + bridgeSlot, err := vcTypes.PciSlotFromInt(slot) + if err != nil { + return err + } + + devSlot, err := vcTypes.PciSlotFromString("0") + if err != nil { + return err + } + + pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + if err != nil { + return err + } + endpoint.SetPciPath(pciPath) + } + + if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil { return err } case removeDevice: + if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { return err } @@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera return err } default: - return fmt.Errorf("Operation is not supported") + return fmt.Errorf("operation is not supported") } return nil } -func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { - var filePath string - err = s.qmpSetup() +func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error { + err := s.qmpSetup() if err != nil { return err } + driver := "virtio-blk-pci" + if s.vmConfig.machineType == MachineTypeMicrovm { + driver = "virtio-blk-mmio" + } + + defer func() { + if err != nil { + s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID) + if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } + } + }() + switch op { case addDevice: - driver := "virtio-blk-mmio" + filePath := drive.File if s.vmConfig.useOzone { filePath, err = s.updateOzoneRes(drive.File, true) - if err != nil { - return fmt.Errorf("Failed to update ozone resources") - } - } else { - filePath = drive.File - } - slot, err := s.getDevSlot(drive.VirtPath, false) - if err != nil { - return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) } if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil { - s.getDevSlot(drive.VirtPath, true) return err } - if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { - s.getDevSlot(drive.VirtPath, true) + bus, slot, err := s.getDevSlot(drive.VirtPath) + if err != nil { + return err + } + + // The slot of block device that hotplugged to the root port + // must be zero. + devAddr := "0x0.0x0" + if s.vmConfig.machineType == MachineTypeMicrovm { + devAddr = fmt.Sprintf("%d", slot) + } else { + bridgeSlot, err := vcTypes.PciSlotFromInt(slot) + if err != nil { + return err + } + + devSlot, err := vcTypes.PciSlotFromString("0") + if err != nil { + return err + } + + drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) + if err != nil { + return err + } + } + + if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil { return err } case removeDevice: if s.vmConfig.useOzone { s.updateOzoneRes(drive.File, false) } + + if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { return err } if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { return err } + default: + return fmt.Errorf("operation is not supported %d", op) + } + + return nil +} + +func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error { + err := s.qmpSetup() + if err != nil { + return err + } + + defer func() { + if err != nil { + if errDel := s.delDevSlot(device.ID); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } + } + }() - s.getDevSlot(drive.VirtPath, true) + switch op { + case addDevice: + var bus string + // The slot of block device that hotplugged to the root port + // must be zero. + devAddr := "0x0.0x0" + // The vfio device BDF format is 0000:1a:00.3 + device.BDF = "0000:" + device.BDF + + bus, _, err = s.getDevSlot(device.ID) + if err != nil { + return err + } + + if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil { + return err + } + case removeDevice: + if errDel := s.delDevSlot(device.ID); errDel != nil { + s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") + } + if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil { + return err + } default: - return fmt.Errorf("Operation is not supported") + return fmt.Errorf("operation is not supported %d", op) } return nil @@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, case netDev: return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice) case blockDev: - return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) + return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice) + case vfioDev: + return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice) default: - return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) + return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType) } } @@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ case netDev: return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice) case blockDev: - return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) + return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice) + case vfioDev: + return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice) default: - return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) + return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType) } } @@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities { var caps types.Capabilities caps.SetBlockDeviceHotplugSupport() + if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS { + caps.SetFsSharingSupport() + } + return caps } @@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int { pids = append(pids, s.state.pid) + if s.state.virtiofsPid != 0 { + pids = append(pids, s.state.virtiofsPid) + } return pids } @@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool { func (s *stratovirt) save() (p persistapi.HypervisorState) { pids := s.getPids() p.Pid = pids[0] + p.VirtiofsdPid = s.state.virtiofsPid p.Type = string(StratovirtHypervisor) return } func (s *stratovirt) load(p persistapi.HypervisorState) { s.state.pid = p.Pid + s.state.virtiofsPid = p.VirtiofsdPid } func (s *stratovirt) setSandbox(sandbox *Sandbox) { -- 2.20.1.windows.1