diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..9d2060f07a18958f3c812a21d8d21b1f2361af4a --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +kata-containers-3.2.0-vendor.tar.gz filter=lfs diff=lfs merge=lfs -text diff --git a/apply-patches b/apply-patches index 1d34b0208bc4f41e9b03ad602abcff7bcd8f5b93..8e0a2fee3c4a464c65394b09c7b7d65b29f8dbac 100755 --- a/apply-patches +++ b/apply-patches @@ -7,8 +7,8 @@ if [[ -f ./patch_flag ]];then exit 0 fi -tar -zxvf kata-containers-2.1.0.tar.gz -cp -rf ./kata-containers-2.1.0/* ./ +tar -zxvf kata-containers-3.2.0.tar.gz +cp -rf ./kata-containers-3.2.0/* ./ cat ./series.conf | while read line do if [[ $line == '' || $line =~ ^\s*# ]]; then diff --git a/kata-containers-3.2.0-vendor.tar.gz b/kata-containers-3.2.0-vendor.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1abbc164940cb9ecdfb530d66d7adf88d9b2eeb2 --- /dev/null +++ b/kata-containers-3.2.0-vendor.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b1a5424a14389860bf34779da3f6475dbc12a8540f17c1166ebf7d45637eeac +size 371103955 diff --git a/kata-containers-2.1.0.tar.gz b/kata-containers-3.2.0.tar.gz similarity index 65% rename from kata-containers-2.1.0.tar.gz rename to kata-containers-3.2.0.tar.gz index 3b2255ea36562c4993fde5068abf7eff369e2904..5512a57e81d4affff896407b6f4c99de57ada014 100644 Binary files a/kata-containers-2.1.0.tar.gz and b/kata-containers-3.2.0.tar.gz differ diff --git a/kata-containers.spec b/kata-containers.spec index a05b33c7283c974d7ac671ffdd33ab6ed5cc9bad..273cc5bab6c98797e29db16b628ca574199a0e81 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -1,8 +1,8 @@ #needsrootforbuild %global debug_package %{nil} -%define VERSION 2.1.0 -%define RELEASE 33 +%define VERSION 3.2.0 +%define RELEASE 1 Name: kata-containers Version: %{VERSION} @@ -23,7 +23,7 @@ This is core component of Kata Container, to make it work, you need a isulad/doc %prep %setup -T -c -a 0 -n kata_integration -%setup -T -c -a 1 -n kata-containers-%{version} +%setup -T -c -a 1 -n kata-containers %setup -T -c -a 2 -n kernel # extract the kata_integration.tar.gz file @@ -46,16 +46,15 @@ cd %{_builddir}/kernel/linux/ make olddefconfig make %{?_smp_mflags} -mv %{_builddir}/kata-containers-%{version} %{_builddir}/kata-containers cd %{_builddir}/kata-containers/ sh -x apply-patches +tar -xzf kata-containers-%{version}-vendor.tar.gz cd %{_builddir}/kata-containers/src/runtime make clean make cd %{_builddir}/kata-containers/src/agent -mkdir vendor && tar -xzf %{_builddir}/kata-containers/vendor.tar.gz -C vendor/ -cp -f ./vendor/version.rs ./src/ +cp %{_builddir}/kata-containers/version.rs ./src/ cat > .cargo/config << EOF [build] rustflags = ["-Clink-arg=-s","-Clink-arg=-lgcc","-Clink-arg=-lfdt"] @@ -84,11 +83,9 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot cd %{_builddir}/kata_integration mkdir -p -m 750 %{buildroot}/usr/bin install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-runtime %{buildroot}/usr/bin/ -install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-netmon %{buildroot}/usr/bin/ install -p -m 750 %{_builddir}/kata-containers/src/runtime/kata-monitor %{buildroot}/usr/bin/ install -p -m 750 %{_builddir}/kata-containers/src/runtime/containerd-shim-kata-v2 %{buildroot}/usr/bin/ -install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml -install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/cli/config/configuration-stratovirt.toml %{buildroot}/usr/share/defaults/kata-containers/configuration-stratovirt.toml +install -p -m 640 -D %{_builddir}/kata-containers/src/runtime/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/ mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/ strip %{buildroot}/usr/bin/kata* @@ -98,17 +95,21 @@ strip %{buildroot}/usr/bin/containerd-shim-kata-v2 %files /usr/bin/kata-runtime -/usr/bin/kata-netmon /usr/bin/kata-monitor /usr/bin/containerd-shim-kata-v2 /var/lib/kata/kernel /var/lib/kata/kata-containers-initrd.img %config(noreplace) /usr/share/defaults/kata-containers/configuration.toml -%config(noreplace) /usr/share/defaults/kata-containers/configuration-stratovirt.toml %doc %changelog +* Tue Feb 2024 Vanient - 3.2.0-1 +- Type:feature +- ID:NA +- SUG:NA +- DESC:update to 3.2.0 + * Tue Sep 2023 xiadanni - 2.1.0-33 - Type:bugfix - ID:NA diff --git a/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch b/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch deleted file mode 100644 index d63805358c9027d94d18eb391cb7ceb110d3e823..0000000000000000000000000000000000000000 --- a/patches/0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch +++ /dev/null @@ -1,837 +0,0 @@ -From f56d66f196bee808526e86df2c3c063a887c6fef Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Sat, 7 Aug 2021 10:39:11 +0800 -Subject: [PATCH 1/6] runtime: add support of new sandbox hypervisor type - StratoVirt. - -Signed-off-by: Wei Gao ---- - src/runtime/pkg/katautils/config.go | 96 +++- - src/runtime/virtcontainers/hypervisor.go | 12 + - src/runtime/virtcontainers/stratovirt.go | 642 +++++++++++++++++++++++ - 3 files changed, 749 insertions(+), 1 deletion(-) - create mode 100644 src/runtime/virtcontainers/stratovirt.go - -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index 6114aa39..f94ac4fd 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -50,6 +50,7 @@ const ( - clhHypervisorTableType = "clh" - qemuHypervisorTableType = "qemu" - acrnHypervisorTableType = "acrn" -+ stratovirtHypervisorTable = "stratovirt" - - // the maximum amount of PCI bridges that can be cold plugged in a VM - maxPCIBridges uint32 = 5 -@@ -870,6 +871,96 @@ func newClhHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - }, nil - } - -+func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { -+ hypervisor, err := h.path() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ kernel, err := h.kernel() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ initrd, image, err := h.getInitrdAndImage() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ if image != "" && initrd != "" { -+ return vc.HypervisorConfig{}, -+ errors.New("having both an image and an initrd defined in the configuration file is not supported") -+ } -+ -+ if image == "" && initrd == "" { -+ return vc.HypervisorConfig{}, -+ errors.New("either image or initrd must be defined in the configuration file") -+ } -+ -+ kernelParams := h.kernelParams() -+ machineType := h.machineType() -+ -+ blockDriver, err := h.blockDeviceDriver() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ sharedFS, err := h.sharedFS() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ if sharedFS == config.VirtioFS && h.VirtioFSDaemon == "" { -+ return vc.HypervisorConfig{}, -+ errors.New("cannot enable virtio-fs without daemon path in configuration file") -+ } -+ -+ if vSock, err := utils.SupportsVsocks(); !vSock { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ return vc.HypervisorConfig{ -+ HypervisorPath: hypervisor, -+ HypervisorPathList: h.HypervisorPathList, -+ KernelPath: kernel, -+ InitrdPath: initrd, -+ ImagePath: image, -+ KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), -+ HypervisorMachineType: machineType, -+ NumVCPUs: h.defaultVCPUs(), -+ DefaultMaxVCPUs: h.defaultMaxVCPUs(), -+ MemorySize: h.defaultMemSz(), -+ MemSlots: h.defaultMemSlots(), -+ MemOffset: h.defaultMemOffset(), -+ EntropySource: h.GetEntropySource(), -+ EntropySourceList: h.EntropySourceList, -+ DefaultBridges: h.defaultBridges(), -+ DisableBlockDeviceUse: h.DisableBlockDeviceUse, -+ SharedFS: sharedFS, -+ VirtioFSDaemon: h.VirtioFSDaemon, -+ VirtioFSDaemonList: h.VirtioFSDaemonList, -+ VirtioFSCacheSize: h.VirtioFSCacheSize, -+ VirtioFSCache: h.defaultVirtioFSCache(), -+ VirtioFSExtraArgs: h.VirtioFSExtraArgs, -+ FileBackedMemRootDir: h.FileBackedMemRootDir, -+ FileBackedMemRootList: h.FileBackedMemRootList, -+ Mlock: !h.Swap, -+ Debug: h.Debug, -+ DisableNestingChecks: h.DisableNestingChecks, -+ BlockDeviceDriver: blockDriver, -+ BlockDeviceCacheSet: h.BlockDeviceCacheSet, -+ BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, -+ BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, -+ EnableIOThreads: h.EnableIOThreads, -+ DisableVhostNet: h.DisableVhostNet, -+ EnableVhostUserStore: h.EnableVhostUserStore, -+ VhostUserStorePath: h.vhostUserStorePath(), -+ VhostUserStorePathList: h.VhostUserStorePathList, -+ GuestHookPath: h.guestHookPath(), -+ EnableAnnotations: h.EnableAnnotations, -+ }, nil -+} -+ - func newFactoryConfig(f factory) (oci.FactoryConfig, error) { - if f.TemplatePath == "" { - f.TemplatePath = defaultTemplatePath -@@ -903,6 +994,9 @@ func updateRuntimeConfigHypervisor(configPath string, tomlConf tomlConfig, confi - case clhHypervisorTableType: - config.HypervisorType = vc.ClhHypervisor - hConfig, err = newClhHypervisorConfig(hypervisor) -+ case stratovirtHypervisorTable: -+ config.HypervisorType = vc.StratovirtHypervisor -+ hConfig, err = newStratovirtHypervisorConfig(hypervisor) - } - - if err != nil { -@@ -1287,7 +1381,7 @@ func checkHypervisorConfig(config vc.HypervisorConfig) error { - memSizeMB := int64(config.MemorySize) - - if memSizeMB == 0 { -- return errors.New("VM memory cannot be zero") -+ return errors.New(fmt.Sprintf("The VM memory cannot be zero, %s", config.ImagePath)) - } - - mb := int64(1024 * 1024) -diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go -index 767215b6..615baa80 100644 ---- a/src/runtime/virtcontainers/hypervisor.go -+++ b/src/runtime/virtcontainers/hypervisor.go -@@ -44,6 +44,9 @@ const ( - // ClhHypervisor is the ICH hypervisor. - ClhHypervisor HypervisorType = "clh" - -+ // StratovirtHypervisor is the StratoVirt hypervisor -+ StratovirtHypervisor HypervisorType = "stratovirt" -+ - // MockHypervisor is a mock hypervisor for testing purposes - MockHypervisor HypervisorType = "mock" - ) -@@ -159,6 +162,9 @@ func (hType *HypervisorType) Set(value string) error { - case "clh": - *hType = ClhHypervisor - return nil -+ case "stratovirt": -+ *hType = StratovirtHypervisor -+ return nil - case "mock": - *hType = MockHypervisor - return nil -@@ -178,6 +184,8 @@ func (hType *HypervisorType) String() string { - return string(AcrnHypervisor) - case ClhHypervisor: - return string(ClhHypervisor) -+ case StratovirtHypervisor: -+ return string(StratovirtHypervisor) - case MockHypervisor: - return string(MockHypervisor) - default: -@@ -207,6 +215,10 @@ func newHypervisor(hType HypervisorType) (hypervisor, error) { - return &cloudHypervisor{ - store: store, - }, nil -+ case StratovirtHypervisor: -+ return &stratovirt{ -+ store: store, -+ }, nil - case MockHypervisor: - return &mockHypervisor{}, nil - default: -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -new file mode 100644 -index 00000000..4fec96d3 ---- /dev/null -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -0,0 +1,642 @@ -+package virtcontainers -+ -+import ( -+ "context" -+ "fmt" -+ "os" -+ "os/exec" -+ "path/filepath" -+ "strconv" -+ "strings" -+ "syscall" -+ "time" -+ -+ govmmQemu "github.com/kata-containers/govmm/qemu" -+ "github.com/pkg/errors" -+ "github.com/sirupsen/logrus" -+ -+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" -+ persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" -+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" -+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" -+ "go.opentelemetry.io/otel" -+ otelLabel "go.opentelemetry.io/otel/label" -+ otelTrace "go.opentelemetry.io/otel/trace" -+) -+ -+const defaultDummyMac = "22:33:44:aa:bb:" -+const mmioBlkCount = 4 -+const mmioNetCount = 2 -+const randomDevice = "/dev/urandom" -+ -+type stratovirtDev struct { -+ dev interface{} -+ devType deviceType -+} -+ -+type stratovirt struct { -+ id string -+ ctx context.Context -+ sandbox *Sandbox -+ store persistapi.PersistDriver -+ config HypervisorConfig -+ pid int -+ consolePath string -+ socketPath string -+ qmpMonitorCh qmpChannel -+ devices []stratovirtDev -+ HotpluggedVCPUs []CPUDevice -+ mmioBlkSlots [mmioBlkCount]bool -+ mmioNetSlots [mmioNetCount]bool -+} -+ -+func (s *stratovirt) Logger() *logrus.Entry { -+ return virtLog.WithField("subsystem", "stratovirt") -+} -+ -+func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, context.Context) { -+ if parent == nil { -+ s.Logger().WithField("type", "bug").Error("trace called before context set") -+ parent = context.Background() -+ } -+ -+ tracer := otel.Tracer("kata") -+ ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id))) -+ -+ return span, ctx -+} -+ -+func (s *stratovirt) getKernelCmdLine(useImage bool) string { -+ var params []string -+ -+ if useImage { -+ params = append(params, "root=/dev/vda") -+ } -+ -+ params = append(params, "pci=off") -+ params = append(params, "reboot=k") -+ params = append(params, "panic=1") -+ params = append(params, "iommu=off") -+ params = append(params, "acpi=off") -+ params = append(params, "quiet") -+ params = append(params, "agent.use_vsock=true") -+ params = append(params, "random.trust_cpu=on") -+ params = append(params, "rw") -+ params = append(params, SerializeParams(s.config.KernelParams, "=")...) -+ -+ return strings.Join(params, " ") -+} -+ -+func (s *stratovirt) hypervisorConfig() HypervisorConfig { -+ return s.config -+} -+ -+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { -+ s.ctx = ctx -+ -+ var span otelTrace.Span -+ span, _ = s.trace(ctx, "createSandbox") -+ defer span.End() -+ -+ s.id = id -+ s.config = *hypervisorConfig -+ -+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") -+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") -+ s.qmpMonitorCh = qmpChannel{ -+ ctx: s.ctx, -+ path: s.socketPath, -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) waitSandBoxStarted(timeout int) error { -+ timeStart := time.Now() -+ for { -+ err := s.qmpSetup() -+ if err == nil { -+ break -+ } -+ -+ if int(time.Since(timeStart).Seconds()) > timeout { -+ return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err) -+ } -+ -+ time.Sleep(time.Duration(50) * time.Millisecond) -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil { -+ s.Logger().WithError(err).Error(qmpCapErrMsg) -+ return err -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { -+ span, _ := s.trace(ctx, "startSandbox") -+ defer span.End() -+ -+ var params []string -+ var use_image bool -+ params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) -+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) -+ -+ if kernelPath, err := s.config.KernelAssetPath(); err == nil { -+ params = append(params, "-kernel", kernelPath) -+ } -+ -+ initrdPath, err := s.config.InitrdAssetPath() -+ if err != nil { -+ return err -+ } -+ -+ if initrdPath == "" { -+ imagePath, err := s.config.ImageAssetPath() -+ if err != nil { -+ return err -+ } -+ use_image = true -+ s.mmioBlkSlots[0] = true -+ params = append(params, "-device", "virtio-blk-device,drive=rootfs") -+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath)) -+ } else { -+ use_image = false -+ params = append(params, "-initrd", initrdPath) -+ } -+ -+ params = append(params, "-append", s.getKernelCmdLine(use_image)) -+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) -+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) -+ params = append(params, "-device", "virtio-serial-device") -+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") -+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) -+ params = append(params, "-device", "virtio-rng-device,rng=objrng0") -+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) -+ params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) -+ -+ // add devices to cmdline -+ for _, d := range s.devices { -+ switch v := d.dev.(type) { -+ case Endpoint: -+ name := v.Name() -+ mac := v.HardwareAddr() -+ tapName := v.NetworkPair().TapInterface.TAPIface.Name -+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) -+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) -+ case config.BlockDrive: -+ id := v.ID -+ path := v.File -+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id)) -+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) -+ case types.VSock: -+ v.VhostFd.Close() -+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) -+ default: -+ s.Logger().Error("Adding device type is unsupported") -+ } -+ } -+ -+ // daemonize -+ params = append(params, "-daemonize") -+ -+ // append logfile only on debug -+ if s.config.Debug { -+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -+ params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) -+ } -+ -+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -+ err = os.MkdirAll(dir, DirMode) -+ if err != nil { -+ return err -+ } -+ defer func() { -+ if err != nil { -+ if err := os.RemoveAll(dir); err != nil { -+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) -+ } -+ } -+ }() -+ -+ binPath, err := s.config.HypervisorAssetPath() -+ if err != nil { -+ s.Logger().WithField("Fail to get hypervisor bin path", err).Error() -+ return err -+ } -+ -+ cmd := exec.CommandContext(s.ctx, binPath, params...) -+ s.Logger().Info("StratoVirt start with params: ", cmd) -+ -+ if err := cmd.Start(); err != nil { -+ s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() -+ return err -+ } -+ s.pid = cmd.Process.Pid -+ -+ if err = s.waitSandBoxStarted(timeout); err != nil { -+ return err -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { -+ span, _ := s.trace(ctx, "stopSandbox") -+ defer span.End() -+ -+ defer func() { -+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -+ link, _ := filepath.EvalSymlinks(dir) -+ -+ if err := os.RemoveAll(dir); err != nil { -+ s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir) -+ } -+ -+ if link != dir && link != "" { -+ if err := os.RemoveAll(link); err != nil { -+ s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link) -+ } -+ } -+ }() -+ -+ if !force { -+ err := s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) -+ if err != nil { -+ s.Logger().WithError(err).Error("Fail to execute qmp: QUIT") -+ return err -+ } -+ } else { -+ if s.pid > 0 { -+ syscall.Kill(s.pid, syscall.SIGKILL) -+ } -+ } -+ return nil -+} -+ -+func (s *stratovirt) pauseSandbox(ctx context.Context) error { -+ return nil -+} -+ -+func (s *stratovirt) saveSandbox() error { -+ return nil -+} -+ -+func (s *stratovirt) resumeSandbox(ctx context.Context) error { -+ return nil -+} -+ -+func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType deviceType) error { -+ span, _ := s.trace(ctx, "addDevice") -+ defer span.End() -+ -+ dev := stratovirtDev{ -+ dev: devInfo, -+ devType: devType, -+ } -+ s.devices = append(s.devices, dev) -+ -+ return nil -+} -+ -+func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { -+ Name = filepath.Base(strings.ToLower(Name)) -+ -+ if strings.HasPrefix(Name, "eth") { -+ idxStr := strings.TrimPrefix(Name, "eth") -+ if idxStr == Name { -+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ } -+ -+ idx, err := strconv.Atoi(idxStr) -+ if err != nil { -+ return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) -+ } -+ -+ if !isPut && s.mmioNetSlots[idx] { -+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) -+ } -+ s.mmioNetSlots[idx] = !isPut -+ -+ return idx, nil -+ } else if strings.HasPrefix(Name, "vd") { -+ charStr := strings.TrimPrefix(Name, "vd") -+ if charStr == Name { -+ return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ } -+ -+ char := []rune(charStr) -+ idx := int(char[0] - 'a') -+ -+ if !isPut && s.mmioBlkSlots[idx] { -+ return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) -+ } -+ s.mmioBlkSlots[idx] = !isPut -+ -+ return idx, nil -+ } -+ -+ return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) -+} -+ -+func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { -+ err = s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ var tap TapInterface -+ -+ switch endpoint.Type() { -+ case VethEndpointType: -+ drive := endpoint.(*VethEndpoint) -+ tap = drive.NetPair.TapInterface -+ case TapEndpointType: -+ drive := endpoint.(*TapEndpoint) -+ tap = drive.TapInterface -+ default: -+ return fmt.Errorf("Endpoint is not supported") -+ } -+ -+ switch op { -+ case addDevice: -+ var ( -+ VMFdNames []string -+ VhostFdNames []string -+ ) -+ for i, VMFd := range tap.VMFds { -+ fdName := fmt.Sprintf("fd%d", i) -+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VMFd); err != nil { -+ return err -+ } -+ VMFdNames = append(VMFdNames, fdName) -+ } -+ for i, VhostFd := range tap.VhostFds { -+ fdName := fmt.Sprintf("vhostfd%d", i) -+ if err := s.qmpMonitorCh.qmp.ExecuteGetFD(s.qmpMonitorCh.ctx, fdName, VhostFd); err != nil { -+ return err -+ } -+ VhostFd.Close() -+ VhostFdNames = append(VhostFdNames, fdName) -+ } -+ -+ slot, err := s.getDevSlot(endpoint.Name(), false) -+ if err != nil { -+ return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) -+ } -+ -+ if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ } else { -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { -+ s.getDevSlot(endpoint.Name(), true) -+ return err -+ } -+ case removeDevice: -+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { -+ return err -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteNetdevDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { -+ return err -+ } -+ default: -+ return fmt.Errorf("Operation is not supported") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { -+ err = s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ switch op { -+ case addDevice: -+ driver := "virtio-blk-pci" -+ slot, err := s.getDevSlot(drive.VirtPath, false) -+ if err != nil { -+ return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil { -+ s.getDevSlot(drive.VirtPath, true) -+ return err -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { -+ s.getDevSlot(drive.VirtPath, true) -+ return err -+ } -+ case removeDevice: -+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { -+ return err -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { -+ return err -+ } -+ -+ s.getDevSlot(drive.VirtPath, true) -+ default: -+ return fmt.Errorf("Operation is not supported") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) { -+ span, _ := s.trace(ctx, "hotplugAddDevice") -+ defer span.End() -+ -+ switch devType { -+ case netDev: -+ return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) -+ case blockDev: -+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) -+ default: -+ return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) -+ } -+} -+ -+func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{}, devType deviceType) (interface{}, error) { -+ span, _ := s.trace(ctx, "hotplugRemoveDevice") -+ defer span.End() -+ -+ switch devType { -+ case netDev: -+ return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) -+ case blockDev: -+ return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) -+ default: -+ return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) -+ } -+} -+ -+func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { -+ return 0, memoryDevice{}, nil -+} -+ -+func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { -+ return 0, 0, nil -+} -+ -+func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) { -+ span, _ := s.trace(ctx, "getSandboxConsole") -+ defer span.End() -+ -+ var consolePath string -+ if s.config.Debug { -+ consolePath = s.consolePath -+ } else { -+ consolePath = "" -+ } -+ consoleURL, err := utils.BuildSocketPath(consolePath) -+ if err != nil { -+ return consoleProtoUnix, "", err -+ } -+ return consoleProtoUnix, consoleURL, nil -+ -+} -+ -+func (s *stratovirt) disconnect(ctx context.Context) { -+ span, _ := s.trace(ctx, "disconnect") -+ defer span.End() -+ -+ s.qmpTeardown() -+} -+ -+func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities { -+ span, _ := s.trace(ctx, "capabilities") -+ defer span.End() -+ -+ var caps types.Capabilities -+ caps.SetBlockDeviceHotplugSupport() -+ -+ return caps -+} -+ -+func (s *stratovirt) qmpTeardown() { -+ if s.qmpMonitorCh.qmp != nil { -+ s.qmpMonitorCh.qmp.Shutdown() -+ <-s.qmpMonitorCh.disconn -+ s.qmpMonitorCh.qmp = nil -+ s.qmpMonitorCh.disconn = nil -+ } -+} -+ -+func (s *stratovirt) qmpSetup() error { -+ s.qmpTeardown() -+ -+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} -+ -+ // Auto-closed by QMPStart(). -+ disconnectCh := make(chan struct{}) -+ -+ qmp, _, err := govmmQemu.QMPStart(s.qmpMonitorCh.ctx, s.qmpMonitorCh.path, cfg, disconnectCh) -+ if err != nil { -+ s.Logger().WithError(err).Error("Failed to connect to StratoVirt instance") -+ return err -+ } -+ -+ s.qmpMonitorCh.qmp = qmp -+ s.qmpMonitorCh.disconn = disconnectCh -+ -+ return nil -+} -+ -+func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) { -+ span, _ := s.trace(ctx, "getThreadIDs") -+ defer span.End() -+ -+ tid := vcpuThreadIDs{} -+ if err := s.qmpSetup(); err != nil { -+ return tid, err -+ } -+ -+ cpuInfos, err := s.qmpMonitorCh.qmp.ExecQueryCpus(s.qmpMonitorCh.ctx) -+ if err != nil { -+ s.Logger().WithError(err).Error("Failed to query cpu infos") -+ return tid, err -+ } -+ -+ tid.vcpus = make(map[int]int, len(cpuInfos)) -+ for _, i := range cpuInfos { -+ if i.ThreadID > 0 { -+ tid.vcpus[i.CPU] = i.ThreadID -+ } -+ } -+ return tid, nil -+} -+ -+func (s *stratovirt) cleanup(ctx context.Context) error { -+ span, _ := s.trace(ctx, "cleanup") -+ defer span.End() -+ -+ s.qmpTeardown() -+ -+ return nil -+} -+ -+func (s *stratovirt) getPids() []int { -+ return []int{s.pid} -+} -+ -+func (s *stratovirt) getVirtioFsPid() *int { -+ return nil -+} -+ -+func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { -+ return errors.New("stratovirt is not supported by VM cache") -+} -+ -+func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) { -+ return nil, errors.New("stratovirt is not supported by VM cache") -+} -+ -+func (s *stratovirt) check() error { -+ if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil { -+ return errors.Wrapf(err, "Failed to ping StratoVirt process") -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) generateSocket(id string) (interface{}, error) { -+ return generateVMSocket(id, s.store.RunVMStoragePath()) -+} -+ -+func (s *stratovirt) isRateLimiterBuiltin() bool { -+ return true -+} -+ -+func (s *stratovirt) save() (p persistapi.HypervisorState) { -+ pids := s.getPids() -+ p.Pid = pids[0] -+ p.Type = string(StratovirtHypervisor) -+ return -+} -+ -+func (s *stratovirt) load(p persistapi.HypervisorState) { -+ s.pid = p.Pid -+ -+ return -+} -+ -+func (s *stratovirt) setSandbox(sandbox *Sandbox) { -+ s.sandbox = sandbox -+ return -+} --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch b/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch deleted file mode 100644 index 3eb330027ae83911c489612079f7b05c0aafb973..0000000000000000000000000000000000000000 --- a/patches/0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch +++ /dev/null @@ -1,174 +0,0 @@ -From 1ffd95187a61582e858dd37c0ab434d3159a0f52 Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Mon, 9 Aug 2021 14:26:35 +0800 -Subject: [PATCH 2/6] agent: add support of new sandbox hypervisor kind - StratoVirt. - -1. add new grpc interface `UpdateInterfaceHwAddrByName`. -2. comment out rescan_pci temporarily. - -Signed-off-by: Wei Gao ---- - src/agent/protocols/protos/agent.proto | 5 +++ - src/agent/src/netlink.rs | 31 ++++++++++++++++ - src/agent/src/rpc.rs | 51 +++++++++++++++++++++++++- - 3 files changed, 85 insertions(+), 2 deletions(-) - -diff --git a/src/agent/protocols/protos/agent.proto b/src/agent/protocols/protos/agent.proto -index 6cbf5a28..e00f5c63 100644 ---- a/src/agent/protocols/protos/agent.proto -+++ b/src/agent/protocols/protos/agent.proto -@@ -46,6 +46,7 @@ service AgentService { - - // networking - rpc UpdateInterface(UpdateInterfaceRequest) returns (types.Interface); -+ rpc UpdateInterfaceHwAddrByName(UpdateInterfaceHwAddrByNameRequest) returns (types.Interface); - rpc UpdateRoutes(UpdateRoutesRequest) returns (Routes); - rpc ListInterfaces(ListInterfacesRequest) returns(Interfaces); - rpc ListRoutes(ListRoutesRequest) returns (Routes); -@@ -308,6 +309,10 @@ message UpdateInterfaceRequest { - types.Interface interface = 1; - } - -+message UpdateInterfaceHwAddrByNameRequest { -+ types.Interface interface = 1; -+} -+ - message UpdateRoutesRequest { - Routes routes = 1; - } -diff --git a/src/agent/src/netlink.rs b/src/agent/src/netlink.rs -index 3ab6dbaa..82632d1b 100644 ---- a/src/agent/src/netlink.rs -+++ b/src/agent/src/netlink.rs -@@ -104,6 +104,29 @@ impl Handle { - Ok(()) - } - -+ pub async fn update_interface_hw_addr_by_name(&mut self, iface: &Interface) -> Result<()> { -+ let link = self.find_link(LinkFilter::Name(&iface.name)).await?; -+ -+ // Delete all addresses associated with the link -+ let addresses = self -+ .list_addresses(AddressFilter::LinkIndex(link.index())) -+ .await?; -+ self.delete_addresses(addresses).await?; -+ -+ if iface.IPAddresses.len() == 0 { -+ self.enable_link(link.index(), false).await?; -+ } -+ -+ // Update hardware mac address -+ let mac_addr = parse_mac_address(iface.get_hwAddr()) -+ .with_context(|| format!("Failed to parse MAC address: {}", iface.get_hwAddr()))?; -+ self.link_set_hw_addr(link.index(), mac_addr) -+ .await -+ .with_context(|| format!("Could not set {:?} to {}", mac_addr, link.name()))?; -+ -+ Ok(()) -+ } -+ - pub async fn handle_localhost(&self) -> Result<()> { - let link = self.find_link(LinkFilter::Name("lo")).await?; - self.enable_link(link.index(), true).await?; -@@ -216,6 +239,14 @@ impl Handle { - Ok(()) - } - -+ async fn link_set_hw_addr(&self, link_index: u32, hw_addr: [u8; 6]) -> Result<()> { -+ let link_req = self.handle.link().set(link_index); -+ let set_req = link_req.address(hw_addr.to_vec()); -+ set_req.execute().await?; -+ -+ Ok(()) -+ } -+ - async fn query_routes( - &self, - ip_version: Option, -diff --git a/src/agent/src/rpc.rs b/src/agent/src/rpc.rs -index 92025af3..2cc1c983 100644 ---- a/src/agent/src/rpc.rs -+++ b/src/agent/src/rpc.rs -@@ -40,7 +40,7 @@ use nix::sys::stat; - use nix::unistd::{self, Pid}; - use rustjail::process::ProcessOperations; - --use crate::device::{add_devices, rescan_pci_bus, update_device_cgroup}; -+use crate::device::{add_devices, update_device_cgroup}; - use crate::linux_abi::*; - use crate::metrics::get_metrics; - use crate::mount::{add_storages, remove_mounts, BareMount, STORAGE_HANDLER_LIST}; -@@ -123,7 +123,9 @@ impl AgentService { - - // re-scan PCI bus - // looking for hidden devices -- rescan_pci_bus().context("Could not rescan PCI bus")?; -+ // FIXME: Comment out this code temporarily, because once the PCIBus is scanned, -+ // the device hot-plug event is lost -+ // rescan_pci_bus().context("Could not rescan PCI bus")?; - - // Some devices need some extra processing (the ones invoked with - // --device for instance), and that's what this call is doing. It -@@ -797,6 +799,34 @@ impl protocols::agent_ttrpc::AgentService for AgentService { - Ok(interface) - } - -+ async fn update_interface_hw_addr_by_name( -+ &self, -+ _ctx: &TtrpcContext, -+ req: protocols::agent::UpdateInterfaceHwAddrByNameRequest, -+ ) -> ttrpc::Result { -+ let interface = req.interface.into_option().ok_or_else(|| { -+ ttrpc_error( -+ ttrpc::Code::INVALID_ARGUMENT, -+ "empty update interface request".to_string(), -+ ) -+ })?; -+ -+ self.sandbox -+ .lock() -+ .await -+ .rtnl -+ .update_interface_hw_addr_by_name(&interface) -+ .await -+ .map_err(|e| { -+ ttrpc_error( -+ ttrpc::Code::INTERNAL, -+ format!("update interface hw addr: {:?}", e), -+ ) -+ })?; -+ -+ Ok(interface) -+ } -+ - async fn update_routes( - &self, - _ctx: &TtrpcContext, -@@ -1670,6 +1700,23 @@ mod tests { - assert!(result.is_err(), "expected update interface to fail"); - } - -+ #[tokio::test] -+ async fn test_update_interface_hw_addr_by_name() { -+ let logger = slog::Logger::root(slog::Discard, o!()); -+ let sandbox = Sandbox::new(&logger).unwrap(); -+ -+ let agent_service = Box::new(AgentService { -+ sandbox: Arc::new(Mutex::new(sandbox)), -+ }); -+ -+ let req = protocols::agent::UpdateInterfaceHwAddrByNameRequest::default(); -+ let ctx = mk_ttrpc_context(); -+ -+ let result = agent_service.update_interface_hw_addr_by_name(&ctx, req).await; -+ -+ assert!(result.is_err(), "expected update interface to fail"); -+ } -+ - #[tokio::test] - async fn test_update_routes() { - let logger = slog::Logger::root(slog::Discard, o!()); --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch b/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch deleted file mode 100644 index 2419f0bb319fd20c6aafd59e3e8bcf914a645aa7..0000000000000000000000000000000000000000 --- a/patches/0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch +++ /dev/null @@ -1,2346 +0,0 @@ -From cfc3ca2b200cd406cb47e0709b31e307bed9c6c9 Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Mon, 9 Aug 2021 14:36:15 +0800 -Subject: [PATCH 3/6] runtime: implement "updateInterfaceHwAddrByName" - interface for agent. - -1. add "updateInterfaceHwAddrByName" grpc interface. -2. update protocol pb.go files. - -Signed-off-by: Wei Gao ---- - src/runtime/virtcontainers/agent.go | 3 + - src/runtime/virtcontainers/kata_agent.go | 88 +- - src/runtime/virtcontainers/mock_agent.go | 5 + - .../pkg/agent/protocols/grpc/agent.pb.go | 936 +++++++++--------- - .../pkg/agent/protocols/grpc/health.pb.go | 15 +- - .../pkg/agent/protocols/grpc/oci.pb.go | 169 +--- - .../pkg/agent/protocols/types.pb.go | 20 +- - src/runtime/virtcontainers/pkg/mock/mock.go | 4 + - src/runtime/virtcontainers/sandbox.go | 5 + - src/runtime/virtcontainers/stratovirt.go | 6 +- - 10 files changed, 607 insertions(+), 644 deletions(-) - -diff --git a/src/runtime/virtcontainers/agent.go b/src/runtime/virtcontainers/agent.go -index b8576c35..19a6c398 100644 ---- a/src/runtime/virtcontainers/agent.go -+++ b/src/runtime/virtcontainers/agent.go -@@ -171,6 +171,9 @@ type agent interface { - // updateInterface will tell the agent to update a nic for an existed Sandbox. - updateInterface(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) - -+ // updateInterfaceHwAddrByName will tell the agent to update a nic for an existed Sandbox by nic name. -+ updateInterfaceHwAddrByName(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) -+ - // listInterfaces will tell the agent to list interfaces of an existed Sandbox - listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) - -diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go -index 12756f0b..13d31658 100644 ---- a/src/runtime/virtcontainers/kata_agent.go -+++ b/src/runtime/virtcontainers/kata_agent.go -@@ -106,37 +106,38 @@ const ( - ) - - const ( -- grpcCheckRequest = "grpc.CheckRequest" -- grpcExecProcessRequest = "grpc.ExecProcessRequest" -- grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" -- grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" -- grpcCreateContainerRequest = "grpc.CreateContainerRequest" -- grpcStartContainerRequest = "grpc.StartContainerRequest" -- grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" -- grpcSignalProcessRequest = "grpc.SignalProcessRequest" -- grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" -- grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" -- grpcListInterfacesRequest = "grpc.ListInterfacesRequest" -- grpcListRoutesRequest = "grpc.ListRoutesRequest" -- grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" -- grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" -- grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" -- grpcWaitProcessRequest = "grpc.WaitProcessRequest" -- grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" -- grpcWriteStreamRequest = "grpc.WriteStreamRequest" -- grpcCloseStdinRequest = "grpc.CloseStdinRequest" -- grpcStatsContainerRequest = "grpc.StatsContainerRequest" -- grpcPauseContainerRequest = "grpc.PauseContainerRequest" -- grpcResumeContainerRequest = "grpc.ResumeContainerRequest" -- grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" -- grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" -- grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" -- grpcCopyFileRequest = "grpc.CopyFileRequest" -- grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" -- grpcStartTracingRequest = "grpc.StartTracingRequest" -- grpcStopTracingRequest = "grpc.StopTracingRequest" -- grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" -- grpcGetMetricsRequest = "grpc.GetMetricsRequest" -+ grpcCheckRequest = "grpc.CheckRequest" -+ grpcExecProcessRequest = "grpc.ExecProcessRequest" -+ grpcCreateSandboxRequest = "grpc.CreateSandboxRequest" -+ grpcDestroySandboxRequest = "grpc.DestroySandboxRequest" -+ grpcCreateContainerRequest = "grpc.CreateContainerRequest" -+ grpcStartContainerRequest = "grpc.StartContainerRequest" -+ grpcRemoveContainerRequest = "grpc.RemoveContainerRequest" -+ grpcSignalProcessRequest = "grpc.SignalProcessRequest" -+ grpcUpdateRoutesRequest = "grpc.UpdateRoutesRequest" -+ grpcUpdateInterfaceRequest = "grpc.UpdateInterfaceRequest" -+ grpcUpdateInterfaceHwAddrByNameRequest = "grpc.UpdateInterfaceHwAddrByNameRequest" -+ grpcListInterfacesRequest = "grpc.ListInterfacesRequest" -+ grpcListRoutesRequest = "grpc.ListRoutesRequest" -+ grpcAddARPNeighborsRequest = "grpc.AddARPNeighborsRequest" -+ grpcOnlineCPUMemRequest = "grpc.OnlineCPUMemRequest" -+ grpcUpdateContainerRequest = "grpc.UpdateContainerRequest" -+ grpcWaitProcessRequest = "grpc.WaitProcessRequest" -+ grpcTtyWinResizeRequest = "grpc.TtyWinResizeRequest" -+ grpcWriteStreamRequest = "grpc.WriteStreamRequest" -+ grpcCloseStdinRequest = "grpc.CloseStdinRequest" -+ grpcStatsContainerRequest = "grpc.StatsContainerRequest" -+ grpcPauseContainerRequest = "grpc.PauseContainerRequest" -+ grpcResumeContainerRequest = "grpc.ResumeContainerRequest" -+ grpcReseedRandomDevRequest = "grpc.ReseedRandomDevRequest" -+ grpcGuestDetailsRequest = "grpc.GuestDetailsRequest" -+ grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest" -+ grpcCopyFileRequest = "grpc.CopyFileRequest" -+ grpcSetGuestDateTimeRequest = "grpc.SetGuestDateTimeRequest" -+ grpcStartTracingRequest = "grpc.StartTracingRequest" -+ grpcStopTracingRequest = "grpc.StopTracingRequest" -+ grpcGetOOMEventRequest = "grpc.GetOOMEventRequest" -+ grpcGetMetricsRequest = "grpc.GetMetricsRequest" - ) - - // newKataAgent returns an agent from an agent type. -@@ -668,6 +669,25 @@ func (k *kataAgent) updateInterfaces(ctx context.Context, interfaces []*pbTypes. - return nil - } - -+func (k *kataAgent) updateInterfaceHwAddrByName(ctx context.Context, ifc *pbTypes.Interface) (*pbTypes.Interface, error) { -+ ifcReq := &grpc.UpdateInterfaceHwAddrByNameRequest{ -+ Interface: ifc, -+ } -+ _, err := k.sendReq(ctx, ifcReq) -+ -+ return nil, err -+} -+ -+func (k *kataAgent) updateInterfacesHwAddrByName(ctx context.Context, interfaces []*pbTypes.Interface) error { -+ for _, ifc := range interfaces { -+ if _, err := k.updateInterfaceHwAddrByName(ctx, ifc); err != nil { -+ return err -+ } -+ } -+ -+ return nil -+} -+ - func (k *kataAgent) updateRoutes(ctx context.Context, routes []*pbTypes.Route) ([]*pbTypes.Route, error) { - if routes != nil { - routesReq := &grpc.UpdateRoutesRequest{ -@@ -819,6 +839,9 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error { - if err != nil { - return err - } -+ if err = k.updateInterfacesHwAddrByName(ctx, interfaces); err != nil { -+ return err -+ } - if err = k.updateInterfaces(ctx, interfaces); err != nil { - return err - } -@@ -1952,6 +1975,9 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) { - k.reqHandlers[grpcUpdateInterfaceRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { - return k.client.AgentServiceClient.UpdateInterface(ctx, req.(*grpc.UpdateInterfaceRequest)) - } -+ k.reqHandlers[grpcUpdateInterfaceHwAddrByNameRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { -+ return k.client.AgentServiceClient.UpdateInterfaceHwAddrByName(ctx, req.(*grpc.UpdateInterfaceHwAddrByNameRequest)) -+ } - k.reqHandlers[grpcListInterfacesRequest] = func(ctx context.Context, req interface{}) (interface{}, error) { - return k.client.AgentServiceClient.ListInterfaces(ctx, req.(*grpc.ListInterfacesRequest)) - } -diff --git a/src/runtime/virtcontainers/mock_agent.go b/src/runtime/virtcontainers/mock_agent.go -index f12093b3..dae8cc5c 100644 ---- a/src/runtime/virtcontainers/mock_agent.go -+++ b/src/runtime/virtcontainers/mock_agent.go -@@ -106,6 +106,11 @@ func (n *mockAgent) updateInterface(ctx context.Context, inf *pbTypes.Interface) - return nil, nil - } - -+// updateInterfaceHwAddrByName is the Noop agent Interface HwAddr update implementation. It does nothing. -+func (n *mockAgent) updateInterfaceHwAddrByName(ctx context.Context, inf *pbTypes.Interface) (*pbTypes.Interface, error) { -+ return nil, nil -+} -+ - // listInterfaces is the Noop agent Interfaces list implementation. It does nothing. - func (n *mockAgent) listInterfaces(ctx context.Context) ([]*pbTypes.Interface, error) { - return nil, nil -diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go -index 3ee3bff7..a5c1e655 100644 ---- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go -+++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/agent.pb.go -@@ -1488,6 +1488,45 @@ func (m *UpdateInterfaceRequest) XXX_DiscardUnknown() { - - var xxx_messageInfo_UpdateInterfaceRequest proto.InternalMessageInfo - -+type UpdateInterfaceHwAddrByNameRequest struct { -+ Interface *protocols.Interface `protobuf:"bytes,1,opt,name=interface,proto3" json:"interface,omitempty"` -+ XXX_NoUnkeyedLiteral struct{} `json:"-"` -+ XXX_unrecognized []byte `json:"-"` -+ XXX_sizecache int32 `json:"-"` -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) Reset() { *m = UpdateInterfaceHwAddrByNameRequest{} } -+func (*UpdateInterfaceHwAddrByNameRequest) ProtoMessage() {} -+func (*UpdateInterfaceHwAddrByNameRequest) Descriptor() ([]byte, []int) { -+ return fileDescriptor_c1460208c38ccf5e, []int{35} -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Unmarshal(b []byte) error { -+ return m.Unmarshal(b) -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { -+ if deterministic { -+ return xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.Marshal(b, m, deterministic) -+ } else { -+ b = b[:cap(b)] -+ n, err := m.MarshalToSizedBuffer(b) -+ if err != nil { -+ return nil, err -+ } -+ return b[:n], nil -+ } -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Merge(src proto.Message) { -+ xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.Merge(m, src) -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) XXX_Size() int { -+ return m.Size() -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) XXX_DiscardUnknown() { -+ xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest.DiscardUnknown(m) -+} -+ -+var xxx_messageInfo_UpdateInterfaceHwAddrByNameRequest proto.InternalMessageInfo -+ - type UpdateRoutesRequest struct { - Routes *Routes `protobuf:"bytes,1,opt,name=routes,proto3" json:"routes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` -@@ -1498,7 +1537,7 @@ type UpdateRoutesRequest struct { - func (m *UpdateRoutesRequest) Reset() { *m = UpdateRoutesRequest{} } - func (*UpdateRoutesRequest) ProtoMessage() {} - func (*UpdateRoutesRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{35} -+ return fileDescriptor_c1460208c38ccf5e, []int{36} - } - func (m *UpdateRoutesRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1536,7 +1575,7 @@ type ListInterfacesRequest struct { - func (m *ListInterfacesRequest) Reset() { *m = ListInterfacesRequest{} } - func (*ListInterfacesRequest) ProtoMessage() {} - func (*ListInterfacesRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{36} -+ return fileDescriptor_c1460208c38ccf5e, []int{37} - } - func (m *ListInterfacesRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1574,7 +1613,7 @@ type ListRoutesRequest struct { - func (m *ListRoutesRequest) Reset() { *m = ListRoutesRequest{} } - func (*ListRoutesRequest) ProtoMessage() {} - func (*ListRoutesRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{37} -+ return fileDescriptor_c1460208c38ccf5e, []int{38} - } - func (m *ListRoutesRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1613,7 +1652,7 @@ type ARPNeighbors struct { - func (m *ARPNeighbors) Reset() { *m = ARPNeighbors{} } - func (*ARPNeighbors) ProtoMessage() {} - func (*ARPNeighbors) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{38} -+ return fileDescriptor_c1460208c38ccf5e, []int{39} - } - func (m *ARPNeighbors) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1652,7 +1691,7 @@ type AddARPNeighborsRequest struct { - func (m *AddARPNeighborsRequest) Reset() { *m = AddARPNeighborsRequest{} } - func (*AddARPNeighborsRequest) ProtoMessage() {} - func (*AddARPNeighborsRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{39} -+ return fileDescriptor_c1460208c38ccf5e, []int{40} - } - func (m *AddARPNeighborsRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1698,7 +1737,7 @@ type OnlineCPUMemRequest struct { - func (m *OnlineCPUMemRequest) Reset() { *m = OnlineCPUMemRequest{} } - func (*OnlineCPUMemRequest) ProtoMessage() {} - func (*OnlineCPUMemRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{40} -+ return fileDescriptor_c1460208c38ccf5e, []int{41} - } - func (m *OnlineCPUMemRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1738,7 +1777,7 @@ type ReseedRandomDevRequest struct { - func (m *ReseedRandomDevRequest) Reset() { *m = ReseedRandomDevRequest{} } - func (*ReseedRandomDevRequest) ProtoMessage() {} - func (*ReseedRandomDevRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{41} -+ return fileDescriptor_c1460208c38ccf5e, []int{42} - } - func (m *ReseedRandomDevRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1788,7 +1827,7 @@ type AgentDetails struct { - func (m *AgentDetails) Reset() { *m = AgentDetails{} } - func (*AgentDetails) ProtoMessage() {} - func (*AgentDetails) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{42} -+ return fileDescriptor_c1460208c38ccf5e, []int{43} - } - func (m *AgentDetails) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1834,7 +1873,7 @@ type GuestDetailsRequest struct { - func (m *GuestDetailsRequest) Reset() { *m = GuestDetailsRequest{} } - func (*GuestDetailsRequest) ProtoMessage() {} - func (*GuestDetailsRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{43} -+ return fileDescriptor_c1460208c38ccf5e, []int{44} - } - func (m *GuestDetailsRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1876,7 +1915,7 @@ type GuestDetailsResponse struct { - func (m *GuestDetailsResponse) Reset() { *m = GuestDetailsResponse{} } - func (*GuestDetailsResponse) ProtoMessage() {} - func (*GuestDetailsResponse) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{44} -+ return fileDescriptor_c1460208c38ccf5e, []int{45} - } - func (m *GuestDetailsResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1917,7 +1956,7 @@ type MemHotplugByProbeRequest struct { - func (m *MemHotplugByProbeRequest) Reset() { *m = MemHotplugByProbeRequest{} } - func (*MemHotplugByProbeRequest) ProtoMessage() {} - func (*MemHotplugByProbeRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{45} -+ return fileDescriptor_c1460208c38ccf5e, []int{46} - } - func (m *MemHotplugByProbeRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -1959,7 +1998,7 @@ type SetGuestDateTimeRequest struct { - func (m *SetGuestDateTimeRequest) Reset() { *m = SetGuestDateTimeRequest{} } - func (*SetGuestDateTimeRequest) ProtoMessage() {} - func (*SetGuestDateTimeRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{46} -+ return fileDescriptor_c1460208c38ccf5e, []int{47} - } - func (m *SetGuestDateTimeRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2024,7 +2063,7 @@ type Storage struct { - func (m *Storage) Reset() { *m = Storage{} } - func (*Storage) ProtoMessage() {} - func (*Storage) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{47} -+ return fileDescriptor_c1460208c38ccf5e, []int{48} - } - func (m *Storage) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2095,7 +2134,7 @@ type Device struct { - func (m *Device) Reset() { *m = Device{} } - func (*Device) ProtoMessage() {} - func (*Device) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{48} -+ return fileDescriptor_c1460208c38ccf5e, []int{49} - } - func (m *Device) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2136,7 +2175,7 @@ type StringUser struct { - func (m *StringUser) Reset() { *m = StringUser{} } - func (*StringUser) ProtoMessage() {} - func (*StringUser) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{49} -+ return fileDescriptor_c1460208c38ccf5e, []int{50} - } - func (m *StringUser) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2193,7 +2232,7 @@ type CopyFileRequest struct { - func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } - func (*CopyFileRequest) ProtoMessage() {} - func (*CopyFileRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{50} -+ return fileDescriptor_c1460208c38ccf5e, []int{51} - } - func (m *CopyFileRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2231,7 +2270,7 @@ type StartTracingRequest struct { - func (m *StartTracingRequest) Reset() { *m = StartTracingRequest{} } - func (*StartTracingRequest) ProtoMessage() {} - func (*StartTracingRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{51} -+ return fileDescriptor_c1460208c38ccf5e, []int{52} - } - func (m *StartTracingRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2269,7 +2308,7 @@ type StopTracingRequest struct { - func (m *StopTracingRequest) Reset() { *m = StopTracingRequest{} } - func (*StopTracingRequest) ProtoMessage() {} - func (*StopTracingRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{52} -+ return fileDescriptor_c1460208c38ccf5e, []int{53} - } - func (m *StopTracingRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2307,7 +2346,7 @@ type GetOOMEventRequest struct { - func (m *GetOOMEventRequest) Reset() { *m = GetOOMEventRequest{} } - func (*GetOOMEventRequest) ProtoMessage() {} - func (*GetOOMEventRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{53} -+ return fileDescriptor_c1460208c38ccf5e, []int{54} - } - func (m *GetOOMEventRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2346,7 +2385,7 @@ type OOMEvent struct { - func (m *OOMEvent) Reset() { *m = OOMEvent{} } - func (*OOMEvent) ProtoMessage() {} - func (*OOMEvent) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{54} -+ return fileDescriptor_c1460208c38ccf5e, []int{55} - } - func (m *OOMEvent) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2384,7 +2423,7 @@ type GetMetricsRequest struct { - func (m *GetMetricsRequest) Reset() { *m = GetMetricsRequest{} } - func (*GetMetricsRequest) ProtoMessage() {} - func (*GetMetricsRequest) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{55} -+ return fileDescriptor_c1460208c38ccf5e, []int{56} - } - func (m *GetMetricsRequest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2423,7 +2462,7 @@ type Metrics struct { - func (m *Metrics) Reset() { *m = Metrics{} } - func (*Metrics) ProtoMessage() {} - func (*Metrics) Descriptor() ([]byte, []int) { -- return fileDescriptor_c1460208c38ccf5e, []int{56} -+ return fileDescriptor_c1460208c38ccf5e, []int{57} - } - func (m *Metrics) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -@@ -2490,6 +2529,7 @@ func init() { - proto.RegisterType((*Interfaces)(nil), "grpc.Interfaces") - proto.RegisterType((*Routes)(nil), "grpc.Routes") - proto.RegisterType((*UpdateInterfaceRequest)(nil), "grpc.UpdateInterfaceRequest") -+ proto.RegisterType((*UpdateInterfaceHwAddrByNameRequest)(nil), "grpc.UpdateInterfaceHwAddrByNameRequest") - proto.RegisterType((*UpdateRoutesRequest)(nil), "grpc.UpdateRoutesRequest") - proto.RegisterType((*ListInterfacesRequest)(nil), "grpc.ListInterfacesRequest") - proto.RegisterType((*ListRoutesRequest)(nil), "grpc.ListRoutesRequest") -@@ -2519,193 +2559,195 @@ func init() { - } - - var fileDescriptor_c1460208c38ccf5e = []byte{ -- // 2971 bytes of a gzipped FileDescriptorProto -- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1a, 0xcb, 0x72, 0x24, 0x47, -- 0xd1, 0xf3, 0x90, 0x66, 0x26, 0xe7, 0xa5, 0x69, 0x69, 0xb5, 0xb3, 0x63, 0x5b, 0xac, 0x7b, 0xed, -- 0xb5, 0x8c, 0xf1, 0xc8, 0x5e, 0x3b, 0x58, 0x3f, 0xc2, 0x6c, 0xac, 0x1e, 0x96, 0x64, 0x5b, 0x96, -- 0xdc, 0x5a, 0x85, 0x09, 0x08, 0xe8, 0x68, 0x75, 0x97, 0x66, 0xca, 0x9a, 0xee, 0x6a, 0x57, 0x57, -- 0x6b, 0x35, 0x26, 0x82, 0x80, 0x0b, 0xdc, 0x38, 0x72, 0xe3, 0x07, 0x08, 0x6e, 0x1c, 0xb9, 0x72, -- 0x70, 0x70, 0xe2, 0xc8, 0x89, 0xc0, 0xfb, 0x09, 0x7c, 0x01, 0x51, 0xaf, 0x7e, 0xcc, 0x43, 0x06, -- 0x85, 0x22, 0xb8, 0x4c, 0x54, 0x66, 0x65, 0xe5, 0xab, 0xaa, 0xb2, 0x32, 0xb3, 0x07, 0x3e, 0x1f, -- 0x60, 0x36, 0x8c, 0x4f, 0xfb, 0x2e, 0xf1, 0x37, 0xce, 0x1d, 0xe6, 0xbc, 0xe1, 0x92, 0x80, 0x39, -- 0x38, 0x40, 0x34, 0x9a, 0x82, 0x23, 0xea, 0x6e, 0x38, 0x03, 0x14, 0xb0, 0x8d, 0x90, 0x12, 0x46, -- 0x5c, 0x32, 0x8a, 0xe4, 0x28, 0x92, 0xe8, 0xbe, 0x00, 0x8c, 0xf2, 0x80, 0x86, 0x6e, 0xaf, 0x46, -- 0x5c, 0x2c, 0x11, 0xbd, 0x3a, 0x1b, 0x87, 0x28, 0x52, 0xc0, 0xf3, 0x03, 0x42, 0x06, 0x23, 0x24, -- 0x17, 0x9e, 0xc6, 0x67, 0x1b, 0xc8, 0x0f, 0xd9, 0x58, 0x4e, 0x9a, 0x7f, 0x28, 0xc2, 0xea, 0x16, -- 0x45, 0x0e, 0x43, 0x5b, 0x5a, 0xac, 0x85, 0xbe, 0x8a, 0x51, 0xc4, 0x8c, 0x97, 0xa0, 0x91, 0xa8, -- 0x62, 0x63, 0xaf, 0x5b, 0xb8, 0x5b, 0x58, 0xaf, 0x59, 0xf5, 0x04, 0xb7, 0xef, 0x19, 0xb7, 0xa1, -- 0x82, 0x2e, 0x91, 0xcb, 0x67, 0x8b, 0x62, 0x76, 0x91, 0x83, 0xfb, 0x9e, 0xf1, 0x16, 0xd4, 0x23, -- 0x46, 0x71, 0x30, 0xb0, 0xe3, 0x08, 0xd1, 0x6e, 0xe9, 0x6e, 0x61, 0xbd, 0xfe, 0x60, 0xa9, 0xcf, -- 0xf5, 0xec, 0x1f, 0x8b, 0x89, 0x93, 0x08, 0x51, 0x0b, 0xa2, 0x64, 0x6c, 0xdc, 0x87, 0x8a, 0x87, -- 0x2e, 0xb0, 0x8b, 0xa2, 0x6e, 0xf9, 0x6e, 0x69, 0xbd, 0xfe, 0xa0, 0x21, 0xc9, 0xb7, 0x05, 0xd2, -- 0xd2, 0x93, 0xc6, 0x6b, 0x50, 0x8d, 0x18, 0xa1, 0xce, 0x00, 0x45, 0xdd, 0x05, 0x41, 0xd8, 0xd4, -- 0x7c, 0x05, 0xd6, 0x4a, 0xa6, 0x8d, 0x17, 0xa0, 0x74, 0xb8, 0xb5, 0xdf, 0x5d, 0x14, 0xd2, 0x41, -- 0x51, 0x85, 0xc8, 0xb5, 0x38, 0xda, 0xb8, 0x07, 0xcd, 0xc8, 0x09, 0xbc, 0x53, 0x72, 0x69, 0x87, -- 0xd8, 0x0b, 0xa2, 0x6e, 0xe5, 0x6e, 0x61, 0xbd, 0x6a, 0x35, 0x14, 0xf2, 0x88, 0xe3, 0xcc, 0xf7, -- 0xe1, 0xd6, 0x31, 0x73, 0x28, 0xbb, 0x86, 0x77, 0xcc, 0x13, 0x58, 0xb5, 0x90, 0x4f, 0x2e, 0xae, -- 0xe5, 0xda, 0x2e, 0x54, 0x18, 0xf6, 0x11, 0x89, 0x99, 0x70, 0x6d, 0xd3, 0xd2, 0xa0, 0xf9, 0xa7, -- 0x02, 0x18, 0x3b, 0x97, 0xc8, 0x3d, 0xa2, 0xc4, 0x45, 0x51, 0xf4, 0x7f, 0xda, 0xae, 0x57, 0xa1, -- 0x12, 0x4a, 0x05, 0xba, 0x65, 0x41, 0xae, 0x76, 0x41, 0x6b, 0xa5, 0x67, 0xcd, 0x2f, 0x61, 0xe5, -- 0x18, 0x0f, 0x02, 0x67, 0x74, 0x83, 0xfa, 0xae, 0xc2, 0x62, 0x24, 0x78, 0x0a, 0x55, 0x9b, 0x96, -- 0x82, 0xcc, 0x23, 0x30, 0xbe, 0x70, 0x30, 0xbb, 0x39, 0x49, 0xe6, 0x1b, 0xb0, 0x9c, 0xe3, 0x18, -- 0x85, 0x24, 0x88, 0x90, 0x50, 0x80, 0x39, 0x2c, 0x8e, 0x04, 0xb3, 0x05, 0x4b, 0x41, 0x26, 0x81, -- 0xd5, 0x93, 0xd0, 0xbb, 0xe6, 0x6d, 0x7a, 0x00, 0x35, 0x8a, 0x22, 0x12, 0x53, 0x7e, 0x07, 0x8a, -- 0xc2, 0xa9, 0x2b, 0xd2, 0xa9, 0x9f, 0xe2, 0x20, 0xbe, 0xb4, 0xf4, 0x9c, 0x95, 0x92, 0xa9, 0xf3, -- 0xc9, 0xa2, 0xeb, 0x9c, 0xcf, 0xf7, 0xe1, 0xd6, 0x91, 0x13, 0x47, 0xd7, 0xd1, 0xd5, 0xfc, 0x80, -- 0x9f, 0xed, 0x28, 0xf6, 0xaf, 0xb5, 0xf8, 0x8f, 0x05, 0xa8, 0x6e, 0x85, 0xf1, 0x49, 0xe4, 0x0c, -- 0x90, 0xf1, 0x3d, 0xa8, 0x33, 0xc2, 0x9c, 0x91, 0x1d, 0x73, 0x50, 0x90, 0x97, 0x2d, 0x10, 0x28, -- 0x49, 0xf0, 0x12, 0x34, 0x42, 0x44, 0xdd, 0x30, 0x56, 0x14, 0xc5, 0xbb, 0xa5, 0xf5, 0xb2, 0x55, -- 0x97, 0x38, 0x49, 0xd2, 0x87, 0x65, 0x31, 0x67, 0xe3, 0xc0, 0x3e, 0x47, 0x34, 0x40, 0x23, 0x9f, -- 0x78, 0x48, 0x1c, 0x8e, 0xb2, 0xd5, 0x11, 0x53, 0xfb, 0xc1, 0x27, 0xc9, 0x84, 0xf1, 0x7d, 0xe8, -- 0x24, 0xf4, 0xfc, 0xc4, 0x0b, 0xea, 0xb2, 0xa0, 0x6e, 0x2b, 0xea, 0x13, 0x85, 0x36, 0x7f, 0x09, -- 0xad, 0x27, 0x43, 0x4a, 0x18, 0x1b, 0xe1, 0x60, 0xb0, 0xed, 0x30, 0x87, 0x5f, 0xcd, 0x10, 0x51, -- 0x4c, 0xbc, 0x48, 0x69, 0xab, 0x41, 0xe3, 0x75, 0xe8, 0x30, 0x49, 0x8b, 0x3c, 0x5b, 0xd3, 0x14, -- 0x05, 0xcd, 0x52, 0x32, 0x71, 0xa4, 0x88, 0x5f, 0x81, 0x56, 0x4a, 0xcc, 0x2f, 0xb7, 0xd2, 0xb7, -- 0x99, 0x60, 0x9f, 0x60, 0x1f, 0x99, 0x17, 0xc2, 0x57, 0x62, 0x93, 0x8d, 0xd7, 0xa1, 0x96, 0xfa, -- 0xa1, 0x20, 0x4e, 0x48, 0x4b, 0x9e, 0x10, 0xed, 0x4e, 0xab, 0x9a, 0x38, 0xe5, 0x43, 0x68, 0xb3, -- 0x44, 0x71, 0xdb, 0x73, 0x98, 0x93, 0x3f, 0x54, 0x79, 0xab, 0xac, 0x16, 0xcb, 0xc1, 0xe6, 0x07, -- 0x50, 0x3b, 0xc2, 0x5e, 0x24, 0x05, 0x77, 0xa1, 0xe2, 0xc6, 0x94, 0xa2, 0x80, 0x69, 0x93, 0x15, -- 0x68, 0xac, 0xc0, 0xc2, 0x08, 0xfb, 0x98, 0x29, 0x33, 0x25, 0x60, 0x12, 0x80, 0x03, 0xe4, 0x13, -- 0x3a, 0x16, 0x0e, 0x5b, 0x81, 0x85, 0xec, 0xe6, 0x4a, 0xc0, 0x78, 0x1e, 0x6a, 0xbe, 0x73, 0x99, -- 0x6c, 0x2a, 0x9f, 0xa9, 0xfa, 0xce, 0xa5, 0x54, 0xbe, 0x0b, 0x95, 0x33, 0x07, 0x8f, 0xdc, 0x80, -- 0x29, 0xaf, 0x68, 0x30, 0x15, 0x58, 0xce, 0x0a, 0xfc, 0x6b, 0x11, 0xea, 0x52, 0xa2, 0x54, 0x78, -- 0x05, 0x16, 0x5c, 0xc7, 0x1d, 0x26, 0x22, 0x05, 0x60, 0xdc, 0xd7, 0x8a, 0x14, 0xb3, 0x11, 0x2e, -- 0xd5, 0x54, 0xab, 0xb6, 0x01, 0x10, 0x3d, 0x75, 0x42, 0xa5, 0x5b, 0x69, 0x0e, 0x71, 0x8d, 0xd3, -- 0x48, 0x75, 0xdf, 0x86, 0x86, 0x3c, 0x77, 0x6a, 0x49, 0x79, 0xce, 0x92, 0xba, 0xa4, 0x92, 0x8b, -- 0xee, 0x41, 0x33, 0x8e, 0x90, 0x3d, 0xc4, 0x88, 0x3a, 0xd4, 0x1d, 0x8e, 0xbb, 0x0b, 0xf2, 0x01, -- 0x8a, 0x23, 0xb4, 0xa7, 0x71, 0xc6, 0x03, 0x58, 0xe0, 0xb1, 0x25, 0xea, 0x2e, 0x8a, 0xb7, 0xee, -- 0x85, 0x2c, 0x4b, 0x61, 0x6a, 0x5f, 0xfc, 0xee, 0x04, 0x8c, 0x8e, 0x2d, 0x49, 0xda, 0x7b, 0x17, -- 0x20, 0x45, 0x1a, 0x4b, 0x50, 0x3a, 0x47, 0x63, 0x75, 0x0f, 0xf9, 0x90, 0x3b, 0xe7, 0xc2, 0x19, -- 0xc5, 0xda, 0xeb, 0x12, 0x78, 0xbf, 0xf8, 0x6e, 0xc1, 0x74, 0xa1, 0xbd, 0x39, 0x3a, 0xc7, 0x24, -- 0xb3, 0x7c, 0x05, 0x16, 0x7c, 0xe7, 0x4b, 0x42, 0xb5, 0x27, 0x05, 0x20, 0xb0, 0x38, 0x20, 0x54, -- 0xb3, 0x10, 0x80, 0xd1, 0x82, 0x22, 0x09, 0x85, 0xbf, 0x6a, 0x56, 0x91, 0x84, 0xa9, 0xa0, 0x72, -- 0x46, 0x90, 0xf9, 0xcf, 0x32, 0x40, 0x2a, 0xc5, 0xb0, 0xa0, 0x87, 0x89, 0x1d, 0x21, 0xca, 0xdf, -- 0x77, 0xfb, 0x74, 0xcc, 0x50, 0x64, 0x53, 0xe4, 0xc6, 0x34, 0xc2, 0x17, 0x7c, 0xff, 0xb8, 0xd9, -- 0xb7, 0xa4, 0xd9, 0x13, 0xba, 0x59, 0xb7, 0x31, 0x39, 0x96, 0xeb, 0x36, 0xf9, 0x32, 0x4b, 0xaf, -- 0x32, 0xf6, 0xe1, 0x56, 0xca, 0xd3, 0xcb, 0xb0, 0x2b, 0x5e, 0xc5, 0x6e, 0x39, 0x61, 0xe7, 0xa5, -- 0xac, 0x76, 0x60, 0x19, 0x13, 0xfb, 0xab, 0x18, 0xc5, 0x39, 0x46, 0xa5, 0xab, 0x18, 0x75, 0x30, -- 0xf9, 0x5c, 0x2c, 0x48, 0xd9, 0x1c, 0xc1, 0x9d, 0x8c, 0x95, 0xfc, 0xba, 0x67, 0x98, 0x95, 0xaf, -- 0x62, 0xb6, 0x9a, 0x68, 0xc5, 0xe3, 0x41, 0xca, 0xf1, 0x63, 0x58, 0xc5, 0xc4, 0x7e, 0xea, 0x60, -- 0x36, 0xc9, 0x6e, 0xe1, 0x3b, 0x8c, 0xe4, 0x2f, 0x5a, 0x9e, 0x97, 0x34, 0xd2, 0x47, 0x74, 0x90, -- 0x33, 0x72, 0xf1, 0x3b, 0x8c, 0x3c, 0x10, 0x0b, 0x52, 0x36, 0x8f, 0xa1, 0x83, 0xc9, 0xa4, 0x36, -- 0x95, 0xab, 0x98, 0xb4, 0x31, 0xc9, 0x6b, 0xb2, 0x09, 0x9d, 0x08, 0xb9, 0x8c, 0xd0, 0xec, 0x21, -- 0xa8, 0x5e, 0xc5, 0x62, 0x49, 0xd1, 0x27, 0x3c, 0xcc, 0x9f, 0x42, 0x63, 0x2f, 0x1e, 0x20, 0x36, -- 0x3a, 0x4d, 0x82, 0xc1, 0x8d, 0xc5, 0x1f, 0xf3, 0xdf, 0x45, 0xa8, 0x6f, 0x0d, 0x28, 0x89, 0xc3, -- 0x5c, 0x4c, 0x96, 0x97, 0x74, 0x32, 0x26, 0x0b, 0x12, 0x11, 0x93, 0x25, 0xf1, 0x3b, 0xd0, 0xf0, -- 0xc5, 0xd5, 0x55, 0xf4, 0x32, 0x0e, 0x75, 0xa6, 0x2e, 0xb5, 0x55, 0xf7, 0x33, 0xc1, 0xac, 0x0f, -- 0x10, 0x62, 0x2f, 0x52, 0x6b, 0x64, 0x38, 0x6a, 0xab, 0x74, 0x4b, 0x87, 0x68, 0xab, 0x16, 0x26, -- 0xd1, 0xfa, 0x2d, 0xa8, 0x9f, 0x72, 0x27, 0xa9, 0x05, 0xb9, 0x60, 0x94, 0x7a, 0xcf, 0x82, 0xd3, -- 0xf4, 0x12, 0xee, 0x41, 0x73, 0x28, 0x5d, 0xa6, 0x16, 0xc9, 0x33, 0x74, 0x4f, 0x59, 0x92, 0xda, -- 0xdb, 0xcf, 0x7a, 0x56, 0x6e, 0x40, 0x63, 0x98, 0x41, 0xf5, 0x8e, 0xa1, 0x33, 0x45, 0x32, 0x23, -- 0x06, 0xad, 0x67, 0x63, 0x50, 0xfd, 0x81, 0x21, 0x05, 0x65, 0x57, 0x66, 0xe3, 0xd2, 0xef, 0x8a, -- 0xd0, 0xf8, 0x0c, 0xb1, 0xa7, 0x84, 0x9e, 0x4b, 0x7d, 0x0d, 0x28, 0x07, 0x8e, 0x8f, 0x14, 0x47, -- 0x31, 0x36, 0xee, 0x40, 0x95, 0x5e, 0xca, 0x00, 0xa2, 0xf6, 0xb3, 0x42, 0x2f, 0x45, 0x60, 0x30, -- 0x5e, 0x04, 0xa0, 0x97, 0x76, 0xe8, 0xb8, 0xe7, 0x48, 0x79, 0xb0, 0x6c, 0xd5, 0xe8, 0xe5, 0x91, -- 0x44, 0xf0, 0xa3, 0x40, 0x2f, 0x6d, 0x44, 0x29, 0xa1, 0x91, 0x8a, 0x55, 0x55, 0x7a, 0xb9, 0x23, -- 0x60, 0xb5, 0xd6, 0xa3, 0x24, 0x0c, 0x91, 0x27, 0x62, 0xb4, 0x58, 0xbb, 0x2d, 0x11, 0x5c, 0x2a, -- 0xd3, 0x52, 0x17, 0xa5, 0x54, 0x96, 0x4a, 0x65, 0xa9, 0xd4, 0x8a, 0x5c, 0xc9, 0xb2, 0x52, 0x59, -- 0x22, 0xb5, 0x2a, 0xa5, 0xb2, 0x8c, 0x54, 0x96, 0x4a, 0xad, 0xe9, 0xb5, 0x4a, 0xaa, 0xf9, 0xdb, -- 0x02, 0xac, 0x4e, 0x26, 0x7e, 0x2a, 0x37, 0x7d, 0x07, 0x1a, 0xae, 0xd8, 0xaf, 0xdc, 0x99, 0xec, -- 0x4c, 0xed, 0xa4, 0x55, 0x77, 0x33, 0xc7, 0xf8, 0x21, 0x34, 0x03, 0xe9, 0xe0, 0xe4, 0x68, 0x96, -- 0xd2, 0x7d, 0xc9, 0xfa, 0xde, 0x6a, 0x04, 0x19, 0xc8, 0xf4, 0xc0, 0xf8, 0x82, 0x62, 0x86, 0x8e, -- 0x19, 0x45, 0x8e, 0x7f, 0x13, 0xd9, 0xbd, 0x01, 0x65, 0x91, 0xad, 0xf0, 0x6d, 0x6a, 0x58, 0x62, -- 0x6c, 0xbe, 0x0a, 0xcb, 0x39, 0x29, 0xca, 0xd6, 0x25, 0x28, 0x8d, 0x50, 0x20, 0xb8, 0x37, 0x2d, -- 0x3e, 0x34, 0x1d, 0xe8, 0x58, 0xc8, 0xf1, 0x6e, 0x4e, 0x1b, 0x25, 0xa2, 0x94, 0x8a, 0x58, 0x07, -- 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x64, 0xb4, 0x3e, 0x84, 0xce, 0xd6, 0x88, 0x44, 0xe8, -- 0x98, 0x79, 0x38, 0xb8, 0x89, 0x72, 0xe4, 0x17, 0xb0, 0xfc, 0x84, 0x8d, 0xbf, 0xe0, 0xcc, 0x22, -- 0xfc, 0x35, 0xba, 0x21, 0xfb, 0x28, 0x79, 0xaa, 0xed, 0xa3, 0xe4, 0x29, 0x2f, 0x6e, 0x5c, 0x32, -- 0x8a, 0xfd, 0x40, 0x5c, 0x85, 0xa6, 0xa5, 0x20, 0x73, 0x13, 0x1a, 0x32, 0x87, 0x3e, 0x20, 0x5e, -- 0x3c, 0x42, 0x33, 0xef, 0xe0, 0x1a, 0x40, 0xe8, 0x50, 0xc7, 0x47, 0x0c, 0x51, 0x79, 0x86, 0x6a, -- 0x56, 0x06, 0x63, 0xfe, 0xbe, 0x08, 0x2b, 0xb2, 0xdf, 0x70, 0x2c, 0xcb, 0x6c, 0x6d, 0x42, 0x0f, -- 0xaa, 0x43, 0x12, 0xb1, 0x0c, 0xc3, 0x04, 0xe6, 0x2a, 0xf2, 0xfa, 0x5c, 0x72, 0xe3, 0xc3, 0x5c, -- 0x13, 0xa0, 0x74, 0x75, 0x13, 0x60, 0xaa, 0xcc, 0x2f, 0x4f, 0x97, 0xf9, 0xfc, 0xb6, 0x69, 0x22, -- 0x2c, 0xef, 0x78, 0xcd, 0xaa, 0x29, 0xcc, 0xbe, 0x67, 0xdc, 0x87, 0xf6, 0x80, 0x6b, 0x69, 0x0f, -- 0x09, 0x39, 0xb7, 0x43, 0x87, 0x0d, 0xc5, 0x55, 0xaf, 0x59, 0x4d, 0x81, 0xde, 0x23, 0xe4, 0xfc, -- 0xc8, 0x61, 0x43, 0xe3, 0x3d, 0x68, 0xa9, 0x34, 0xd0, 0x17, 0x2e, 0x8a, 0xd4, 0xe3, 0xa7, 0x6e, -- 0x51, 0xd6, 0x7b, 0x56, 0xf3, 0x3c, 0x03, 0x45, 0xe6, 0x6d, 0xb8, 0xb5, 0x8d, 0x22, 0x46, 0xc9, -- 0x38, 0xef, 0x18, 0xf3, 0x47, 0x00, 0xfb, 0x01, 0x43, 0xf4, 0xcc, 0x71, 0x51, 0x64, 0xbc, 0x99, -- 0x85, 0x54, 0x72, 0xb4, 0xd4, 0x97, 0xed, 0x9e, 0x64, 0xc2, 0xca, 0xd0, 0x98, 0x7d, 0x58, 0xb4, -- 0x48, 0xcc, 0xc3, 0xd1, 0xcb, 0x7a, 0xa4, 0xd6, 0x35, 0xd4, 0x3a, 0x81, 0xb4, 0xd4, 0x9c, 0xb9, -- 0xa7, 0x4b, 0xd8, 0x94, 0x9d, 0xda, 0xa2, 0x3e, 0xd4, 0xb0, 0xc6, 0xa9, 0xa8, 0x32, 0x2d, 0x3a, -- 0x25, 0x31, 0x3f, 0x80, 0x65, 0xc9, 0x49, 0x72, 0xd6, 0x6c, 0x5e, 0x86, 0x45, 0xaa, 0xd5, 0x28, -- 0xa4, 0x7d, 0x1e, 0x45, 0xa4, 0xe6, 0xb8, 0x3f, 0x3e, 0xc5, 0x11, 0x4b, 0x0d, 0xd1, 0xfe, 0x58, -- 0x86, 0x0e, 0x9f, 0xc8, 0xf1, 0x34, 0x3f, 0x82, 0xc6, 0x63, 0xeb, 0xe8, 0x33, 0x84, 0x07, 0xc3, -- 0x53, 0x1e, 0x3d, 0x7f, 0x98, 0x87, 0x95, 0xc1, 0x86, 0xd2, 0x36, 0x33, 0x65, 0xe5, 0xe8, 0xcc, -- 0x8f, 0x61, 0xf5, 0xb1, 0xe7, 0x65, 0x51, 0x5a, 0xeb, 0x37, 0xa1, 0x16, 0x64, 0xd8, 0x65, 0xde, -- 0xac, 0x1c, 0x75, 0x4a, 0x64, 0xfe, 0x0c, 0x96, 0x0f, 0x83, 0x11, 0x0e, 0xd0, 0xd6, 0xd1, 0xc9, -- 0x01, 0x4a, 0x62, 0x91, 0x01, 0x65, 0x9e, 0xb3, 0x09, 0x1e, 0x55, 0x4b, 0x8c, 0xf9, 0xe5, 0x0c, -- 0x4e, 0x6d, 0x37, 0x8c, 0x23, 0xd5, 0xec, 0x59, 0x0c, 0x4e, 0xb7, 0xc2, 0x38, 0xe2, 0x8f, 0x0b, -- 0x4f, 0x2e, 0x48, 0x30, 0x1a, 0x8b, 0x1b, 0x5a, 0xb5, 0x2a, 0x6e, 0x18, 0x1f, 0x06, 0xa3, 0xb1, -- 0xf9, 0x03, 0x51, 0x81, 0x23, 0xe4, 0x59, 0x4e, 0xe0, 0x11, 0x7f, 0x1b, 0x5d, 0x64, 0x24, 0x24, -- 0xd5, 0x9e, 0x8e, 0x44, 0xdf, 0x14, 0xa0, 0xf1, 0x78, 0x80, 0x02, 0xb6, 0x8d, 0x98, 0x83, 0x47, -- 0xa2, 0xa2, 0xbb, 0x40, 0x34, 0xc2, 0x24, 0x50, 0xd7, 0x4d, 0x83, 0xbc, 0x20, 0xc7, 0x01, 0x66, -- 0xb6, 0xe7, 0x20, 0x9f, 0x04, 0x82, 0x4b, 0xd5, 0x02, 0x8e, 0xda, 0x16, 0x18, 0xe3, 0x55, 0x68, -- 0xcb, 0x66, 0x9c, 0x3d, 0x74, 0x02, 0x6f, 0xc4, 0x2f, 0x7a, 0x49, 0x5c, 0xcd, 0x96, 0x44, 0xef, -- 0x29, 0xac, 0xf1, 0x1a, 0x2c, 0xa9, 0x6b, 0x98, 0x52, 0x96, 0x05, 0x65, 0x5b, 0xe1, 0x73, 0xa4, -- 0x71, 0x18, 0x12, 0xca, 0x22, 0x3b, 0x42, 0xae, 0x4b, 0xfc, 0x50, 0x95, 0x43, 0x6d, 0x8d, 0x3f, -- 0x96, 0x68, 0x73, 0x00, 0xcb, 0xbb, 0xdc, 0x4e, 0x65, 0x49, 0x7a, 0xac, 0x5a, 0x3e, 0xf2, 0xed, -- 0xd3, 0x11, 0x71, 0xcf, 0x6d, 0x1e, 0x1c, 0x95, 0x87, 0x79, 0xc2, 0xb5, 0xc9, 0x91, 0xc7, 0xf8, -- 0x6b, 0x51, 0xf9, 0x73, 0xaa, 0x21, 0x61, 0xe1, 0x28, 0x1e, 0xd8, 0x21, 0x25, 0xa7, 0x48, 0x99, -- 0xd8, 0xf6, 0x91, 0xbf, 0x27, 0xf1, 0x47, 0x1c, 0x6d, 0xfe, 0xa5, 0x00, 0x2b, 0x79, 0x49, 0x2a, -- 0xd4, 0x6f, 0xc0, 0x4a, 0x5e, 0x94, 0x7a, 0xfe, 0x65, 0x7a, 0xd9, 0xc9, 0x0a, 0x94, 0x89, 0xc0, -- 0x43, 0x68, 0x8a, 0x7e, 0xad, 0xed, 0x49, 0x4e, 0xf9, 0xa4, 0x27, 0xbb, 0x2f, 0x56, 0xc3, 0xc9, -- 0xee, 0xd2, 0x7b, 0x70, 0x47, 0x99, 0x6f, 0x4f, 0xab, 0x2d, 0x0f, 0xc4, 0xaa, 0x22, 0x38, 0x98, -- 0xd0, 0xfe, 0x53, 0xe8, 0xa6, 0xa8, 0xcd, 0xb1, 0x40, 0xa6, 0x87, 0x79, 0x79, 0xc2, 0xd8, 0xc7, -- 0x9e, 0x47, 0xc5, 0x2d, 0x29, 0x5b, 0xb3, 0xa6, 0xcc, 0x47, 0x70, 0xfb, 0x18, 0x31, 0xe9, 0x0d, -- 0x87, 0xa9, 0x4a, 0x44, 0x32, 0x5b, 0x82, 0xd2, 0x31, 0x72, 0x85, 0xf1, 0x25, 0x8b, 0x0f, 0xf9, -- 0x01, 0x3c, 0x89, 0x90, 0x2b, 0xac, 0x2c, 0x59, 0x62, 0x6c, 0xfe, 0xb9, 0x00, 0x15, 0x15, 0x9c, -- 0xf9, 0x03, 0xe3, 0x51, 0x7c, 0x81, 0xa8, 0x3a, 0x7a, 0x0a, 0x32, 0x5e, 0x81, 0x96, 0x1c, 0xd9, -- 0x24, 0x64, 0x98, 0x24, 0x21, 0xbf, 0x29, 0xb1, 0x87, 0x12, 0x29, 0x9a, 0x6f, 0xa2, 0xfd, 0xa5, -- 0x2a, 0x4d, 0x05, 0x71, 0xfc, 0x59, 0xc4, 0x6f, 0xb8, 0x08, 0xf1, 0x35, 0x4b, 0x41, 0xfc, 0xa8, -- 0x6b, 0x7e, 0x0b, 0x82, 0x9f, 0x06, 0xf9, 0x51, 0xf7, 0x49, 0x1c, 0x30, 0x3b, 0x24, 0x38, 0x60, -- 0x2a, 0xa6, 0x83, 0x40, 0x1d, 0x71, 0x8c, 0xf9, 0x9b, 0x02, 0x2c, 0xca, 0x06, 0x34, 0xaf, 0x6d, -- 0x93, 0x97, 0xb5, 0x88, 0x45, 0x96, 0x22, 0x64, 0xc9, 0xd7, 0x54, 0x8c, 0xf9, 0x3d, 0xbe, 0xf0, -- 0xe5, 0xfb, 0xa0, 0x54, 0xbb, 0xf0, 0xc5, 0xc3, 0xf0, 0x0a, 0xb4, 0xd2, 0x07, 0x5a, 0xcc, 0x4b, -- 0x15, 0x9b, 0x09, 0x56, 0x90, 0xcd, 0xd5, 0xd4, 0xfc, 0x31, 0x2f, 0xe9, 0x93, 0xe6, 0xeb, 0x12, -- 0x94, 0xe2, 0x44, 0x19, 0x3e, 0xe4, 0x98, 0x41, 0xf2, 0xb4, 0xf3, 0xa1, 0x71, 0x1f, 0x5a, 0x8e, -- 0xe7, 0x61, 0xbe, 0xdc, 0x19, 0xed, 0x62, 0x2f, 0xb9, 0xa4, 0x79, 0xac, 0xf9, 0xb7, 0x02, 0xb4, -- 0xb7, 0x48, 0x38, 0xfe, 0x08, 0x8f, 0x50, 0x26, 0x82, 0x08, 0x25, 0xd5, 0xcb, 0xce, 0xc7, 0x3c, -- 0x5b, 0x3d, 0xc3, 0x23, 0x24, 0xaf, 0x96, 0xdc, 0xd9, 0x2a, 0x47, 0x88, 0x6b, 0xa5, 0x27, 0x93, -- 0xb6, 0x5b, 0x53, 0x4e, 0x1e, 0x10, 0x4f, 0xe4, 0xe5, 0x1e, 0xa6, 0x76, 0xd2, 0x64, 0x6b, 0x5a, -- 0x15, 0x0f, 0x53, 0x31, 0xa5, 0x0c, 0x59, 0x10, 0x4d, 0xd4, 0xac, 0x21, 0x8b, 0x12, 0xc3, 0x0d, -- 0x59, 0x85, 0x45, 0x72, 0x76, 0x16, 0x21, 0x26, 0x32, 0xe8, 0x92, 0xa5, 0xa0, 0x24, 0xcc, 0x55, -- 0x33, 0x61, 0xee, 0x16, 0x2c, 0x8b, 0x76, 0xfd, 0x13, 0xea, 0xb8, 0x38, 0x18, 0xe8, 0xe7, 0x61, -- 0x05, 0x8c, 0x63, 0x46, 0xc2, 0x69, 0xec, 0x2e, 0x62, 0x87, 0x87, 0x07, 0x3b, 0x17, 0x28, 0x60, -- 0x1a, 0xfb, 0x06, 0x54, 0x35, 0xea, 0xbf, 0xe9, 0x65, 0x2e, 0x43, 0x67, 0x17, 0xb1, 0x03, 0xc4, -- 0x28, 0x76, 0x93, 0xe7, 0xe8, 0x1e, 0x54, 0x14, 0x86, 0x6f, 0xa9, 0x2f, 0x87, 0x3a, 0xce, 0x2a, -- 0xf0, 0xc1, 0xaf, 0x3b, 0x2a, 0x24, 0xab, 0xea, 0xde, 0xd8, 0x85, 0xf6, 0xc4, 0xa7, 0x18, 0x43, -- 0xb5, 0x7b, 0x66, 0x7f, 0xa1, 0xe9, 0xad, 0xf6, 0xe5, 0xa7, 0x9d, 0xbe, 0xfe, 0xb4, 0xd3, 0xdf, -- 0xf1, 0x43, 0x36, 0x36, 0x76, 0xa0, 0x95, 0xff, 0x68, 0x61, 0x3c, 0xaf, 0xb3, 0xa3, 0x19, 0x9f, -- 0x32, 0xe6, 0xb2, 0xd9, 0x85, 0xf6, 0xc4, 0xf7, 0x0b, 0xad, 0xcf, 0xec, 0xcf, 0x1a, 0x73, 0x19, -- 0x3d, 0x82, 0x7a, 0xe6, 0x83, 0x85, 0xd1, 0x95, 0x4c, 0xa6, 0xbf, 0x61, 0xcc, 0x65, 0xb0, 0x05, -- 0xcd, 0xdc, 0x37, 0x04, 0xa3, 0xa7, 0xec, 0x99, 0xf1, 0x61, 0x61, 0x2e, 0x93, 0x4d, 0xa8, 0x67, -- 0x5a, 0xf9, 0x5a, 0x8b, 0xe9, 0xef, 0x05, 0xbd, 0x3b, 0x33, 0x66, 0x54, 0xe4, 0xdf, 0x85, 0xf6, -- 0x44, 0x7f, 0x5f, 0xbb, 0x64, 0x76, 0xdb, 0x7f, 0xae, 0x32, 0x9f, 0x88, 0x2d, 0xca, 0x94, 0x6f, -- 0x99, 0x2d, 0x9a, 0xee, 0xe6, 0xf7, 0x5e, 0x98, 0x3d, 0xa9, 0xb4, 0xda, 0x81, 0x56, 0xbe, 0x91, -- 0xaf, 0x99, 0xcd, 0x6c, 0xef, 0x5f, 0xbd, 0xdf, 0xb9, 0x9e, 0x7e, 0xba, 0xdf, 0xb3, 0x5a, 0xfd, -- 0x73, 0x19, 0x3d, 0x06, 0x50, 0xc5, 0x9a, 0x87, 0x83, 0xc4, 0xd1, 0x53, 0x45, 0x62, 0xe2, 0xe8, -- 0x19, 0x85, 0xdd, 0x23, 0x00, 0x59, 0x63, 0x79, 0x24, 0x66, 0xc6, 0x6d, 0xad, 0xc6, 0x44, 0x61, -- 0xd7, 0xeb, 0x4e, 0x4f, 0x4c, 0x31, 0x40, 0x94, 0x5e, 0x87, 0xc1, 0x87, 0x00, 0x69, 0xed, 0xa6, -- 0x19, 0x4c, 0x55, 0x73, 0x57, 0xf8, 0xa0, 0x91, 0xad, 0xd4, 0x0c, 0x65, 0xeb, 0x8c, 0xea, 0xed, -- 0x0a, 0x16, 0xed, 0x89, 0x4c, 0x3c, 0x7f, 0xd8, 0x26, 0x13, 0xf4, 0xde, 0x54, 0x36, 0x6e, 0x3c, -- 0x84, 0x46, 0x36, 0x05, 0xd7, 0x5a, 0xcc, 0x48, 0xcb, 0x7b, 0xb9, 0x34, 0xdc, 0x78, 0x04, 0xad, -- 0x7c, 0xfa, 0xad, 0x8f, 0xd4, 0xcc, 0xa4, 0xbc, 0xa7, 0x9a, 0x4b, 0x19, 0xf2, 0xb7, 0x01, 0xd2, -- 0x34, 0x5d, 0xbb, 0x6f, 0x2a, 0x71, 0x9f, 0x90, 0xba, 0x0b, 0xed, 0x89, 0xf4, 0x5b, 0x5b, 0x3c, -- 0x3b, 0x2b, 0xbf, 0xca, 0xfb, 0xd9, 0x77, 0x40, 0xdb, 0x3d, 0xe3, 0x6d, 0xb8, 0x2a, 0x68, 0x65, -- 0xde, 0x0c, 0x7d, 0x8a, 0xa7, 0x9f, 0x91, 0xb9, 0x0c, 0xde, 0x01, 0x48, 0x5f, 0x06, 0xed, 0x81, -- 0xa9, 0xb7, 0xa2, 0xd7, 0xd4, 0xcd, 0x3f, 0x49, 0xb7, 0x05, 0xcd, 0x5c, 0x7d, 0xac, 0x43, 0xdd, -- 0xac, 0xa2, 0xf9, 0xaa, 0x07, 0x20, 0x5f, 0x4c, 0xea, 0xdd, 0x9b, 0x59, 0x62, 0x5e, 0xe5, 0xc5, -- 0x6c, 0x05, 0xa3, 0xbd, 0x38, 0xa3, 0xaa, 0xf9, 0x8e, 0x98, 0x92, 0xad, 0x52, 0x32, 0x31, 0x65, -- 0x46, 0xf1, 0x32, 0x97, 0xd1, 0x1e, 0xb4, 0x77, 0x75, 0x02, 0xaa, 0x92, 0x63, 0xa5, 0xce, 0x8c, -- 0x62, 0xa0, 0xd7, 0x9b, 0x35, 0xa5, 0x2e, 0xf6, 0x27, 0xd0, 0x99, 0x4a, 0x8c, 0x8d, 0xb5, 0xa4, -- 0x05, 0x3b, 0x33, 0x63, 0x9e, 0xab, 0xd6, 0x3e, 0x2c, 0x4d, 0xe6, 0xc5, 0xc6, 0x8b, 0xea, 0xa8, -- 0xcc, 0xce, 0x97, 0xe7, 0xb2, 0x7a, 0x0f, 0xaa, 0x3a, 0x0f, 0x33, 0x54, 0xab, 0x7b, 0x22, 0x2f, -- 0x9b, 0xbb, 0xf4, 0x21, 0xd4, 0x33, 0x99, 0x8c, 0x3e, 0xab, 0xd3, 0xc9, 0x4d, 0x4f, 0x75, 0xa6, -- 0x35, 0x7a, 0xf3, 0xf2, 0x9b, 0x6f, 0xd7, 0x9e, 0xfb, 0xc7, 0xb7, 0x6b, 0xcf, 0xfd, 0xea, 0xd9, -- 0x5a, 0xe1, 0x9b, 0x67, 0x6b, 0x85, 0xbf, 0x3f, 0x5b, 0x2b, 0xfc, 0xeb, 0xd9, 0x5a, 0xe1, 0x27, -- 0x3f, 0xff, 0x1f, 0xff, 0xa6, 0x42, 0xe3, 0x80, 0x61, 0x1f, 0x6d, 0x5c, 0x60, 0xca, 0x32, 0x53, -- 0xe1, 0xf9, 0x60, 0xea, 0x1f, 0x2c, 0x5c, 0x85, 0xd3, 0x45, 0x01, 0xbf, 0xfd, 0x9f, 0x00, 0x00, -- 0x00, 0xff, 0xff, 0x23, 0xa0, 0xd8, 0x6f, 0x0f, 0x23, 0x00, 0x00, -+ // 3007 bytes of a gzipped FileDescriptorProto -+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3a, 0x4b, 0x73, 0x1b, 0xc7, -+ 0xd1, 0xc6, 0x83, 0x04, 0xd0, 0x78, 0x11, 0x4b, 0x8a, 0x82, 0x20, 0x9b, 0x9f, 0xbc, 0xb2, 0x65, -+ 0xfa, 0xf3, 0x67, 0xc8, 0x96, 0x5d, 0x9f, 0xfc, 0x28, 0x7f, 0x2a, 0x91, 0xa2, 0x49, 0xda, 0xa6, -+ 0x45, 0x2f, 0xa5, 0xf2, 0x97, 0xa4, 0x92, 0xad, 0xe5, 0xee, 0x08, 0x18, 0x13, 0xbb, 0xb3, 0x9e, -+ 0x9d, 0xa5, 0x08, 0xa7, 0x2a, 0x95, 0x53, 0x72, 0xcb, 0x31, 0xb7, 0xfc, 0x81, 0x54, 0x6e, 0x39, -+ 0xe6, 0x9a, 0x83, 0x2b, 0xa7, 0x1c, 0x73, 0x48, 0xa5, 0x62, 0xfd, 0x84, 0xfc, 0x82, 0xd4, 0xbc, -+ 0xf6, 0x01, 0x2c, 0xe0, 0x84, 0xa5, 0xaa, 0x5c, 0x50, 0xdb, 0x3d, 0x3d, 0xfd, 0x9a, 0xe9, 0x9e, -+ 0xee, 0x19, 0xc0, 0x17, 0x23, 0xcc, 0xc6, 0xf1, 0xe9, 0xd0, 0x25, 0xfe, 0xed, 0x33, 0x87, 0x39, -+ 0x6f, 0xba, 0x24, 0x60, 0x0e, 0x0e, 0x10, 0x8d, 0xe6, 0xe0, 0x88, 0xba, 0xb7, 0x9d, 0x11, 0x0a, -+ 0xd8, 0xed, 0x90, 0x12, 0x46, 0x5c, 0x32, 0x89, 0xe4, 0x57, 0x24, 0xd1, 0x43, 0x01, 0x18, 0xd5, -+ 0x11, 0x0d, 0xdd, 0x41, 0x83, 0xb8, 0x58, 0x22, 0x06, 0x4d, 0x36, 0x0d, 0x51, 0xa4, 0x80, 0xeb, -+ 0x23, 0x42, 0x46, 0x13, 0x24, 0x27, 0x9e, 0xc6, 0x4f, 0x6e, 0x23, 0x3f, 0x64, 0x53, 0x39, 0x68, -+ 0xfe, 0xa6, 0x0c, 0x9b, 0xbb, 0x14, 0x39, 0x0c, 0xed, 0x6a, 0xb1, 0x16, 0xfa, 0x3a, 0x46, 0x11, -+ 0x33, 0x5e, 0x86, 0x56, 0xa2, 0x8a, 0x8d, 0xbd, 0x7e, 0xe9, 0x46, 0x69, 0xbb, 0x61, 0x35, 0x13, -+ 0xdc, 0xa1, 0x67, 0x5c, 0x85, 0x1a, 0xba, 0x40, 0x2e, 0x1f, 0x2d, 0x8b, 0xd1, 0x55, 0x0e, 0x1e, -+ 0x7a, 0xc6, 0xdb, 0xd0, 0x8c, 0x18, 0xc5, 0xc1, 0xc8, 0x8e, 0x23, 0x44, 0xfb, 0x95, 0x1b, 0xa5, -+ 0xed, 0xe6, 0x9d, 0xb5, 0x21, 0xd7, 0x73, 0x78, 0x22, 0x06, 0x1e, 0x47, 0x88, 0x5a, 0x10, 0x25, -+ 0xdf, 0xc6, 0x2d, 0xa8, 0x79, 0xe8, 0x1c, 0xbb, 0x28, 0xea, 0x57, 0x6f, 0x54, 0xb6, 0x9b, 0x77, -+ 0x5a, 0x92, 0xfc, 0x81, 0x40, 0x5a, 0x7a, 0xd0, 0x78, 0x1d, 0xea, 0x11, 0x23, 0xd4, 0x19, 0xa1, -+ 0xa8, 0xbf, 0x22, 0x08, 0xdb, 0x9a, 0xaf, 0xc0, 0x5a, 0xc9, 0xb0, 0xf1, 0x22, 0x54, 0x1e, 0xee, -+ 0x1e, 0xf6, 0x57, 0x85, 0x74, 0x50, 0x54, 0x21, 0x72, 0x2d, 0x8e, 0x36, 0x6e, 0x42, 0x3b, 0x72, -+ 0x02, 0xef, 0x94, 0x5c, 0xd8, 0x21, 0xf6, 0x82, 0xa8, 0x5f, 0xbb, 0x51, 0xda, 0xae, 0x5b, 0x2d, -+ 0x85, 0x3c, 0xe6, 0x38, 0xf3, 0x03, 0xb8, 0x72, 0xc2, 0x1c, 0xca, 0x2e, 0xe1, 0x1d, 0xf3, 0x31, -+ 0x6c, 0x5a, 0xc8, 0x27, 0xe7, 0x97, 0x72, 0x6d, 0x1f, 0x6a, 0x0c, 0xfb, 0x88, 0xc4, 0x4c, 0xb8, -+ 0xb6, 0x6d, 0x69, 0xd0, 0xfc, 0x5d, 0x09, 0x8c, 0xbd, 0x0b, 0xe4, 0x1e, 0x53, 0xe2, 0xa2, 0x28, -+ 0xfa, 0x0f, 0x2d, 0xd7, 0x6b, 0x50, 0x0b, 0xa5, 0x02, 0xfd, 0xaa, 0x20, 0x57, 0xab, 0xa0, 0xb5, -+ 0xd2, 0xa3, 0xe6, 0x57, 0xb0, 0x71, 0x82, 0x47, 0x81, 0x33, 0x79, 0x8e, 0xfa, 0x6e, 0xc2, 0x6a, -+ 0x24, 0x78, 0x0a, 0x55, 0xdb, 0x96, 0x82, 0xcc, 0x63, 0x30, 0xbe, 0x74, 0x30, 0x7b, 0x7e, 0x92, -+ 0xcc, 0x37, 0x61, 0x3d, 0xc7, 0x31, 0x0a, 0x49, 0x10, 0x21, 0xa1, 0x00, 0x73, 0x58, 0x1c, 0x09, -+ 0x66, 0x2b, 0x96, 0x82, 0x4c, 0x02, 0x9b, 0x8f, 0x43, 0xef, 0x92, 0xd1, 0x74, 0x07, 0x1a, 0x14, -+ 0x45, 0x24, 0xa6, 0x3c, 0x06, 0xca, 0xc2, 0xa9, 0x1b, 0xd2, 0xa9, 0x9f, 0xe1, 0x20, 0xbe, 0xb0, -+ 0xf4, 0x98, 0x95, 0x92, 0xa9, 0xfd, 0xc9, 0xa2, 0xcb, 0xec, 0xcf, 0x0f, 0xe0, 0xca, 0xb1, 0x13, -+ 0x47, 0x97, 0xd1, 0xd5, 0xfc, 0x90, 0xef, 0xed, 0x28, 0xf6, 0x2f, 0x35, 0xf9, 0xb7, 0x25, 0xa8, -+ 0xef, 0x86, 0xf1, 0xe3, 0xc8, 0x19, 0x21, 0xe3, 0xbf, 0xa0, 0xc9, 0x08, 0x73, 0x26, 0x76, 0xcc, -+ 0x41, 0x41, 0x5e, 0xb5, 0x40, 0xa0, 0x24, 0xc1, 0xcb, 0xd0, 0x0a, 0x11, 0x75, 0xc3, 0x58, 0x51, -+ 0x94, 0x6f, 0x54, 0xb6, 0xab, 0x56, 0x53, 0xe2, 0x24, 0xc9, 0x10, 0xd6, 0xc5, 0x98, 0x8d, 0x03, -+ 0xfb, 0x0c, 0xd1, 0x00, 0x4d, 0x7c, 0xe2, 0x21, 0xb1, 0x39, 0xaa, 0x56, 0x4f, 0x0c, 0x1d, 0x06, -+ 0x9f, 0x26, 0x03, 0xc6, 0x7f, 0x43, 0x2f, 0xa1, 0xe7, 0x3b, 0x5e, 0x50, 0x57, 0x05, 0x75, 0x57, -+ 0x51, 0x3f, 0x56, 0x68, 0xf3, 0x67, 0xd0, 0x79, 0x34, 0xa6, 0x84, 0xb1, 0x09, 0x0e, 0x46, 0x0f, -+ 0x1c, 0xe6, 0xf0, 0xd0, 0x0c, 0x11, 0xc5, 0xc4, 0x8b, 0x94, 0xb6, 0x1a, 0x34, 0xde, 0x80, 0x1e, -+ 0x93, 0xb4, 0xc8, 0xb3, 0x35, 0x4d, 0x59, 0xd0, 0xac, 0x25, 0x03, 0xc7, 0x8a, 0xf8, 0x55, 0xe8, -+ 0xa4, 0xc4, 0x3c, 0xb8, 0x95, 0xbe, 0xed, 0x04, 0xfb, 0x08, 0xfb, 0xc8, 0x3c, 0x17, 0xbe, 0x12, -+ 0x8b, 0x6c, 0xbc, 0x01, 0x8d, 0xd4, 0x0f, 0x25, 0xb1, 0x43, 0x3a, 0x72, 0x87, 0x68, 0x77, 0x5a, -+ 0xf5, 0xc4, 0x29, 0x1f, 0x41, 0x97, 0x25, 0x8a, 0xdb, 0x9e, 0xc3, 0x9c, 0xfc, 0xa6, 0xca, 0x5b, -+ 0x65, 0x75, 0x58, 0x0e, 0x36, 0x3f, 0x84, 0xc6, 0x31, 0xf6, 0x22, 0x29, 0xb8, 0x0f, 0x35, 0x37, -+ 0xa6, 0x14, 0x05, 0x4c, 0x9b, 0xac, 0x40, 0x63, 0x03, 0x56, 0x26, 0xd8, 0xc7, 0x4c, 0x99, 0x29, -+ 0x01, 0x93, 0x00, 0x1c, 0x21, 0x9f, 0xd0, 0xa9, 0x70, 0xd8, 0x06, 0xac, 0x64, 0x17, 0x57, 0x02, -+ 0xc6, 0x75, 0x68, 0xf8, 0xce, 0x45, 0xb2, 0xa8, 0x7c, 0xa4, 0xee, 0x3b, 0x17, 0x52, 0xf9, 0x3e, -+ 0xd4, 0x9e, 0x38, 0x78, 0xe2, 0x06, 0x4c, 0x79, 0x45, 0x83, 0xa9, 0xc0, 0x6a, 0x56, 0xe0, 0x1f, -+ 0xcb, 0xd0, 0x94, 0x12, 0xa5, 0xc2, 0x1b, 0xb0, 0xe2, 0x3a, 0xee, 0x38, 0x11, 0x29, 0x00, 0xe3, -+ 0x96, 0x56, 0xa4, 0x9c, 0xcd, 0x70, 0xa9, 0xa6, 0x5a, 0xb5, 0xdb, 0x00, 0xd1, 0x53, 0x27, 0x54, -+ 0xba, 0x55, 0x16, 0x10, 0x37, 0x38, 0x8d, 0x54, 0xf7, 0x1d, 0x68, 0xc9, 0x7d, 0xa7, 0xa6, 0x54, -+ 0x17, 0x4c, 0x69, 0x4a, 0x2a, 0x39, 0xe9, 0x26, 0xb4, 0xe3, 0x08, 0xd9, 0x63, 0x8c, 0xa8, 0x43, -+ 0xdd, 0xf1, 0xb4, 0xbf, 0x22, 0x0f, 0xa0, 0x38, 0x42, 0x07, 0x1a, 0x67, 0xdc, 0x81, 0x15, 0x9e, -+ 0x5b, 0xa2, 0xfe, 0xaa, 0x38, 0xeb, 0x5e, 0xcc, 0xb2, 0x14, 0xa6, 0x0e, 0xc5, 0xef, 0x5e, 0xc0, -+ 0xe8, 0xd4, 0x92, 0xa4, 0x83, 0xf7, 0x00, 0x52, 0xa4, 0xb1, 0x06, 0x95, 0x33, 0x34, 0x55, 0x71, -+ 0xc8, 0x3f, 0xb9, 0x73, 0xce, 0x9d, 0x49, 0xac, 0xbd, 0x2e, 0x81, 0x0f, 0xca, 0xef, 0x95, 0x4c, -+ 0x17, 0xba, 0x3b, 0x93, 0x33, 0x4c, 0x32, 0xd3, 0x37, 0x60, 0xc5, 0x77, 0xbe, 0x22, 0x54, 0x7b, -+ 0x52, 0x00, 0x02, 0x8b, 0x03, 0x42, 0x35, 0x0b, 0x01, 0x18, 0x1d, 0x28, 0x93, 0x50, 0xf8, 0xab, -+ 0x61, 0x95, 0x49, 0x98, 0x0a, 0xaa, 0x66, 0x04, 0x99, 0x7f, 0xab, 0x02, 0xa4, 0x52, 0x0c, 0x0b, -+ 0x06, 0x98, 0xd8, 0x11, 0xa2, 0xfc, 0x7c, 0xb7, 0x4f, 0xa7, 0x0c, 0x45, 0x36, 0x45, 0x6e, 0x4c, -+ 0x23, 0x7c, 0xce, 0xd7, 0x8f, 0x9b, 0x7d, 0x45, 0x9a, 0x3d, 0xa3, 0x9b, 0x75, 0x15, 0x93, 0x13, -+ 0x39, 0x6f, 0x87, 0x4f, 0xb3, 0xf4, 0x2c, 0xe3, 0x10, 0xae, 0xa4, 0x3c, 0xbd, 0x0c, 0xbb, 0xf2, -+ 0x32, 0x76, 0xeb, 0x09, 0x3b, 0x2f, 0x65, 0xb5, 0x07, 0xeb, 0x98, 0xd8, 0x5f, 0xc7, 0x28, 0xce, -+ 0x31, 0xaa, 0x2c, 0x63, 0xd4, 0xc3, 0xe4, 0x0b, 0x31, 0x21, 0x65, 0x73, 0x0c, 0xd7, 0x32, 0x56, -+ 0xf2, 0x70, 0xcf, 0x30, 0xab, 0x2e, 0x63, 0xb6, 0x99, 0x68, 0xc5, 0xf3, 0x41, 0xca, 0xf1, 0x13, -+ 0xd8, 0xc4, 0xc4, 0x7e, 0xea, 0x60, 0x36, 0xcb, 0x6e, 0xe5, 0x7b, 0x8c, 0xe4, 0x27, 0x5a, 0x9e, -+ 0x97, 0x34, 0xd2, 0x47, 0x74, 0x94, 0x33, 0x72, 0xf5, 0x7b, 0x8c, 0x3c, 0x12, 0x13, 0x52, 0x36, -+ 0xf7, 0xa1, 0x87, 0xc9, 0xac, 0x36, 0xb5, 0x65, 0x4c, 0xba, 0x98, 0xe4, 0x35, 0xd9, 0x81, 0x5e, -+ 0x84, 0x5c, 0x46, 0x68, 0x76, 0x13, 0xd4, 0x97, 0xb1, 0x58, 0x53, 0xf4, 0x09, 0x0f, 0xf3, 0x47, -+ 0xd0, 0x3a, 0x88, 0x47, 0x88, 0x4d, 0x4e, 0x93, 0x64, 0xf0, 0xdc, 0xf2, 0x8f, 0xf9, 0x8f, 0x32, -+ 0x34, 0x77, 0x47, 0x94, 0xc4, 0x61, 0x2e, 0x27, 0xcb, 0x20, 0x9d, 0xcd, 0xc9, 0x82, 0x44, 0xe4, -+ 0x64, 0x49, 0xfc, 0x2e, 0xb4, 0x7c, 0x11, 0xba, 0x8a, 0x5e, 0xe6, 0xa1, 0xde, 0x5c, 0x50, 0x5b, -+ 0x4d, 0x3f, 0x93, 0xcc, 0x86, 0x00, 0x21, 0xf6, 0x22, 0x35, 0x47, 0xa6, 0xa3, 0xae, 0x2a, 0xb7, -+ 0x74, 0x8a, 0xb6, 0x1a, 0x61, 0x92, 0xad, 0xdf, 0x86, 0xe6, 0x29, 0x77, 0x92, 0x9a, 0x90, 0x4b, -+ 0x46, 0xa9, 0xf7, 0x2c, 0x38, 0x4d, 0x83, 0xf0, 0x00, 0xda, 0x63, 0xe9, 0x32, 0x35, 0x49, 0xee, -+ 0xa1, 0x9b, 0xca, 0x92, 0xd4, 0xde, 0x61, 0xd6, 0xb3, 0x72, 0x01, 0x5a, 0xe3, 0x0c, 0x6a, 0x70, -+ 0x02, 0xbd, 0x39, 0x92, 0x82, 0x1c, 0xb4, 0x9d, 0xcd, 0x41, 0xcd, 0x3b, 0x86, 0x14, 0x94, 0x9d, -+ 0x99, 0xcd, 0x4b, 0xbf, 0x2a, 0x43, 0xeb, 0x73, 0xc4, 0x9e, 0x12, 0x7a, 0x26, 0xf5, 0x35, 0xa0, -+ 0x1a, 0x38, 0x3e, 0x52, 0x1c, 0xc5, 0xb7, 0x71, 0x0d, 0xea, 0xf4, 0x42, 0x26, 0x10, 0xb5, 0x9e, -+ 0x35, 0x7a, 0x21, 0x12, 0x83, 0xf1, 0x12, 0x00, 0xbd, 0xb0, 0x43, 0xc7, 0x3d, 0x43, 0xca, 0x83, -+ 0x55, 0xab, 0x41, 0x2f, 0x8e, 0x25, 0x82, 0x6f, 0x05, 0x7a, 0x61, 0x23, 0x4a, 0x09, 0x8d, 0x54, -+ 0xae, 0xaa, 0xd3, 0x8b, 0x3d, 0x01, 0xab, 0xb9, 0x1e, 0x25, 0x61, 0x88, 0x3c, 0x91, 0xa3, 0xc5, -+ 0xdc, 0x07, 0x12, 0xc1, 0xa5, 0x32, 0x2d, 0x75, 0x55, 0x4a, 0x65, 0xa9, 0x54, 0x96, 0x4a, 0xad, -+ 0xc9, 0x99, 0x2c, 0x2b, 0x95, 0x25, 0x52, 0xeb, 0x52, 0x2a, 0xcb, 0x48, 0x65, 0xa9, 0xd4, 0x86, -+ 0x9e, 0xab, 0xa4, 0x9a, 0xbf, 0x2c, 0xc1, 0xe6, 0x6c, 0xe1, 0xa7, 0x6a, 0xd3, 0x77, 0xa1, 0xe5, -+ 0x8a, 0xf5, 0xca, 0xed, 0xc9, 0xde, 0xdc, 0x4a, 0x5a, 0x4d, 0x37, 0xb3, 0x8d, 0xef, 0x42, 0x3b, -+ 0x90, 0x0e, 0x4e, 0xb6, 0x66, 0x25, 0x5d, 0x97, 0xac, 0xef, 0xad, 0x56, 0x90, 0x81, 0x4c, 0x0f, -+ 0x8c, 0x2f, 0x29, 0x66, 0xe8, 0x84, 0x51, 0xe4, 0xf8, 0xcf, 0xa3, 0xba, 0x37, 0xa0, 0x2a, 0xaa, -+ 0x15, 0xbe, 0x4c, 0x2d, 0x4b, 0x7c, 0x9b, 0xaf, 0xc1, 0x7a, 0x4e, 0x8a, 0xb2, 0x75, 0x0d, 0x2a, -+ 0x13, 0x14, 0x08, 0xee, 0x6d, 0x8b, 0x7f, 0x9a, 0x0e, 0xf4, 0x2c, 0xe4, 0x78, 0xcf, 0x4f, 0x1b, -+ 0x25, 0xa2, 0x92, 0x8a, 0xd8, 0x06, 0x23, 0x2b, 0x42, 0xa9, 0xa2, 0xb5, 0x2e, 0x65, 0xb4, 0x7e, -+ 0x08, 0xbd, 0xdd, 0x09, 0x89, 0xd0, 0x09, 0xf3, 0x70, 0xf0, 0x3c, 0xda, 0x91, 0x9f, 0xc2, 0xfa, -+ 0x23, 0x36, 0xfd, 0x92, 0x33, 0x8b, 0xf0, 0x37, 0xe8, 0x39, 0xd9, 0x47, 0xc9, 0x53, 0x6d, 0x1f, -+ 0x25, 0x4f, 0x79, 0x73, 0xe3, 0x92, 0x49, 0xec, 0x07, 0x22, 0x14, 0xda, 0x96, 0x82, 0xcc, 0x1d, -+ 0x68, 0xc9, 0x1a, 0xfa, 0x88, 0x78, 0xf1, 0x04, 0x15, 0xc6, 0xe0, 0x16, 0x40, 0xe8, 0x50, 0xc7, -+ 0x47, 0x0c, 0x51, 0xb9, 0x87, 0x1a, 0x56, 0x06, 0x63, 0xfe, 0xba, 0x0c, 0x1b, 0xf2, 0xbe, 0xe1, -+ 0x44, 0xb6, 0xd9, 0xda, 0x84, 0x01, 0xd4, 0xc7, 0x24, 0x62, 0x19, 0x86, 0x09, 0xcc, 0x55, 0xe4, -+ 0xfd, 0xb9, 0xe4, 0xc6, 0x3f, 0x73, 0x97, 0x00, 0x95, 0xe5, 0x97, 0x00, 0x73, 0x6d, 0x7e, 0x75, -+ 0xbe, 0xcd, 0xe7, 0xd1, 0xa6, 0x89, 0xb0, 0x8c, 0xf1, 0x86, 0xd5, 0x50, 0x98, 0x43, 0xcf, 0xb8, -+ 0x05, 0xdd, 0x11, 0xd7, 0xd2, 0x1e, 0x13, 0x72, 0x66, 0x87, 0x0e, 0x1b, 0x8b, 0x50, 0x6f, 0x58, -+ 0x6d, 0x81, 0x3e, 0x20, 0xe4, 0xec, 0xd8, 0x61, 0x63, 0xe3, 0x7d, 0xe8, 0xa8, 0x32, 0xd0, 0x17, -+ 0x2e, 0x8a, 0xd4, 0xe1, 0xa7, 0xa2, 0x28, 0xeb, 0x3d, 0xab, 0x7d, 0x96, 0x81, 0x22, 0xf3, 0x2a, -+ 0x5c, 0x79, 0x80, 0x22, 0x46, 0xc9, 0x34, 0xef, 0x18, 0xf3, 0xff, 0x00, 0x0e, 0x03, 0x86, 0xe8, -+ 0x13, 0xc7, 0x45, 0x91, 0xf1, 0x56, 0x16, 0x52, 0xc5, 0xd1, 0xda, 0x50, 0x5e, 0xf7, 0x24, 0x03, -+ 0x56, 0x86, 0xc6, 0x1c, 0xc2, 0xaa, 0x45, 0x62, 0x9e, 0x8e, 0x5e, 0xd1, 0x5f, 0x6a, 0x5e, 0x4b, -+ 0xcd, 0x13, 0x48, 0x4b, 0x8d, 0x99, 0x07, 0xba, 0x85, 0x4d, 0xd9, 0xa9, 0x25, 0x1a, 0x42, 0x03, -+ 0x6b, 0x9c, 0xca, 0x2a, 0xf3, 0xa2, 0x53, 0x12, 0xf3, 0x11, 0x98, 0x33, 0x9c, 0x0e, 0x9e, 0xde, -+ 0xf7, 0x3c, 0xba, 0x33, 0xfd, 0xdc, 0xf1, 0x2f, 0xcd, 0xf5, 0x43, 0x58, 0x97, 0x5c, 0xa5, 0xbe, -+ 0x9a, 0xcd, 0x2b, 0xb0, 0x4a, 0xb5, 0x71, 0xa5, 0xf4, 0xf6, 0x48, 0x11, 0xa9, 0x31, 0xee, 0xe5, -+ 0xcf, 0x70, 0xc4, 0x52, 0xf7, 0x68, 0x2f, 0xaf, 0x43, 0x8f, 0x0f, 0xe4, 0x78, 0x9a, 0x1f, 0x43, -+ 0xeb, 0xbe, 0x75, 0xfc, 0x39, 0xc2, 0xa3, 0xf1, 0x29, 0xcf, 0xc9, 0xff, 0x9b, 0x87, 0x95, 0x1b, -+ 0x0d, 0xa5, 0x6d, 0x66, 0xc8, 0xca, 0xd1, 0x99, 0x9f, 0xc0, 0xe6, 0x7d, 0xcf, 0xcb, 0xa2, 0xb4, -+ 0xd6, 0x6f, 0x41, 0x23, 0xc8, 0xb0, 0xcb, 0x9c, 0x84, 0x39, 0xea, 0x94, 0xc8, 0xfc, 0x31, 0xac, -+ 0x3f, 0x0c, 0x26, 0x38, 0x40, 0xbb, 0xc7, 0x8f, 0x8f, 0x50, 0x92, 0xe1, 0x0c, 0xa8, 0xf2, 0x4a, -+ 0x50, 0xf0, 0xa8, 0x5b, 0xe2, 0x9b, 0x87, 0x7c, 0x70, 0x6a, 0xbb, 0x61, 0x1c, 0xa9, 0x2b, 0xa4, -+ 0xd5, 0xe0, 0x74, 0x37, 0x8c, 0x23, 0x7e, 0x64, 0xf1, 0x92, 0x85, 0x04, 0x93, 0xa9, 0x88, 0xfb, -+ 0xba, 0x55, 0x73, 0xc3, 0xf8, 0x61, 0x30, 0x99, 0x9a, 0xff, 0x23, 0xfa, 0x7a, 0x84, 0x3c, 0xcb, -+ 0x09, 0x3c, 0xe2, 0x3f, 0x40, 0xe7, 0x19, 0x09, 0x49, 0x0f, 0xa9, 0xf3, 0xdb, 0xb7, 0x25, 0x68, -+ 0xdd, 0x1f, 0xa1, 0x80, 0x3d, 0x40, 0xcc, 0xc1, 0x13, 0xd1, 0x27, 0x9e, 0x23, 0x1a, 0x61, 0x12, -+ 0xa8, 0x20, 0xd6, 0x20, 0x6f, 0xf3, 0x71, 0x80, 0x99, 0xed, 0x39, 0xc8, 0x27, 0x81, 0xe0, 0x52, -+ 0xb7, 0x80, 0xa3, 0x1e, 0x08, 0x8c, 0xf1, 0x1a, 0x74, 0xe5, 0x15, 0x9f, 0x3d, 0x76, 0x02, 0x6f, -+ 0xc2, 0xd3, 0x47, 0x45, 0x04, 0x7c, 0x47, 0xa2, 0x0f, 0x14, 0xd6, 0x78, 0x1d, 0xd6, 0x54, 0x70, -+ 0xa7, 0x94, 0x55, 0x41, 0xd9, 0x55, 0xf8, 0x1c, 0x69, 0x1c, 0x86, 0x84, 0xb2, 0xc8, 0x8e, 0x90, -+ 0xeb, 0x12, 0x3f, 0x54, 0x4d, 0x56, 0x57, 0xe3, 0x4f, 0x24, 0xda, 0x1c, 0xc1, 0xfa, 0x3e, 0xb7, -+ 0x53, 0x59, 0x92, 0x6e, 0xab, 0x8e, 0x8f, 0x7c, 0xfb, 0x74, 0x42, 0xdc, 0x33, 0x9b, 0xa7, 0x5c, -+ 0xe5, 0x61, 0x5e, 0xc6, 0xed, 0x70, 0xe4, 0x09, 0xfe, 0x46, 0xdc, 0x27, 0x70, 0xaa, 0x31, 0x61, -+ 0xe1, 0x24, 0x1e, 0xd9, 0x21, 0x25, 0xa7, 0x48, 0x99, 0xd8, 0xf5, 0x91, 0x7f, 0x20, 0xf1, 0xc7, -+ 0x1c, 0x6d, 0xfe, 0xa1, 0x04, 0x1b, 0x79, 0x49, 0xea, 0x00, 0xb9, 0x0d, 0x1b, 0x79, 0x51, 0xaa, -+ 0xa8, 0x90, 0x45, 0x6b, 0x2f, 0x2b, 0x50, 0x96, 0x17, 0x77, 0xa1, 0x2d, 0x6e, 0x81, 0x6d, 0x4f, -+ 0x72, 0xca, 0x97, 0x52, 0xd9, 0x75, 0xb1, 0x5a, 0x4e, 0x76, 0x95, 0xde, 0x87, 0x6b, 0xca, 0x7c, -+ 0x7b, 0x5e, 0x6d, 0xb9, 0x21, 0x36, 0x15, 0xc1, 0xd1, 0x8c, 0xf6, 0x9f, 0x41, 0x3f, 0x45, 0xed, -+ 0x4c, 0x05, 0x32, 0xdd, 0xcc, 0xeb, 0x33, 0xc6, 0xf2, 0x68, 0x17, 0x51, 0x52, 0xb5, 0x8a, 0x86, -+ 0xcc, 0x7b, 0x70, 0xf5, 0x04, 0x31, 0xe9, 0x0d, 0x87, 0xa9, 0xfe, 0x46, 0x32, 0x5b, 0x83, 0xca, -+ 0x09, 0x72, 0x85, 0xf1, 0x15, 0x8b, 0x7f, 0xf2, 0x0d, 0xf8, 0x38, 0x42, 0xae, 0xb0, 0xb2, 0x62, -+ 0x89, 0x6f, 0xf3, 0xf7, 0x25, 0xa8, 0xa9, 0x94, 0xcf, 0x8f, 0x2d, 0x8f, 0xe2, 0x73, 0x44, 0xd5, -+ 0xd6, 0x53, 0x90, 0xf1, 0x2a, 0x74, 0xe4, 0x97, 0x4d, 0x42, 0x86, 0x49, 0x72, 0x90, 0xb4, 0x25, -+ 0xf6, 0xa1, 0x44, 0x8a, 0x2b, 0x3d, 0x71, 0xa9, 0xa6, 0xfa, 0x57, 0x05, 0x71, 0xfc, 0x93, 0x88, -+ 0x47, 0xb8, 0x38, 0x38, 0x1a, 0x96, 0x82, 0xf8, 0x56, 0xd7, 0xfc, 0x56, 0x04, 0x3f, 0x0d, 0xf2, -+ 0xad, 0xee, 0x93, 0x38, 0x60, 0x76, 0x48, 0x70, 0xc0, 0xd4, 0x49, 0x01, 0x02, 0x75, 0xcc, 0x31, -+ 0xe6, 0x2f, 0x4a, 0xb0, 0x2a, 0xaf, 0xb5, 0x79, 0xc7, 0x9c, 0x9c, 0xd7, 0x65, 0x2c, 0x6a, 0x1f, -+ 0x21, 0x4b, 0x9e, 0xd1, 0xe2, 0x9b, 0xc7, 0xf1, 0xb9, 0x2f, 0x4f, 0x1d, 0xa5, 0xda, 0xb9, 0x2f, -+ 0x8e, 0x9b, 0x57, 0xa1, 0x93, 0x1e, 0xfb, 0x62, 0x5c, 0xaa, 0xd8, 0x4e, 0xb0, 0x82, 0x6c, 0xa1, -+ 0xa6, 0xe6, 0xff, 0x03, 0xa4, 0xd7, 0xbb, 0xdc, 0xe5, 0x71, 0xa2, 0x0c, 0xff, 0xe4, 0x98, 0x51, -+ 0x52, 0x30, 0xf0, 0x4f, 0xe3, 0x16, 0x74, 0x1c, 0xcf, 0xc3, 0x7c, 0xba, 0x33, 0xd9, 0xc7, 0x5e, -+ 0x12, 0xa4, 0x79, 0xac, 0xf9, 0xa7, 0x12, 0x74, 0x77, 0x49, 0x38, 0xfd, 0x18, 0x4f, 0x50, 0x26, -+ 0x83, 0x08, 0x25, 0x55, 0xbd, 0xc0, 0xbf, 0x79, 0x0d, 0xfc, 0x04, 0x4f, 0x90, 0x0c, 0x2d, 0xb9, -+ 0xb2, 0x75, 0x8e, 0x10, 0x61, 0xa5, 0x07, 0x93, 0xcb, 0xbc, 0xb6, 0x1c, 0x3c, 0x22, 0x9e, 0xa8, -+ 0xf6, 0x3d, 0x4c, 0xed, 0xe4, 0xea, 0xae, 0x6d, 0xd5, 0x3c, 0x4c, 0xc5, 0x90, 0x32, 0x64, 0x45, -+ 0x5c, 0xcd, 0x66, 0x0d, 0x59, 0x95, 0x18, 0x6e, 0xc8, 0x26, 0xac, 0x92, 0x27, 0x4f, 0x22, 0xc4, -+ 0x44, 0x5d, 0x5e, 0xb1, 0x14, 0x94, 0xa4, 0xb9, 0x7a, 0x26, 0xcd, 0x5d, 0x81, 0x75, 0xf1, 0x08, -+ 0xf0, 0x88, 0x3a, 0x2e, 0x0e, 0x46, 0xfa, 0x78, 0xd8, 0x00, 0xe3, 0x84, 0x91, 0x70, 0x1e, 0xbb, -+ 0x8f, 0xd8, 0xc3, 0x87, 0x47, 0x7b, 0xe7, 0x28, 0x60, 0x1a, 0xfb, 0x26, 0xd4, 0x35, 0xea, 0x5f, -+ 0xb9, 0x21, 0x5d, 0x87, 0xde, 0x3e, 0x62, 0x47, 0x88, 0x51, 0xec, 0x26, 0xc7, 0xd1, 0x4d, 0xa8, -+ 0x29, 0x0c, 0x5f, 0x52, 0x5f, 0x7e, 0xea, 0x3c, 0xab, 0xc0, 0x3b, 0x7f, 0xed, 0xa9, 0x94, 0xac, -+ 0xee, 0x0c, 0x8c, 0x7d, 0xe8, 0xce, 0x3c, 0xf0, 0x18, 0xea, 0x12, 0xa9, 0xf8, 0xdd, 0x67, 0xb0, -+ 0x39, 0x94, 0x0f, 0x46, 0x43, 0xfd, 0x60, 0x34, 0xdc, 0xf3, 0x43, 0x36, 0x35, 0xf6, 0xa0, 0x93, -+ 0x7f, 0x0a, 0x31, 0xae, 0xeb, 0x9a, 0xab, 0xe0, 0x81, 0x64, 0x21, 0x9b, 0x7d, 0xe8, 0xce, 0xbc, -+ 0x8a, 0x68, 0x7d, 0x8a, 0x1f, 0x4b, 0x16, 0x32, 0xba, 0x07, 0xcd, 0xcc, 0x33, 0x88, 0xd1, 0x97, -+ 0x4c, 0xe6, 0x5f, 0x46, 0x16, 0x32, 0xd8, 0x85, 0x76, 0xee, 0x65, 0xc2, 0x18, 0x28, 0x7b, 0x0a, -+ 0x9e, 0x2b, 0x16, 0x32, 0xd9, 0x81, 0x66, 0xe6, 0x81, 0x40, 0x6b, 0x31, 0xff, 0x0a, 0x31, 0xb8, -+ 0x56, 0x30, 0xa2, 0x32, 0xff, 0x3e, 0x74, 0x67, 0x5e, 0x0d, 0xb4, 0x4b, 0x8a, 0x1f, 0x13, 0x16, -+ 0x2a, 0xf3, 0xa9, 0x58, 0xa2, 0x4c, 0x53, 0x98, 0x59, 0xa2, 0xf9, 0x37, 0x82, 0xc1, 0x8b, 0xc5, -+ 0x83, 0x4a, 0xab, 0x3d, 0xe8, 0xe4, 0x9f, 0x07, 0x34, 0xb3, 0xc2, 0x47, 0x83, 0xe5, 0xeb, 0x9d, -+ 0x7b, 0x29, 0x48, 0xd7, 0xbb, 0xe8, 0x01, 0x61, 0x21, 0xa3, 0xfb, 0x00, 0xaa, 0x05, 0xf4, 0x70, -+ 0x90, 0x38, 0x7a, 0xae, 0xf5, 0x4c, 0x1c, 0x5d, 0xd0, 0x2e, 0xde, 0x03, 0x90, 0x9d, 0x9b, 0x47, -+ 0x62, 0x66, 0x5c, 0xd5, 0x6a, 0xcc, 0xb4, 0x8b, 0x83, 0xfe, 0xfc, 0xc0, 0x1c, 0x03, 0x44, 0xe9, -+ 0x65, 0x18, 0x7c, 0x04, 0x90, 0x76, 0x84, 0x9a, 0xc1, 0x5c, 0x8f, 0xb8, 0xc4, 0x07, 0xad, 0x6c, -+ 0xff, 0x67, 0x28, 0x5b, 0x0b, 0x7a, 0xc2, 0x25, 0x2c, 0xba, 0x33, 0x55, 0x79, 0x7e, 0xb3, 0xcd, -+ 0x96, 0xfd, 0x83, 0xb9, 0x6a, 0xdc, 0xf8, 0x01, 0x5c, 0x5f, 0x52, 0xd8, 0x1b, 0xdb, 0x85, 0xec, -+ 0x0a, 0x6a, 0xff, 0x02, 0xd6, 0x77, 0xa1, 0x95, 0xad, 0xee, 0xb5, 0x81, 0x05, 0x15, 0xff, 0x20, -+ 0x57, 0xe1, 0x1b, 0xf7, 0xa0, 0x93, 0xaf, 0xec, 0xf5, 0x6e, 0x2d, 0xac, 0xf7, 0x07, 0xea, 0x36, -+ 0x2c, 0x43, 0xfe, 0x0e, 0x40, 0xda, 0x01, 0xe8, 0x95, 0x99, 0xeb, 0x09, 0x66, 0xa4, 0xee, 0x43, -+ 0x77, 0xa6, 0xb2, 0xd7, 0xce, 0x2c, 0x2e, 0xf8, 0x97, 0x2d, 0x6c, 0xf6, 0x88, 0xd1, 0x76, 0x17, -+ 0x1c, 0x3b, 0xcb, 0xf2, 0x61, 0xe6, 0x38, 0xd2, 0x01, 0x32, 0x7f, 0x42, 0x2d, 0x64, 0xf0, 0x2e, -+ 0x40, 0x7a, 0xe8, 0x68, 0x0f, 0xcc, 0x1d, 0x43, 0x83, 0xb6, 0xbe, 0xad, 0x94, 0x74, 0xbb, 0xd0, -+ 0xce, 0x35, 0xf4, 0x3a, 0x8b, 0x16, 0x75, 0xf9, 0xcb, 0xce, 0x96, 0x7c, 0xf7, 0xab, 0x57, 0xaf, -+ 0xb0, 0x27, 0x5e, 0xe6, 0xc5, 0x6c, 0x73, 0xa4, 0xbd, 0x58, 0xd0, 0x30, 0x7d, 0x4f, 0xba, 0xca, -+ 0x36, 0x40, 0x99, 0x74, 0x55, 0xd0, 0x17, 0x2d, 0x64, 0x74, 0x00, 0xdd, 0x7d, 0x5d, 0xdb, 0xaa, -+ 0xba, 0x5b, 0xa9, 0x53, 0xd0, 0x67, 0x0c, 0x06, 0x45, 0x43, 0x2a, 0x67, 0x7c, 0x0a, 0xbd, 0xb9, -+ 0x9a, 0xdb, 0xd8, 0x4a, 0xee, 0x8c, 0x0b, 0x8b, 0xf1, 0x85, 0x6a, 0x1d, 0xc2, 0xda, 0x6c, 0xc9, -+ 0x6d, 0xbc, 0xa4, 0xb6, 0x4a, 0x71, 0x29, 0xbe, 0x90, 0xd5, 0xfb, 0x50, 0xd7, 0x25, 0x9e, 0xa1, -+ 0xee, 0xe6, 0x67, 0x4a, 0xbe, 0x85, 0x53, 0xef, 0x42, 0x33, 0x53, 0x24, 0xe9, 0xbd, 0x3a, 0x5f, -+ 0x37, 0x0d, 0xd4, 0x55, 0xba, 0x46, 0xef, 0x5c, 0x7c, 0xfb, 0xdd, 0xd6, 0x0b, 0x7f, 0xf9, 0x6e, -+ 0xeb, 0x85, 0x9f, 0x3f, 0xdb, 0x2a, 0x7d, 0xfb, 0x6c, 0xab, 0xf4, 0xe7, 0x67, 0x5b, 0xa5, 0xbf, -+ 0x3f, 0xdb, 0x2a, 0xfd, 0xf0, 0x27, 0xff, 0xe6, 0xff, 0x6a, 0x68, 0x1c, 0x30, 0xec, 0xa3, 0xdb, -+ 0xe7, 0x98, 0xb2, 0xcc, 0x50, 0x78, 0x36, 0x9a, 0xfb, 0xcb, 0x0d, 0x57, 0xe1, 0x74, 0x55, 0xc0, -+ 0xef, 0xfc, 0x33, 0x00, 0x00, 0xff, 0xff, 0x45, 0x71, 0x2f, 0x90, 0xc0, 0x23, 0x00, 0x00, - } - - func (m *CreateContainerRequest) Marshal() (dAtA []byte, err error) { -@@ -4544,6 +4586,45 @@ func (m *UpdateInterfaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) - return len(dAtA) - i, nil - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Marshal() (dAtA []byte, err error) { -+ size := m.Size() -+ dAtA = make([]byte, size) -+ n, err := m.MarshalToSizedBuffer(dAtA[:size]) -+ if err != nil { -+ return nil, err -+ } -+ return dAtA[:n], nil -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) MarshalTo(dAtA []byte) (int, error) { -+ size := m.Size() -+ return m.MarshalToSizedBuffer(dAtA[:size]) -+} -+ -+func (m *UpdateInterfaceHwAddrByNameRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { -+ i := len(dAtA) -+ _ = i -+ var l int -+ _ = l -+ if m.XXX_unrecognized != nil { -+ i -= len(m.XXX_unrecognized) -+ copy(dAtA[i:], m.XXX_unrecognized) -+ } -+ if m.Interface != nil { -+ { -+ size, err := m.Interface.MarshalToSizedBuffer(dAtA[:i]) -+ if err != nil { -+ return 0, err -+ } -+ i -= size -+ i = encodeVarintAgent(dAtA, i, uint64(size)) -+ } -+ i-- -+ dAtA[i] = 0xa -+ } -+ return len(dAtA) - i, nil -+} -+ - func (m *UpdateRoutesRequest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) -@@ -5001,20 +5082,20 @@ func (m *MemHotplugByProbeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error - copy(dAtA[i:], m.XXX_unrecognized) - } - if len(m.MemHotplugProbeAddr) > 0 { -- dAtA24 := make([]byte, len(m.MemHotplugProbeAddr)*10) -- var j23 int -+ dAtA25 := make([]byte, len(m.MemHotplugProbeAddr)*10) -+ var j24 int - for _, num := range m.MemHotplugProbeAddr { - for num >= 1<<7 { -- dAtA24[j23] = uint8(uint64(num)&0x7f | 0x80) -+ dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 -- j23++ -+ j24++ - } -- dAtA24[j23] = uint8(num) -- j23++ -+ dAtA25[j24] = uint8(num) -+ j24++ - } -- i -= j23 -- copy(dAtA[i:], dAtA24[:j23]) -- i = encodeVarintAgent(dAtA, i, uint64(j23)) -+ i -= j24 -+ copy(dAtA[i:], dAtA25[:j24]) -+ i = encodeVarintAgent(dAtA, i, uint64(j24)) - i-- - dAtA[i] = 0xa - } -@@ -6348,6 +6429,22 @@ func (m *UpdateInterfaceRequest) Size() (n int) { - return n - } - -+func (m *UpdateInterfaceHwAddrByNameRequest) Size() (n int) { -+ if m == nil { -+ return 0 -+ } -+ var l int -+ _ = l -+ if m.Interface != nil { -+ l = m.Interface.Size() -+ n += 1 + l + sovAgent(uint64(l)) -+ } -+ if m.XXX_unrecognized != nil { -+ n += len(m.XXX_unrecognized) -+ } -+ return n -+} -+ - func (m *UpdateRoutesRequest) Size() (n int) { - if m == nil { - return 0 -@@ -7009,7 +7106,7 @@ func (this *MemoryStats) String() string { - return "nil" - } - keysForStats := make([]string, 0, len(this.Stats)) -- for k := range this.Stats { -+ for k, _ := range this.Stats { - keysForStats = append(keysForStats, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForStats) -@@ -7120,7 +7217,7 @@ func (this *CgroupStats) String() string { - return "nil" - } - keysForHugetlbStats := make([]string, 0, len(this.HugetlbStats)) -- for k := range this.HugetlbStats { -+ for k, _ := range this.HugetlbStats { - keysForHugetlbStats = append(keysForHugetlbStats, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForHugetlbStats) -@@ -7342,6 +7439,17 @@ func (this *UpdateInterfaceRequest) String() string { - }, "") - return s - } -+func (this *UpdateInterfaceHwAddrByNameRequest) String() string { -+ if this == nil { -+ return "nil" -+ } -+ s := strings.Join([]string{`&UpdateInterfaceHwAddrByNameRequest{`, -+ `Interface:` + strings.Replace(fmt.Sprintf("%v", this.Interface), "Interface", "protocols.Interface", 1) + `,`, -+ `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, -+ `}`, -+ }, "") -+ return s -+} - func (this *UpdateRoutesRequest) String() string { - if this == nil { - return "nil" -@@ -7637,6 +7745,7 @@ type AgentServiceService interface { - CloseStdin(ctx context.Context, req *CloseStdinRequest) (*types.Empty, error) - TtyWinResize(ctx context.Context, req *TtyWinResizeRequest) (*types.Empty, error) - UpdateInterface(ctx context.Context, req *UpdateInterfaceRequest) (*protocols.Interface, error) -+ UpdateInterfaceHwAddrByName(ctx context.Context, req *UpdateInterfaceHwAddrByNameRequest) (*protocols.Interface, error) - UpdateRoutes(ctx context.Context, req *UpdateRoutesRequest) (*Routes, error) - ListInterfaces(ctx context.Context, req *ListInterfacesRequest) (*Interfaces, error) - ListRoutes(ctx context.Context, req *ListRoutesRequest) (*Routes, error) -@@ -7769,6 +7878,13 @@ func RegisterAgentServiceService(srv *github_com_containerd_ttrpc.Server, svc Ag - } - return svc.UpdateInterface(ctx, &req) - }, -+ "UpdateInterfaceHwAddrByName": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { -+ var req UpdateInterfaceHwAddrByNameRequest -+ if err := unmarshal(&req); err != nil { -+ return nil, err -+ } -+ return svc.UpdateInterfaceHwAddrByName(ctx, &req) -+ }, - "UpdateRoutes": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { - var req UpdateRoutesRequest - if err := unmarshal(&req); err != nil { -@@ -8022,6 +8138,14 @@ func (c *agentServiceClient) UpdateInterface(ctx context.Context, req *UpdateInt - return &resp, nil - } - -+func (c *agentServiceClient) UpdateInterfaceHwAddrByName(ctx context.Context, req *UpdateInterfaceHwAddrByNameRequest) (*protocols.Interface, error) { -+ var resp protocols.Interface -+ if err := c.client.Call(ctx, "grpc.AgentService", "UpdateInterfaceHwAddrByName", req, &resp); err != nil { -+ return nil, err -+ } -+ return &resp, nil -+} -+ - func (c *agentServiceClient) UpdateRoutes(ctx context.Context, req *UpdateRoutesRequest) (*Routes, error) { - var resp Routes - if err := c.client.Call(ctx, "grpc.AgentService", "UpdateRoutes", req, &resp); err != nil { -@@ -8408,10 +8532,7 @@ func (m *CreateContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -8494,10 +8615,7 @@ func (m *StartContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -8599,10 +8717,7 @@ func (m *RemoveContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -8789,10 +8904,7 @@ func (m *ExecProcessRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -8926,10 +9038,7 @@ func (m *SignalProcessRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9044,10 +9153,7 @@ func (m *WaitProcessRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9117,10 +9223,7 @@ func (m *WaitProcessResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9239,10 +9342,7 @@ func (m *UpdateContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9325,10 +9425,7 @@ func (m *StatsContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9411,10 +9508,7 @@ func (m *PauseContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9497,10 +9591,7 @@ func (m *ResumeContainerRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9684,10 +9775,7 @@ func (m *CpuUsage) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9795,10 +9883,7 @@ func (m *ThrottlingData) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -9921,10 +10006,7 @@ func (m *CpuStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -10013,10 +10095,7 @@ func (m *PidsStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -10143,10 +10222,7 @@ func (m *MemoryData) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -10440,7 +10516,7 @@ func (m *MemoryStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > postIndex { -@@ -10457,10 +10533,7 @@ func (m *MemoryStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -10600,10 +10673,7 @@ func (m *BlkioStatsEntry) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -10926,10 +10996,7 @@ func (m *BlkioStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11037,10 +11104,7 @@ func (m *HugetlbStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11347,7 +11411,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > postIndex { -@@ -11364,10 +11428,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11602,10 +11663,7 @@ func (m *NetworkStats) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11726,10 +11784,7 @@ func (m *StatsContainerResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11878,10 +11933,7 @@ func (m *WriteStreamRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -11951,10 +12003,7 @@ func (m *WriteStreamResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12088,10 +12137,7 @@ func (m *ReadStreamRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12176,10 +12222,7 @@ func (m *ReadStreamResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12294,10 +12337,7 @@ func (m *CloseStdinRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12450,10 +12490,7 @@ func (m *TtyWinResizeRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12568,10 +12605,7 @@ func (m *KernelModule) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12838,10 +12872,7 @@ func (m *CreateSandboxRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12892,10 +12923,7 @@ func (m *DestroySandboxRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -12980,10 +13008,7 @@ func (m *Interfaces) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13068,10 +13093,7 @@ func (m *Routes) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13158,10 +13180,94 @@ func (m *UpdateInterfaceRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { -+ return ErrInvalidLengthAgent -+ } -+ if (iNdEx + skippy) > l { -+ return io.ErrUnexpectedEOF -+ } -+ m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) -+ iNdEx += skippy -+ } -+ } -+ -+ if iNdEx > l { -+ return io.ErrUnexpectedEOF -+ } -+ return nil -+} -+func (m *UpdateInterfaceHwAddrByNameRequest) Unmarshal(dAtA []byte) error { -+ l := len(dAtA) -+ iNdEx := 0 -+ for iNdEx < l { -+ preIndex := iNdEx -+ var wire uint64 -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ wire |= uint64(b&0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ fieldNum := int32(wire >> 3) -+ wireType := int(wire & 0x7) -+ if wireType == 4 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: wiretype end group for non-group") -+ } -+ if fieldNum <= 0 { -+ return fmt.Errorf("proto: UpdateInterfaceHwAddrByNameRequest: illegal tag %d (wire type %d)", fieldNum, wire) -+ } -+ switch fieldNum { -+ case 1: -+ if wireType != 2 { -+ return fmt.Errorf("proto: wrong wireType = %d for field Interface", wireType) -+ } -+ var msglen int -+ for shift := uint(0); ; shift += 7 { -+ if shift >= 64 { -+ return ErrIntOverflowAgent -+ } -+ if iNdEx >= l { -+ return io.ErrUnexpectedEOF -+ } -+ b := dAtA[iNdEx] -+ iNdEx++ -+ msglen |= int(b&0x7F) << shift -+ if b < 0x80 { -+ break -+ } -+ } -+ if msglen < 0 { -+ return ErrInvalidLengthAgent -+ } -+ postIndex := iNdEx + msglen -+ if postIndex < 0 { - return ErrInvalidLengthAgent - } -- if (iNdEx + skippy) < 0 { -+ if postIndex > l { -+ return io.ErrUnexpectedEOF -+ } -+ if m.Interface == nil { -+ m.Interface = &protocols.Interface{} -+ } -+ if err := m.Interface.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { -+ return err -+ } -+ iNdEx = postIndex -+ default: -+ iNdEx = preIndex -+ skippy, err := skipAgent(dAtA[iNdEx:]) -+ if err != nil { -+ return err -+ } -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13248,10 +13354,7 @@ func (m *UpdateRoutesRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13302,10 +13405,7 @@ func (m *ListInterfacesRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13356,10 +13456,7 @@ func (m *ListRoutesRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13444,10 +13541,7 @@ func (m *ARPNeighbors) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13534,10 +13628,7 @@ func (m *AddARPNeighborsRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13647,10 +13738,7 @@ func (m *OnlineCPUMemRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13735,10 +13823,7 @@ func (m *ReseedRandomDevRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -13925,10 +14010,7 @@ func (m *AgentDetails) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14019,10 +14101,7 @@ func (m *GuestDetailsRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14148,10 +14227,7 @@ func (m *GuestDetailsResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14278,10 +14354,7 @@ func (m *MemHotplugByProbeRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14370,10 +14443,7 @@ func (m *SetGuestDateTimeRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14616,10 +14686,7 @@ func (m *Storage) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14830,10 +14897,7 @@ func (m *Device) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -14980,10 +15044,7 @@ func (m *StringUser) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15214,10 +15275,7 @@ func (m *CopyFileRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15268,10 +15326,7 @@ func (m *StartTracingRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15322,10 +15377,7 @@ func (m *StopTracingRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15376,10 +15428,7 @@ func (m *GetOOMEventRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15462,10 +15511,7 @@ func (m *OOMEvent) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15516,10 +15562,7 @@ func (m *GetMetricsRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -@@ -15602,10 +15645,7 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthAgent -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthAgent - } - if (iNdEx + skippy) > l { -diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go -index 931605fb..7a46e51a 100644 ---- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go -+++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/health.pb.go -@@ -732,10 +732,7 @@ func (m *CheckRequest) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthHealth -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthHealth - } - if (iNdEx + skippy) > l { -@@ -805,10 +802,7 @@ func (m *HealthCheckResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthHealth -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthHealth - } - if (iNdEx + skippy) > l { -@@ -923,10 +917,7 @@ func (m *VersionCheckResponse) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthHealth -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthHealth - } - if (iNdEx + skippy) > l { -diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go -index 4a7cd2d0..2dc8fd75 100644 ---- a/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go -+++ b/src/runtime/virtcontainers/pkg/agent/protocols/grpc/oci.pb.go -@@ -6797,7 +6797,7 @@ func (this *Spec) String() string { - } - repeatedStringForMounts += "}" - keysForAnnotations := make([]string, 0, len(this.Annotations)) -- for k := range this.Annotations { -+ for k, _ := range this.Annotations { - keysForAnnotations = append(keysForAnnotations, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForAnnotations) -@@ -6996,7 +6996,7 @@ func (this *Linux) String() string { - } - repeatedStringForDevices += "}" - keysForSysctl := make([]string, 0, len(this.Sysctl)) -- for k := range this.Sysctl { -+ for k, _ := range this.Sysctl { - keysForSysctl = append(keysForSysctl, k) - } - github_com_gogo_protobuf_sortkeys.Strings(keysForSysctl) -@@ -7717,10 +7717,7 @@ func (m *Spec) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > postIndex { -@@ -7845,10 +7842,7 @@ func (m *Spec) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -8257,10 +8251,7 @@ func (m *Process) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -8349,10 +8340,7 @@ func (m *Box) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -8549,10 +8537,7 @@ func (m *User) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -8763,10 +8748,7 @@ func (m *LinuxCapabilities) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -8887,10 +8869,7 @@ func (m *POSIXRlimit) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -9069,10 +9048,7 @@ func (m *Mount) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -9175,10 +9151,7 @@ func (m *Root) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -9331,10 +9304,7 @@ func (m *Hooks) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -9500,10 +9470,7 @@ func (m *Hook) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -9732,10 +9699,7 @@ func (m *Linux) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > postIndex { -@@ -10088,10 +10052,7 @@ func (m *Linux) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -10174,10 +10135,7 @@ func (m *Windows) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -10260,10 +10218,7 @@ func (m *Solaris) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -10371,10 +10326,7 @@ func (m *LinuxIDMapping) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -10489,10 +10441,7 @@ func (m *LinuxNamespace) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -10702,10 +10651,7 @@ func (m *LinuxDevice) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11004,10 +10950,7 @@ func (m *LinuxResources) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11192,10 +11135,7 @@ func (m *LinuxMemory) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11405,10 +11345,7 @@ func (m *LinuxCPU) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11535,10 +11472,7 @@ func (m *LinuxWeightDevice) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11646,10 +11580,7 @@ func (m *LinuxThrottleDevice) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11908,10 +11839,7 @@ func (m *LinuxBlockIO) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -11981,10 +11909,7 @@ func (m *LinuxPids) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12157,10 +12082,7 @@ func (m *LinuxDeviceCgroup) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12264,10 +12186,7 @@ func (m *LinuxNetwork) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12369,10 +12288,7 @@ func (m *LinuxHugepageLimit) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12474,10 +12390,7 @@ func (m *LinuxInterfacePriority) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12658,10 +12571,7 @@ func (m *LinuxSeccomp) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12801,10 +12711,7 @@ func (m *LinuxSeccompArg) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -12973,10 +12880,7 @@ func (m *LinuxSyscall) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -@@ -13059,10 +12963,7 @@ func (m *LinuxIntelRdt) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthOci -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthOci - } - if (iNdEx + skippy) > l { -diff --git a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go -index 497cb55a..263afa8f 100644 ---- a/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go -+++ b/src/runtime/virtcontainers/pkg/agent/protocols/types.pb.go -@@ -865,10 +865,7 @@ func (m *IPAddress) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthTypes -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { -@@ -1151,10 +1148,7 @@ func (m *Interface) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthTypes -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { -@@ -1352,10 +1346,7 @@ func (m *Route) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthTypes -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { -@@ -1544,10 +1535,7 @@ func (m *ARPNeighbor) Unmarshal(dAtA []byte) error { - if err != nil { - return err - } -- if skippy < 0 { -- return ErrInvalidLengthTypes -- } -- if (iNdEx + skippy) < 0 { -+ if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { -diff --git a/src/runtime/virtcontainers/pkg/mock/mock.go b/src/runtime/virtcontainers/pkg/mock/mock.go -index bec736d6..0cf4a253 100644 ---- a/src/runtime/virtcontainers/pkg/mock/mock.go -+++ b/src/runtime/virtcontainers/pkg/mock/mock.go -@@ -147,6 +147,10 @@ func (p *HybridVSockTTRPCMockImp) UpdateInterface(ctx context.Context, req *pb.U - return &aTypes.Interface{}, nil - } - -+func (p *HybridVSockTTRPCMockImp) UpdateInterfaceHwAddrByName(ctx context.Context, req *pb.UpdateInterfaceHwAddrByNameRequest) (*aTypes.Interface, error) { -+ return &aTypes.Interface{}, nil -+} -+ - func (p *HybridVSockTTRPCMockImp) UpdateRoutes(ctx context.Context, req *pb.UpdateRoutesRequest) (*pb.Routes, error) { - return &pb.Routes{}, nil - } -diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go -index dc4d3bf7..408b23ed 100644 ---- a/src/runtime/virtcontainers/sandbox.go -+++ b/src/runtime/virtcontainers/sandbox.go -@@ -855,6 +855,11 @@ func (s *Sandbox) AddInterface(ctx context.Context, inf *pbTypes.Interface) (*pb - - // Add network for vm - inf.PciPath = endpoint.PciPath().String() -+ -+ // Update MAC address for network -+ if _, err := s.agent.updateInterfaceHwAddrByName(ctx, inf); err != nil { -+ return nil, err -+ } - return s.agent.updateInterface(ctx, inf) - } - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 4fec96d3..0f473e31 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -345,7 +345,7 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { - return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) - } - --func (s *stratovirt) hotplugNet(endpoint Endpoint, op operation) (err error) { -+func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) { - err = s.qmpSetup() - if err != nil { - return err -@@ -464,7 +464,7 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, - - switch devType { - case netDev: -- return nil, s.hotplugNet(devInfo.(Endpoint), addDevice) -+ return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice) - case blockDev: - return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) - default: -@@ -478,7 +478,7 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ - - switch devType { - case netDev: -- return nil, s.hotplugNet(devInfo.(Endpoint), removeDevice) -+ return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice) - case blockDev: - return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) - default: --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0004-configuration-add-configuration-generator-for-hyperv.patch b/patches/0004-configuration-add-configuration-generator-for-hyperv.patch deleted file mode 100644 index 4279ed77daa14acb3e4f826429235451a0d7b043..0000000000000000000000000000000000000000 --- a/patches/0004-configuration-add-configuration-generator-for-hyperv.patch +++ /dev/null @@ -1,542 +0,0 @@ -From 950c0db14a9a9baccefd83e87893d7f40c2bd13d Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Mon, 9 Aug 2021 14:47:19 +0800 -Subject: [PATCH 4/6] configuration: add configuration generator for hypervisor - type stratovirt. - -Signed-off-by: Wei Gao ---- - src/runtime/.gitignore | 1 + - src/runtime/Makefile | 40 +- - src/runtime/arch/amd64-options.mk | 3 + - src/runtime/arch/arm64-options.mk | 3 + - .../config/configuration-stratovirt.toml.in | 356 ++++++++++++++++++ - 5 files changed, 402 insertions(+), 1 deletion(-) - create mode 100644 src/runtime/cli/config/configuration-stratovirt.toml.in - -diff --git a/src/runtime/.gitignore b/src/runtime/.gitignore -index 52b9e4e5..0a630a07 100644 ---- a/src/runtime/.gitignore -+++ b/src/runtime/.gitignore -@@ -10,6 +10,7 @@ coverage.html - /cli/config/configuration-fc.toml - /cli/config/configuration-qemu.toml - /cli/config/configuration-clh.toml -+/cli/config/configuration-stratovirt.toml - /cli/config-generated.go - /cli/containerd-shim-kata-v2/config-generated.go - /cli/coverage.html -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index 4a69c05c..ea2cd296 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -74,6 +74,7 @@ QEMUBINDIR := $(PREFIXDEPS)/bin - CLHBINDIR := $(PREFIXDEPS)/bin - FCBINDIR := $(PREFIXDEPS)/bin - ACRNBINDIR := $(PREFIXDEPS)/bin -+STRATOVIRTBINDIR := $(PREFIXDEPS)/bin - SYSCONFDIR := /etc - LOCALSTATEDIR := /var - -@@ -93,6 +94,7 @@ GENERATED_VARS = \ - CONFIG_QEMU_IN \ - CONFIG_CLH_IN \ - CONFIG_FC_IN \ -+ CONFIG_STRATOVIRT_IN \ - $(USER_VARS) - SCRIPTS += $(COLLECT_SCRIPT) - SCRIPTS_DIR := $(BINDIR) -@@ -116,12 +118,13 @@ HYPERVISOR_ACRN = acrn - HYPERVISOR_FC = firecracker - HYPERVISOR_QEMU = qemu - HYPERVISOR_CLH = cloud-hypervisor -+HYPERVISOR_STRATOVIRT = stratovirt - - # Determines which hypervisor is specified in $(CONFIG_FILE). - DEFAULT_HYPERVISOR ?= $(HYPERVISOR_QEMU) - - # List of hypervisors this build system can generate configuration for. --HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) -+HYPERVISORS := $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH) $(HYPERVISOR_STRATOVIRT) - - QEMUPATH := $(QEMUBINDIR)/$(QEMUCMD) - QEMUVALIDHYPERVISORPATHS := [\"$(QEMUPATH)\"] -@@ -141,6 +144,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"] - ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD) - ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] - -+STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) -+STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] -+ - NETMONCMD := $(BIN_PREFIX)-netmon - NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) - -@@ -267,6 +273,29 @@ ifneq (,$(CLHCMD)) - KERNELPATH_CLH = $(KERNELDIR)/$(KERNEL_NAME_CLH) - endif - -+ifneq (,$(STRATOVIRTCMD)) -+ KNOWN_HYPERVISORS += $(HYPERVISOR_STRATOVIRT) -+ -+ CONFIG_FILE_STRATOVIRT = configuration-stratovirt.toml -+ CONFIG_STRATOVIRT = $(CLI_DIR)/config/$(CONFIG_FILE_STRATOVIRT) -+ CONFIG_STRATOVIRT_IN = $(CONFIG_STRATOVIRT).in -+ -+ CONFIG_PATH_STRATOVIRT = $(abspath $(CONFDIR)/$(CONFIG_FILE_STRATOVIRT)) -+ CONFIG_PATHS += $(CONFIG_PATH_STRATOVIRT) -+ -+ SYSCONFIG_STRATOVIRT = $(abspath $(SYSCONFDIR)/$(CONFIG_FILE_STRATOVIRT)) -+ SYSCONFIG_PATHS += $(SYSCONFIG_STRATOVIRT) -+ -+ CONFIGS += $(CONFIG_STRATOVIRT) -+ -+ # stratovirt-specific options (all should be suffixed by "_STRATOVIRT") -+ DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio -+ DEFNETWORKMODEL_STRATOVIRT := none -+ KENRELTYPE_STRATOVIRT = uncompressed -+ KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT)) -+ KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT) -+endif -+ - ifneq (,$(FCCMD)) - KNOWN_HYPERVISORS += $(HYPERVISOR_FC) - -@@ -363,6 +392,7 @@ USER_VARS += BINDIR - USER_VARS += CONFIG_ACRN_IN - USER_VARS += CONFIG_CLH_IN - USER_VARS += CONFIG_FC_IN -+USER_VARS += CONFIG_STRATOVIRT_IN - USER_VARS += CONFIG_PATH - USER_VARS += CONFIG_QEMU_IN - USER_VARS += DESTDIR -@@ -382,6 +412,8 @@ USER_VARS += FCPATH - USER_VARS += FCVALIDHYPERVISORPATHS - USER_VARS += FCJAILERPATH - USER_VARS += FCVALIDJAILERPATHS -+USER_VARS += STRATOVIRTPATH -+USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS - USER_VARS += SYSCONFIG - USER_VARS += IMAGENAME - USER_VARS += IMAGEPATH -@@ -395,6 +427,7 @@ USER_VARS += KERNELPATH_ACRN - USER_VARS += KERNELPATH - USER_VARS += KERNELPATH_CLH - USER_VARS += KERNELPATH_FC -+USER_VARS += KERNELPATH_STRATOVIRT - USER_VARS += KERNELVIRTIOFSPATH - USER_VARS += FIRMWAREPATH - USER_VARS += MACHINEACCELERATORS -@@ -434,12 +467,14 @@ USER_VARS += DEFNETWORKMODEL_ACRN - USER_VARS += DEFNETWORKMODEL_CLH - USER_VARS += DEFNETWORKMODEL_FC - USER_VARS += DEFNETWORKMODEL_QEMU -+USER_VARS += DEFNETWORKMODEL_STRATOVIRT - USER_VARS += DEFDISABLEGUESTSECCOMP - USER_VARS += DEFAULTEXPFEATURES - USER_VARS += DEFDISABLEBLOCK - USER_VARS += DEFBLOCKSTORAGEDRIVER_ACRN - USER_VARS += DEFBLOCKSTORAGEDRIVER_FC - USER_VARS += DEFBLOCKSTORAGEDRIVER_QEMU -+USER_VARS += DEFBLOCKSTORAGEDRIVER_STRATOVIRT - USER_VARS += DEFSHAREDFS_QEMU_VIRTIOFS - USER_VARS += DEFVIRTIOFSDAEMON - USER_VARS += DEFVALIDVIRTIOFSDAEMONPATHS -@@ -773,6 +808,9 @@ ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS))) - endif - ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS))) - @printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH)) -+endif -+ifneq (,$(findstring $(HYPERVISOR_STRATOVIRT),$(KNOWN_HYPERVISORS))) -+ @printf "\t$(HYPERVISOR_STRATOVIRT) hypervisor path (STRATOVIRTPATH) : %s\n" $(abspath $(STRATOVIRTPATH)) - endif - @printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR)) - @printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR)) -diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk -index 83af8cc0..ff2af9e6 100644 ---- a/src/runtime/arch/amd64-options.mk -+++ b/src/runtime/arch/amd64-options.mk -@@ -23,3 +23,6 @@ ACRNCTLCMD := acrnctl - - # cloud-hypervisor binary name - CLHCMD := cloud-hypervisor -+ -+# stratovirt binary name -+STRATOVIRTCMD := stratovirt -\ No newline at end of file -diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk -index ad5ef5d4..2ad3f657 100644 ---- a/src/runtime/arch/arm64-options.mk -+++ b/src/runtime/arch/arm64-options.mk -@@ -19,3 +19,6 @@ FCJAILERCMD := jailer - - # cloud-hypervisor binary name - CLHCMD := cloud-hypervisor -+ -+# stratovirt binary name -+STRATOVIRTCMD := stratovirt -\ No newline at end of file -diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in -new file mode 100644 -index 00000000..5c83c3c9 ---- /dev/null -+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in -@@ -0,0 +1,356 @@ -+# Copyright (c) 2017-2019 Intel Corporation -+# -+# SPDX-License-Identifier: Apache-2.0 -+# -+ -+# XXX: WARNING: this file is auto-generated. -+# XXX: -+# XXX: Source file: "@CONFIG_STRATOVIRT_IN@" -+# XXX: Project: -+# XXX: Name: @PROJECT_NAME@ -+# XXX: Type: @PROJECT_TYPE@ -+ -+[hypervisor.stratovirt] -+path = "@STRATOVIRTPATH@" -+kernel = "@KERNELPATH_STRATOVIRT@" -+image = "@IMAGEPATH@" -+ -+# List of valid annotation names for the hypervisor -+# Each member of the list is a regular expression, which is the base name -+# of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path" -+enable_annotations = @DEFENABLEANNOTATIONS@ -+ -+# List of valid annotations values for the hypervisor -+# Each member of the list is a path pattern as described by glob(3). -+# The default if not set is empty (all annotations rejected.) -+# Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@ -+valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ -+ -+# Optional space-separated list of options to pass to the guest kernel. -+# For example, use `kernel_params = "vsyscall=emulate"` if you are having -+# trouble running pre-2.15 glibc. -+# -+# WARNING: - any parameter specified here will take priority over the default -+# parameter value of the same name used to start the virtual machine. -+# Do not set values here unless you understand the impact of doing so as you -+# may stop the virtual machine from booting. -+# To see the list of default parameters, enable hypervisor debug, create a -+# container and look for 'default-kernel-parameters' log entries. -+kernel_params = "@KERNELPARAMS@" -+ -+# Default number of vCPUs per SB/VM: -+# unspecified or 0 --> will be set to @DEFVCPUS@ -+# < 0 --> will be set to the actual number of physical cores -+# > 0 <= number of physical cores --> will be set to the specified number -+# > number of physical cores --> will be set to the actual number of physical cores -+default_vcpus = 1 -+ -+# Default maximum number of vCPUs per SB/VM: -+# unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number -+# of vCPUs supported by KVM if that number is exceeded -+# > 0 <= number of physical cores --> will be set to the specified number -+# > number of physical cores --> will be set to the actual number of physical cores or to the maximum number -+# of vCPUs supported by KVM if that number is exceeded -+# WARNING: Depending of the architecture, the maximum number of vCPUs supported by KVM is used when -+# the actual number of physical cores is greater than it. -+# WARNING: Be aware that this value impacts the virtual machine's memory footprint and CPU -+# the hotplug functionality. For example, `default_maxvcpus = 240` specifies that until 240 vCPUs -+# can be added to a SB/VM, but the memory footprint will be big. Another example, with -+# `default_maxvcpus = 8` the memory footprint will be small, but 8 will be the maximum number of -+# vCPUs supported by the SB/VM. In general, we recommend that you do not edit this variable, -+# unless you know what are you doing. -+# NOTICE: on arm platform with gicv2 interrupt controller, set it to 8. -+default_maxvcpus = @DEFMAXVCPUS@ -+ -+# Bridges can be used to hot plug devices. -+# Limitations: -+# * Currently only pci bridges are supported -+# * Until 30 devices per bridge can be hot plugged. -+# * Until 5 PCI bridges can be cold plugged per VM. -+# This limitation could be a bug in the kernel -+# Default number of bridges per SB/VM: -+# unspecified or 0 --> will be set to @DEFBRIDGES@ -+# > 1 <= 5 --> will be set to the specified number -+# > 5 --> will be set to 5 -+default_bridges = @DEFBRIDGES@ -+ -+# Default memory size in MiB for SB/VM. -+# If unspecified then it will be set @DEFMEMSZ@ MiB. -+default_memory = @DEFMEMSZ@ -+# -+# Default memory slots per SB/VM. -+# If unspecified then it will be set @DEFMEMSLOTS@. -+# This is will determine the times that memory will be hotadded to sandbox/VM. -+# memory_slots = @DEFMEMSLOTS@ -+ -+# The size in MiB will be plused to max memory of hypervisor. -+# It is the memory address space for the NVDIMM devie. -+# If set block storage driver (block_device_driver) to "nvdimm", -+# should set memory_offset to the size of block device. -+# Default 0 -+# memory_offset = 0 -+ -+# Disable block device from being used for a container's rootfs. -+# In case of a storage driver like devicemapper where a container's -+# root file system is backed by a block device, the block device is passed -+# directly to the hypervisor for performance reasons. -+# This flag prevents the block device from being passed to the hypervisor, -+# 9pfs is used instead to pass the rootfs. -+disable_block_device_use = @DEFDISABLEBLOCK@ -+ -+# Block storage driver to be used for the hypervisor in case the container -+# rootfs is backed by a block device. This is virtio-scsi, virtio-blk -+# or nvdimm. -+block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" -+ -+# Specifies cache-related options will be set to block devices or not. -+# Default false -+#block_device_cache_set = true -+ -+# Specifies cache-related options for block devices. -+# Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. -+# Default false -+# block_device_cache_direct = true -+ -+# Specifies cache-related options for block devices. -+# Denotes whether flush requests for the device are ignored. -+# Default false -+# block_device_cache_noflush = true -+ -+# Enable pre allocation of VM RAM, default false -+# Enabling this will result in lower container density -+# as all of the memory will be allocated and locked -+# This is useful when you want to reserve all the memory -+# upfront or in the cases where you want memory latencies -+# to be very predictable -+# Default false -+# enable_mem_prealloc = true -+ -+# Enable huge pages for VM RAM, default false -+# Enabling this will result in the VM memory -+# being allocated using huge pages. -+# This is useful when you want to use vhost-user network -+# stacks within the container. This will automatically -+# result in memory pre allocation -+# enable_hugepages = true -+ -+# Enable vIOMMU, default false -+# Enabling this will result in the VM having a vIOMMU device -+# This will also add the following options to the kernel's -+# command line: intel_iommu=on,iommu=pt -+# enable_iommu = true -+ -+# Enable swap of vm memory. Default false. -+# The behaviour is undefined if mem_prealloc is also set to true -+# enable_swap = true -+ -+# This option changes the default hypervisor and kernel parameters -+# to enable debug output where available. -+# -+# Default false -+# enable_debug = true -+ -+# Disable the customizations done in the runtime when it detects -+# that it is running on top a VMM. This will result in the runtime -+# behaving as it would when running on bare metal. -+# -+# disable_nesting_checks = true -+ -+# This is the msize used for 9p shares. It is the number of bytes -+# used for 9p packet payload. -+# msize_9p = -+ -+# VFIO devices are hotplugged on a bridge by default. -+# Enable hotplugging on root bus. This may be required for devices with -+# a large PCI bar, as this is a current limitation with hotplugging on -+# a bridge. -+# Default false -+# hotplug_vfio_on_root_bus = true -+ -+# -+# Default entropy source. -+# The path to a host source of entropy (including a real hardware RNG) -+# /dev/urandom and /dev/random are two main options. -+# Be aware that /dev/random is a blocking source of entropy. If the host -+# runs out of entropy, the VMs boot time will increase leading to get startup -+# timeouts. -+# The source of entropy /dev/urandom is non-blocking and provides a -+# generally acceptable source of entropy. It should work well for pretty much -+# all practical purposes. -+# entropy_source= "" -+ -+# List of valid annotations values for entropy_source -+# The default if not set is empty (all annotations rejected.) -+# Your distribution recommends: @DEFVALIDENTROPYSOURCES@ -+valid_entropy_sources = @DEFVALIDENTROPYSOURCES@ -+ -+# Path to OCI hook binaries in the *guest rootfs*. -+# This does not affect host-side hooks which must instead be added to -+# the OCI spec passed to the runtime. -+# -+# You can create a rootfs with hooks by customizing the osbuilder scripts: -+# https://github.com/kata-containers/kata-containers/tree/main/tools/osbuilder -+# -+# Hooks must be stored in a subdirectory of guest_hook_path according to their -+# hook type, i.e. "guest_hook_path/{prestart,poststart,poststop}". -+# The agent will scan these directories for executable files and add them, in -+# lexicographical order, to the lifecycle of the guest container. -+# Hooks are executed in the runtime namespace of the guest. See the official documentation: -+# https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks -+# Warnings will be logged if any error is encountered will scanning for hooks, -+# but it will not abort container execution. -+# guest_hook_path = "/usr/share/oci/hooks" -+ -+[factory] -+# VM templating support. Once enabled, new VMs are created from template -+# using vm cloning. They will share the same initial kernel, initramfs and -+# agent memory by mapping it readonly. It helps speeding up new container -+# creation and saves a lot of memory if there are many kata containers running -+# on the same host. -+# -+# When disabled, new VMs are created from scratch. -+# -+# Note: Requires "initrd=" to be set ("image=" is not supported). -+# -+# Default false -+#enable_template = true -+ -+[agent.@PROJECT_TYPE@] -+# If enabled, make the agent display debug-level messages. -+# (default: disabled) -+#enable_debug = true -+ -+# Enable agent tracing. -+# -+# If enabled, the default trace mode is "dynamic" and the -+# default trace type is "isolated". The trace mode and type are set -+# explicity with the `trace_type=` and `trace_mode=` options. -+# -+# Notes: -+# -+# - Tracing is ONLY enabled when `enable_tracing` is set: explicitly -+# setting `trace_mode=` and/or `trace_type=` without setting `enable_tracing` -+# will NOT activate agent tracing. -+# -+# - See https://github.com/kata-containers/agent/blob/master/TRACING.md for -+# full details. -+# -+# (default: disabled) -+#enable_tracing = true -+# -+#trace_mode = "dynamic" -+#trace_type = "isolated" -+ -+# Comma separated list of kernel modules and their parameters. -+# These modules will be loaded in the guest kernel using modprobe(8). -+# The following example can be used to load two kernel modules with parameters -+# - kernel_modules=["e1000e InterruptThrottleRate=3000,3000,3000 EEE=1", "i915 enable_ppgtt=0"] -+# The first word is considered as the module name and the rest as its parameters. -+# Container will not be started when: -+# * A kernel module is specified and the modprobe command is not installed in the guest -+# or it fails loading the module. -+# * The module is not available in the guest or it doesn't met the guest kernel -+# requirements, like architecture and version. -+# -+kernel_modules=[] -+ -+# Enable debug console. -+ -+# If enabled, user can connect guest OS running inside hypervisor -+# through "kata-runtime exec " command -+ -+#debug_console_enabled = true -+ -+# Agent connection dialing timeout value in seconds -+# (default: 30) -+#dial_timeout = 30 -+ -+[netmon] -+# If enabled, the network monitoring process gets started when the -+# sandbox is created. This allows for the detection of some additional -+# network being added to the existing network namespace, after the -+# sandbox has been created. -+# (default: disabled) -+#enable_netmon = true -+ -+# Specify the path to the netmon binary. -+path = "@NETMONPATH@" -+ -+# If enabled, netmon messages will be sent to the system log -+# (default: disabled) -+#enable_debug = true -+ -+[runtime] -+# If enabled, the runtime will log additional debug messages to the -+# system log -+# (default: disabled) -+#enable_debug = true -+# -+# Internetworking model -+# Determines how the VM should be connected to the -+# the container network interface -+# Options: -+# -+# - macvtap -+# Used when the Container network interface can be bridged using -+# macvtap. -+# -+# - none -+# Used when customize network. Only creates a tap device. No veth pair. -+# -+# - tcfilter -+# Uses tc filter rules to redirect traffic from the network interface -+# provided by plugin to a tap interface connected to the VM. -+# -+internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@" -+ -+# disable guest seccomp -+# Determines whether container seccomp profiles are passed to the virtual -+# machine and applied by the kata agent. If set to true, seccomp is not applied -+# within the guest -+# (default: true) -+disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ -+ -+# If enabled, the runtime will create opentracing.io traces and spans. -+# (See https://www.jaegertracing.io/docs/getting-started). -+# (default: disabled) -+#enable_tracing = true -+ -+# Set the full url to the Jaeger HTTP Thrift collector. -+# The default if not set will be "http://localhost:14268/api/traces" -+#jaeger_endpoint = "" -+ -+# Sets the username to be used if basic auth is required for Jaeger. -+#jaeger_user = "" -+ -+# Sets the password to be used if basic auth is required for Jaeger. -+#jaeger_password = "" -+ -+# If enabled, the runtime will not create a network namespace for shim and hypervisor processes. -+# This option may have some potential impacts to your host. It should only be used when you know what you're doing. -+# `disable_new_netns` conflicts with `enable_netmon` -+# `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only -+# with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge -+# (like OVS) directly. -+# If you are using docker, `disable_new_netns` only works with `docker run --net=none` -+# (default: false) -+#disable_new_netns = true -+ -+# if enable, the runtime will add all the kata processes inside one dedicated cgroup. -+# The container cgroups in the host are not created, just one single cgroup per sandbox. -+# The runtime caller is free to restrict or collect cgroup stats of the overall Kata sandbox. -+# The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. -+# The sandbox cgroup is constrained if there is no container type annotation. -+# See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType -+sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ -+ -+# Enabled experimental feature list, format: ["a", "b"]. -+# Experimental features are features not stable enough for production, -+# they may break compatibility, and are prepared for a big version bump. -+# Supported experimental features: -+# (default: []) -+experimental=@DEFAULTEXPFEATURES@ -+ -+# If enabled, user can run pprof tools with shim v2 process through kata-monitor. -+# (default: false) -+# enable_pprof = true --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch b/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch deleted file mode 100644 index a8727ba7f3ee415d9bddc0b0a35ad9b0f7483811..0000000000000000000000000000000000000000 --- a/patches/0005-runtime-add-the-secure-component-ozone-support-for-h.patch +++ /dev/null @@ -1,655 +0,0 @@ -From 45c8e108497eb93d69afd38e6281b837e65cf3ec Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Mon, 9 Aug 2021 14:55:41 +0800 -Subject: [PATCH 5/6] runtime: add the secure component "ozone" support for - hypervisor type stratovirt. - -Signed-off-by: Wei Gao ---- - src/runtime/Makefile | 4 + - src/runtime/arch/amd64-options.mk | 4 +- - src/runtime/arch/arm64-options.mk | 4 +- - .../config/configuration-stratovirt.toml.in | 10 + - .../pkg/katautils/config-settings.go.in | 1 + - src/runtime/pkg/katautils/config.go | 18 + - src/runtime/virtcontainers/hypervisor.go | 3 + - src/runtime/virtcontainers/persist.go | 1 + - .../virtcontainers/persist/api/config.go | 3 + - src/runtime/virtcontainers/stratovirt.go | 309 ++++++++++++++---- - 10 files changed, 292 insertions(+), 65 deletions(-) - -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index ea2cd296..745bcc10 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -146,6 +146,8 @@ ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"] - - STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) - STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] -+STRATOVIRTOZONEPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTOZONECMD) -+STRATOVIRTVALIDOZONEPATHS = [\"$(STRATOVIRTOZONEPATH)\"] - - NETMONCMD := $(BIN_PREFIX)-netmon - NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD) -@@ -414,6 +416,8 @@ USER_VARS += FCJAILERPATH - USER_VARS += FCVALIDJAILERPATHS - USER_VARS += STRATOVIRTPATH - USER_VARS += STRATOVIRTVALIDHYPERVISORPATHS -+USER_VARS += STRATOVIRTOZONEPATH -+USER_VARS += STRATOVIRTVALIDOZONEPATHS - USER_VARS += SYSCONFIG - USER_VARS += IMAGENAME - USER_VARS += IMAGEPATH -diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk -index ff2af9e6..4c6c329a 100644 ---- a/src/runtime/arch/amd64-options.mk -+++ b/src/runtime/arch/amd64-options.mk -@@ -25,4 +25,6 @@ ACRNCTLCMD := acrnctl - CLHCMD := cloud-hypervisor - - # stratovirt binary name --STRATOVIRTCMD := stratovirt -\ No newline at end of file -+STRATOVIRTCMD := stratovirt -+# stratovirt's ozone binary name -+STRATOVIRTOZONECMD := ozone -\ No newline at end of file -diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk -index 2ad3f657..5dfa2c80 100644 ---- a/src/runtime/arch/arm64-options.mk -+++ b/src/runtime/arch/arm64-options.mk -@@ -21,4 +21,6 @@ FCJAILERCMD := jailer - CLHCMD := cloud-hypervisor - - # stratovirt binary name --STRATOVIRTCMD := stratovirt -\ No newline at end of file -+STRATOVIRTCMD := stratovirt -+# stratovirt's ozone binary name -+STRATOVIRTOZONECMD := ozone -\ No newline at end of file -diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in -index 5c83c3c9..b557b71f 100644 ---- a/src/runtime/cli/config/configuration-stratovirt.toml.in -+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in -@@ -26,6 +26,16 @@ enable_annotations = @DEFENABLEANNOTATIONS@ - # Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@ - valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ - -+# Path for the ozone specific to stratovirt -+# If the ozone path is set, stratovirt will be launched in -+# ozone secure environment. It is disabled by default. -+# ozone_path = "@STRATOVIRTOZONEPATH@" -+ -+# List of valid ozone path values for the hypervisor -+# Each member of the list can be a regular expression -+# The default if not set is empty (all annotations rejected.) -+# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@ -+ - # Optional space-separated list of options to pass to the guest kernel. - # For example, use `kernel_params = "vsyscall=emulate"` if you are having - # trouble running pre-2.15 glibc. -diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in -index 7cd9138b..c168c608 100644 ---- a/src/runtime/pkg/katautils/config-settings.go.in -+++ b/src/runtime/pkg/katautils/config-settings.go.in -@@ -17,6 +17,7 @@ var defaultInitrdPath = "/usr/share/kata-containers/kata-containers-initrd.img" - var defaultFirmwarePath = "" - var defaultMachineAccelerators = "" - var defaultCPUFeatures = "" -+var defaultOzonePath = "/usr/bin/ozone" - var systemdUnitName = "kata-containers.target" - - const defaultKernelParams = "" -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index f94ac4fd..828c2a43 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -92,6 +92,7 @@ type hypervisor struct { - FileBackedMemRootDir string `toml:"file_mem_backend"` - GuestHookPath string `toml:"guest_hook_path"` - GuestMemoryDumpPath string `toml:"guest_memory_dump_path"` -+ OzonePath string `toml:"ozone_path"` - HypervisorPathList []string `toml:"valid_hypervisor_paths"` - JailerPathList []string `toml:"valid_jailer_paths"` - CtlPathList []string `toml:"valid_ctlpaths"` -@@ -452,6 +453,16 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error) - return - } - -+func (h hypervisor) ozonePath() (string, error) { -+ p := h.OzonePath -+ -+ if h.OzonePath == "" { -+ return "", nil -+ } -+ -+ return ResolvePath(p) -+} -+ - func (h hypervisor) getRxRateLimiterCfg() uint64 { - return h.RxRateLimiterMaxRate - } -@@ -877,6 +888,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - return vc.HypervisorConfig{}, err - } - -+ ozone, err := h.ozonePath() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ - kernel, err := h.kernel() - if err != nil { - return vc.HypervisorConfig{}, err -@@ -925,6 +941,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - KernelPath: kernel, - InitrdPath: initrd, - ImagePath: image, -+ OzonePath: ozone, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), - HypervisorMachineType: machineType, - NumVCPUs: h.defaultVCPUs(), -@@ -1155,6 +1172,7 @@ func GetDefaultHypervisorConfig() vc.HypervisorConfig { - RxRateLimiterMaxRate: defaultRxRateLimiterMaxRate, - TxRateLimiterMaxRate: defaultTxRateLimiterMaxRate, - SGXEPCSize: defaultSGXEPCSize, -+ OzonePath: defaultOzonePath, - } - } - -diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go -index 615baa80..04e14b4e 100644 ---- a/src/runtime/virtcontainers/hypervisor.go -+++ b/src/runtime/virtcontainers/hypervisor.go -@@ -302,6 +302,9 @@ type HypervisorConfig struct { - // JailerPathList is the list of jailer paths names allowed in annotations - JailerPathList []string - -+ // OzonePath is the ozone executable host path. -+ OzonePath string -+ - // BlockDeviceDriver specifies the driver to be used for block device - // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver - BlockDeviceDriver string -diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go -index 203495e8..ae499c97 100644 ---- a/src/runtime/virtcontainers/persist.go -+++ b/src/runtime/virtcontainers/persist.go -@@ -219,6 +219,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) { - HypervisorCtlPathList: sconfig.HypervisorConfig.HypervisorCtlPathList, - JailerPath: sconfig.HypervisorConfig.JailerPath, - JailerPathList: sconfig.HypervisorConfig.JailerPathList, -+ OzonePath: sconfig.HypervisorConfig.OzonePath, - BlockDeviceDriver: sconfig.HypervisorConfig.BlockDeviceDriver, - HypervisorMachineType: sconfig.HypervisorConfig.HypervisorMachineType, - MemoryPath: sconfig.HypervisorConfig.MemoryPath, -diff --git a/src/runtime/virtcontainers/persist/api/config.go b/src/runtime/virtcontainers/persist/api/config.go -index 3bd5567d..88903723 100644 ---- a/src/runtime/virtcontainers/persist/api/config.go -+++ b/src/runtime/virtcontainers/persist/api/config.go -@@ -76,6 +76,9 @@ type HypervisorConfig struct { - // JailerPathList is the list of jailer paths names allowed in annotations - JailerPathList []string - -+ // OzonePath is the ozone executable host path. -+ OzonePath string -+ - // BlockDeviceDriver specifies the driver to be used for block device - // either VirtioSCSI or VirtioBlock with the default driver being defaultBlockDriver - BlockDeviceDriver string -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 0f473e31..47daa817 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -3,6 +3,7 @@ package virtcontainers - import ( - "context" - "fmt" -+ "io/ioutil" - "os" - "os/exec" - "path/filepath" -@@ -24,10 +25,15 @@ import ( - otelTrace "go.opentelemetry.io/otel/trace" - ) - --const defaultDummyMac = "22:33:44:aa:bb:" --const mmioBlkCount = 4 --const mmioNetCount = 2 --const randomDevice = "/dev/urandom" -+const ( -+ apiSocket = "qmp.socket" -+ debugSocket = "console.socket" -+ ozoneBaseDir = "/srv/ozone/stratovirt" -+ defaultDummyMac = "22:33:44:aa:bb:" -+ mmioBlkCount = 4 -+ mmioNetCount = 2 -+ randomDevice = "/dev/urandom" -+) - - type stratovirtDev struct { - dev interface{} -@@ -40,10 +46,19 @@ type stratovirt struct { - sandbox *Sandbox - store persistapi.PersistDriver - config HypervisorConfig -+ rootfsPath string -+ kernelPath string - pid int - consolePath string - socketPath string -+ netNSPath string - qmpMonitorCh qmpChannel -+ ozoneRoot string -+ ozoneRes []string -+ useOzone bool -+ useImage bool -+ pidfile string -+ logfile string - devices []stratovirtDev - HotpluggedVCPUs []CPUDevice - mmioBlkSlots [mmioBlkCount]bool -@@ -66,10 +81,10 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, - return span, ctx - } - --func (s *stratovirt) getKernelCmdLine(useImage bool) string { -+func (s *stratovirt) getKernelCmdLine() string { - var params []string - -- if useImage { -+ if s.useImage { - params = append(params, "root=/dev/vda") - } - -@@ -100,14 +115,49 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net - - s.id = id - s.config = *hypervisorConfig -- -- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, "qmp.sock") -- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, "console.sock") -+ if s.config.OzonePath == "" { -+ s.useOzone = false -+ s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") -+ s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") -+ s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket) -+ s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket) -+ } else { -+ s.useOzone = true -+ s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id) -+ s.pidfile = filepath.Join(s.ozoneRoot, "pid") -+ s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log") -+ s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) -+ s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) -+ } -+ s.netNSPath = networkNS.NetNsPath - s.qmpMonitorCh = qmpChannel{ - ctx: s.ctx, - path: s.socketPath, - } - -+ if kernelPath, err := s.config.KernelAssetPath(); err == nil { -+ s.kernelPath = kernelPath -+ s.ozoneRes = append(s.ozoneRes, s.kernelPath) -+ } -+ -+ initrdPath, err := s.config.InitrdAssetPath() -+ if err != nil { -+ return err -+ } -+ -+ if initrdPath == "" { -+ imagePath, err := s.config.ImageAssetPath() -+ if err != nil { -+ return err -+ } -+ s.useImage = true -+ s.rootfsPath = imagePath -+ } else { -+ s.useImage = false -+ s.rootfsPath = initrdPath -+ } -+ s.ozoneRes = append(s.ozoneRes, s.rootfsPath) -+ - return nil - } - -@@ -134,48 +184,43 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error { - return nil - } - --func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { -- span, _ := s.trace(ctx, "startSandbox") -- defer span.End() -- -+func (s *stratovirt) createbaseParams() []string { - var params []string -- var use_image bool -+ - params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) -- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) -+ params = append(params, "-append", s.getKernelCmdLine()) -+ params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) -+ params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) -+ params = append(params, "-device", "virtio-serial-device") -+ params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") -+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) -+ params = append(params, "-device", "virtio-rng-device,rng=objrng0") - -- if kernelPath, err := s.config.KernelAssetPath(); err == nil { -- params = append(params, "-kernel", kernelPath) -- } -+ // daemonize -+ params = append(params, "-daemonize") - -- initrdPath, err := s.config.InitrdAssetPath() -- if err != nil { -- return err -+ return params -+} -+ -+func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { -+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket)) -+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket)) -+ params = append(params, "-kernel", filepath.Base(s.kernelPath)) -+ params = append(params, "-pidfile", filepath.Base(s.pidfile)) -+ -+ // append logfile only on debug -+ if s.config.Debug { -+ params = append(params, "-D", filepath.Base(s.logfile)) - } - -- if initrdPath == "" { -- imagePath, err := s.config.ImageAssetPath() -- if err != nil { -- return err -- } -- use_image = true -+ if s.useImage { - s.mmioBlkSlots[0] = true - params = append(params, "-device", "virtio-blk-device,drive=rootfs") -- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", imagePath)) -+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath))) - } else { -- use_image = false -- params = append(params, "-initrd", initrdPath) -+ params = append(params, "-initrd", filepath.Base(s.rootfsPath)) - } - -- params = append(params, "-append", s.getKernelCmdLine(use_image)) -- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) -- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) -- params = append(params, "-device", "virtio-serial-device") -- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") -- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) -- params = append(params, "-device", "virtio-rng-device,rng=objrng0") -- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) -- params = append(params, "-pidfile", filepath.Join(s.store.RunVMStoragePath(), s.id, "pid")) -- - // add devices to cmdline - for _, d := range s.devices { - switch v := d.dev.(type) { -@@ -188,8 +233,9 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { - case config.BlockDrive: - id := v.ID - path := v.File -- params = append(params, "-device", fmt.Sprintf("virtio-blk-device, drive=%s", id)) -- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) -+ s.ozoneRes = append(s.ozoneRes, path) -+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) -+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path))) - case types.VSock: - v.VhostFd.Close() - params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) -@@ -198,42 +244,125 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { - } - } - -- // daemonize -- params = append(params, "-daemonize") -+ return params, nil -+} -+ -+func (s *stratovirt) createParams(params []string) ([]string, error) { -+ params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) -+ params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) -+ params = append(params, "-kernel", s.kernelPath) -+ params = append(params, "-pidfile", s.pidfile) - - // append logfile only on debug - if s.config.Debug { -- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -- params = append(params, "-D", fmt.Sprintf("%s/stratovirt.log", dir)) -+ params = append(params, "-D", s.logfile) -+ } -+ -+ if s.useImage { -+ s.mmioBlkSlots[0] = true -+ params = append(params, "-device", "virtio-blk-device,drive=rootfs") -+ params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath)) -+ } else { -+ params = append(params, "-initrd", s.rootfsPath) - } - -- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -- err = os.MkdirAll(dir, DirMode) -+ // add devices to cmdline -+ for _, d := range s.devices { -+ switch v := d.dev.(type) { -+ case Endpoint: -+ name := v.Name() -+ mac := v.HardwareAddr() -+ tapName := v.NetworkPair().TapInterface.TAPIface.Name -+ params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) -+ params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) -+ case config.BlockDrive: -+ id := v.ID -+ path := v.File -+ params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) -+ params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) -+ case types.VSock: -+ v.VhostFd.Close() -+ params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) -+ default: -+ s.Logger().Error("Adding device type is unsupported") -+ } -+ } -+ -+ return params, nil -+} -+ -+func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { -+ span, _ := s.trace(ctx, "startSandbox") -+ defer span.End() -+ -+ var err error -+ var cmd *exec.Cmd -+ -+ params := s.createbaseParams() -+ -+ stratovirtBinPath, err := s.config.HypervisorAssetPath() - if err != nil { - return err - } -- defer func() { -+ -+ if s.useOzone { -+ var ozoneParams []string -+ extend_params, err := s.createOzoneParams(params) - if err != nil { -- if err := os.RemoveAll(dir); err != nil { -- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) -+ return err -+ } -+ ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath) -+ ozoneParams = append(ozoneParams, "-name", s.id) -+ ozoneParams = append(ozoneParams, "-gid", "0") -+ ozoneParams = append(ozoneParams, "-uid", "0") -+ if s.netNSPath != "" { -+ ozoneParams = append(ozoneParams, "-netns", s.netNSPath) -+ } -+ -+ ozoneParams = append(ozoneParams, "-source") -+ ozoneParams = append(ozoneParams, s.ozoneRes...) -+ -+ defer func() { -+ if err != nil { -+ ozoneParams = append(ozoneParams, "-clean-resource") -+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) -+ if err := cmd.Run(); err != nil { -+ s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot) -+ } - } -+ }() -+ -+ ozoneParams = append(ozoneParams, "--") -+ ozoneParams = append(ozoneParams, extend_params...) -+ cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) -+ s.Logger().Info("StratoVirt/Ozone start with params: ", cmd) -+ } else { -+ params, err = s.createParams(params) -+ if err != nil { -+ return err - } -- }() - -- binPath, err := s.config.HypervisorAssetPath() -- if err != nil { -- s.Logger().WithField("Fail to get hypervisor bin path", err).Error() -- return err -- } -+ dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -+ err = os.MkdirAll(dir, DirMode) -+ if err != nil { -+ return err -+ } -+ defer func() { -+ if err != nil { -+ if err := os.RemoveAll(dir); err != nil { -+ s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) -+ } -+ } -+ }() - -- cmd := exec.CommandContext(s.ctx, binPath, params...) -- s.Logger().Info("StratoVirt start with params: ", cmd) -+ cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...) -+ s.Logger().Info("StratoVirt start with params: ", cmd) -+ } - - if err := cmd.Start(); err != nil { - s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() - return err - } -- s.pid = cmd.Process.Pid - - if err = s.waitSandBoxStarted(timeout); err != nil { - return err -@@ -420,6 +549,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera - } - - func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { -+ var filePath string - err = s.qmpSetup() - if err != nil { - return err -@@ -427,13 +557,18 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err - - switch op { - case addDevice: -- driver := "virtio-blk-pci" -+ driver := "virtio-blk-mmio" -+ if s.useOzone { -+ filePath, err = s.updateOzoneRes(drive.File, true) -+ } else { -+ filePath = drive.File -+ } - slot, err := s.getDevSlot(drive.VirtPath, false) - if err != nil { - return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) - } - -- if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, drive.File, drive.ID, false); err != nil { -+ if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil { - s.getDevSlot(drive.VirtPath, true) - return err - } -@@ -443,6 +578,9 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err - return err - } - case removeDevice: -+ if s.useOzone { -+ s.updateOzoneRes(drive.File, false) -+ } - if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { - return err - } -@@ -582,17 +720,62 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) { - return tid, nil - } - -+func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { -+ dst := filepath.Join(s.ozoneRoot, filepath.Base(src)) -+ if add { -+ if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil { -+ s.Logger().WithField("bindMount failed", err).Error() -+ return "", err -+ } -+ } else { -+ syscall.Unmount(dst, syscall.MNT_DETACH) -+ } -+ return filepath.Base(src), nil -+} -+ -+func (s *stratovirt) cleanOzoneRes() { -+ s.updateOzoneRes(s.rootfsPath, false) -+ s.updateOzoneRes(s.kernelPath, false) -+ -+ if err := os.RemoveAll(s.ozoneRoot); err != nil { -+ s.Logger().WithField("cleanupOzone failed", err).Error() -+ } -+} -+ - func (s *stratovirt) cleanup(ctx context.Context) error { - span, _ := s.trace(ctx, "cleanup") - defer span.End() - - s.qmpTeardown() -+ if s.useOzone { -+ s.cleanOzoneRes() -+ } - - return nil - } - - func (s *stratovirt) getPids() []int { -- return []int{s.pid} -+ var pids []int -+ if s.pid != 0 { -+ pids = append(pids, s.pid) -+ } else { -+ pid, err := ioutil.ReadFile(s.pidfile) -+ if err != nil { -+ s.Logger().WithError(err).Error("Read pid file failed.") -+ return []int{0} -+ } -+ -+ p, err := strconv.Atoi(strings.Trim(string(pid), "\n\t ")) -+ if err != nil { -+ s.Logger().WithError(err).Error("Get pid from pid file failed.") -+ return []int{0} -+ } -+ -+ pids = append(pids, p) -+ s.pid = p -+ } -+ -+ return pids - } - - func (s *stratovirt) getVirtioFsPid() *int { --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch b/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch deleted file mode 100644 index 4a297eab2d9efd7e513cb2c13c2a2d5d464b5224..0000000000000000000000000000000000000000 --- a/patches/0006-factory-add-the-template-factory-support-for-hypervi.patch +++ /dev/null @@ -1,295 +0,0 @@ -From 77ed6fefe70edde63b01d797b76f389bc82bb1a0 Mon Sep 17 00:00:00 2001 -From: Wei Gao -Date: Mon, 9 Aug 2021 14:57:06 +0800 -Subject: [PATCH 6/6] factory: add the template factory support for hypervisor - type stratovirt. - -Signed-off-by: Wei Gao ---- - src/runtime/pkg/katautils/config.go | 2 +- - .../factory/template/template.go | 21 +++-- - src/runtime/virtcontainers/kata_agent.go | 7 +- - src/runtime/virtcontainers/stratovirt.go | 89 +++++++++++++++++-- - src/runtime/virtcontainers/vm.go | 28 ++++-- - 5 files changed, 125 insertions(+), 22 deletions(-) - -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index 828c2a43..718677b4 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -1363,7 +1363,7 @@ func checkNetNsConfig(config oci.RuntimeConfig) error { - // checkFactoryConfig ensures the VM factory configuration is valid. - func checkFactoryConfig(config oci.RuntimeConfig) error { - if config.FactoryConfig.Template { -- if config.HypervisorConfig.InitrdPath == "" { -+ if config.HypervisorConfig.InitrdPath == "" && (config.HypervisorType != vc.StratovirtHypervisor) { - return errors.New("Factory option enable_template requires an initrd image") - } - } -diff --git a/src/runtime/virtcontainers/factory/template/template.go b/src/runtime/virtcontainers/factory/template/template.go -index 66070126..02497097 100644 ---- a/src/runtime/virtcontainers/factory/template/template.go -+++ b/src/runtime/virtcontainers/factory/template/template.go -@@ -96,11 +96,15 @@ func (t *template) prepareTemplateFiles() error { - if err != nil { - return err - } -- flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) -- opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) -- if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { -- t.close() -- return err -+ -+ // If use hypervisor stratovirt, no need to create template path with ramdisk. -+ if t.config.HypervisorType != vc.StratovirtHypervisor { -+ flags := uintptr(syscall.MS_NOSUID | syscall.MS_NODEV) -+ opts := fmt.Sprintf("size=%dM", t.config.HypervisorConfig.MemorySize+templateDeviceStateSize) -+ if err = syscall.Mount("tmpfs", t.statePath, "tmpfs", flags, opts); err != nil { -+ t.close() -+ return err -+ } - } - f, err := os.Create(t.statePath + "/memory") - if err != nil { -@@ -126,8 +130,11 @@ func (t *template) createTemplateVM(ctx context.Context) error { - } - defer vm.Stop(ctx) - -- if err = vm.Disconnect(ctx); err != nil { -- return err -+ // Create template on hypervisor stratovirt, don't have connection with agent. -+ if config.HypervisorType != vc.StratovirtHypervisor { -+ if err = vm.Disconnect(ctx); err != nil { -+ return err -+ } - } - - // Sleep a bit to let the agent grpc server clean up -diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go -index 13d31658..bc882c70 100644 ---- a/src/runtime/virtcontainers/kata_agent.go -+++ b/src/runtime/virtcontainers/kata_agent.go -@@ -1306,8 +1306,11 @@ func (k *kataAgent) buildContainerRootfs(ctx context.Context, sandbox *Sandbox, - // TODO: remove dependency on shared fs path. shared fs is just one kind of storage source. - // we should not always use shared fs path for all kinds of storage. Instead, all storage - // should be bind mounted to a tmpfs path for containers to use. -- if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { -- return nil, err -+ // If boot from template on stratovirt, no need to mkdir mount path. -+ if !((sandbox.config.HypervisorType == StratovirtHypervisor) && sandbox.config.HypervisorConfig.BootFromTemplate) { -+ if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { -+ return nil, err -+ } - } - return rootfs, nil - } -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 47daa817..e9b2ba85 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -48,6 +48,7 @@ type stratovirt struct { - config HypervisorConfig - rootfsPath string - kernelPath string -+ templatePath string - pid int - consolePath string - socketPath string -@@ -115,7 +116,7 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net - - s.id = id - s.config = *hypervisorConfig -- if s.config.OzonePath == "" { -+ if (s.config.OzonePath == "") || s.config.BootToBeTemplate { - s.useOzone = false - s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") - s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") -@@ -129,6 +130,20 @@ func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS Net - s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) - s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) - } -+ -+ if s.config.VMid != "" && s.useOzone { -+ // Make sure the symlinks do not exist -+ os.RemoveAll(s.ozoneRoot) -+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) -+ if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil { -+ return err -+ } -+ } -+ -+ if s.config.BootFromTemplate || s.config.BootToBeTemplate { -+ s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1) -+ } -+ - s.netNSPath = networkNS.NetNsPath - s.qmpMonitorCh = qmpChannel{ - ctx: s.ctx, -@@ -221,6 +236,12 @@ func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { - params = append(params, "-initrd", filepath.Base(s.rootfsPath)) - } - -+ // handle boot from template -+ if s.config.BootFromTemplate { -+ s.ozoneRes = append(s.ozoneRes, s.templatePath) -+ params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath))) -+ } -+ - // add devices to cmdline - for _, d := range s.devices { - switch v := d.dev.(type) { -@@ -266,6 +287,11 @@ func (s *stratovirt) createParams(params []string) ([]string, error) { - params = append(params, "-initrd", s.rootfsPath) - } - -+ // handle boot from template -+ if s.config.BootFromTemplate { -+ params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath)) -+ } -+ - // add devices to cmdline - for _, d := range s.devices { - switch v := d.dev.(type) { -@@ -410,14 +436,55 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { - } - - func (s *stratovirt) pauseSandbox(ctx context.Context) error { -- return nil -+ span, _ := s.trace(ctx, "pauseSandbox") -+ defer span.End() -+ -+ return s.togglePauseSandbox(ctx, true) - } - - func (s *stratovirt) saveSandbox() error { -+ s.Logger().Info("save sandbox") -+ -+ err := s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ // BootToBeTemplate sets the VM to be a template that other VMs can can clone from. -+ // We would want to bypass shared memory when saving VM to local file through migrate. -+ if s.config.BootToBeTemplate { -+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath)) -+ if err != nil { -+ s.Logger().WithError(err).Error("exec migration") -+ return err -+ } -+ } -+ - return nil - } - - func (s *stratovirt) resumeSandbox(ctx context.Context) error { -+ span, _ := s.trace(ctx, "resumeSandbox") -+ defer span.End() -+ -+ return s.togglePauseSandbox(ctx, false) -+} -+ -+func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error { -+ span, _ := s.trace(ctx, "togglePauseSandbox") -+ defer span.End() -+ -+ err := s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ if pause { -+ s.qmpMonitorCh.qmp.ExecuteStop(s.qmpMonitorCh.ctx) -+ } else { -+ s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx) -+ } -+ - return nil - } - -@@ -734,11 +801,23 @@ func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { - } - - func (s *stratovirt) cleanOzoneRes() { -- s.updateOzoneRes(s.rootfsPath, false) -- s.updateOzoneRes(s.kernelPath, false) -+ // Umount all resource in ozoneRoot -+ if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil { -+ for _, file := range dir { -+ syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH) -+ } -+ } - - if err := os.RemoveAll(s.ozoneRoot); err != nil { -- s.Logger().WithField("cleanupOzone failed", err).Error() -+ s.Logger().WithField("cleanup Ozone failed", err).Error() -+ } -+ -+ // If have VMid, the VM is boot from template. ozoneVmRoot also need clean. -+ if s.config.VMid != "" { -+ ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) -+ if err := os.RemoveAll(ozoneVmRoot); err != nil { -+ s.Logger().WithField("cleanup Ozone failed", err).Error() -+ } - } - } - -diff --git a/src/runtime/virtcontainers/vm.go b/src/runtime/virtcontainers/vm.go -index e6f02b6e..c4f9df73 100644 ---- a/src/runtime/virtcontainers/vm.go -+++ b/src/runtime/virtcontainers/vm.go -@@ -142,13 +142,19 @@ func NewVM(ctx context.Context, config VMConfig) (*VM, error) { - }() - - // 4. check agent aliveness -- // VMs booted from template are paused, do not check -- if !config.HypervisorConfig.BootFromTemplate { -+ // On hypervisor StratoVirt, VMs booted from template are running, check agent -+ // On other hypervisors, VMs booted from template are paused, do not check -+ if config.HypervisorType == StratovirtHypervisor { -+ if !config.HypervisorConfig.BootToBeTemplate { -+ virtLog.WithField("vm", id).Info("check agent status") -+ err = agent.check(ctx) -+ } -+ } else if !config.HypervisorConfig.BootFromTemplate { - virtLog.WithField("vm", id).Info("check agent status") - err = agent.check(ctx) -- if err != nil { -- return nil, err -- } -+ } -+ if err != nil { -+ return nil, err - } - - return &VM{ -@@ -329,9 +335,16 @@ func (v *VM) assignSandbox(s *Sandbox) error { - // - link 9pfs share path from sandbox dir (/run/kata-containers/shared/sandboxes/sbid/) to vm dir (/run/vc/vm/vmid/shared/) - - vmSharePath := buildVMSharePath(v.id, v.store.RunVMStoragePath()) -- vmSockDir := filepath.Join(v.store.RunVMStoragePath(), v.id) - sbSharePath := getMountPath(s.id) -- sbSockDir := filepath.Join(v.store.RunVMStoragePath(), s.id) -+ var vmSockDir string -+ var sbSockDir string -+ if v.hypervisor.hypervisorConfig().OzonePath != "" { -+ vmSockDir = filepath.Join(ozoneBaseDir, v.id) -+ sbSockDir = filepath.Join(ozoneBaseDir, s.id) -+ } else { -+ vmSockDir = filepath.Join(v.store.RunVMStoragePath(), v.id) -+ sbSockDir = filepath.Join(v.store.RunVMStoragePath(), s.id) -+ } - - v.logger().WithFields(logrus.Fields{ - "vmSharePath": vmSharePath, -@@ -359,6 +372,7 @@ func (v *VM) assignSandbox(s *Sandbox) error { - - s.hypervisor = v.hypervisor - s.config.HypervisorConfig.VMid = v.id -+ s.config.HypervisorConfig.BootFromTemplate = true - - return nil - } --- -2.21.1 (Apple Git-122.3) - diff --git a/patches/0007-kata-containers-support-with-iSulad.patch b/patches/0007-kata-containers-support-with-iSulad.patch deleted file mode 100644 index c9c1d1a0a61acf1c800c65572d335f443f64cd21..0000000000000000000000000000000000000000 --- a/patches/0007-kata-containers-support-with-iSulad.patch +++ /dev/null @@ -1,224 +0,0 @@ -From d4605dafaa9c326a5cf24c28d0c1efe6c9997f49 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Sat, 21 Aug 2021 17:08:46 +0800 -Subject: [PATCH] kata-containers: support with iSulad - -reason: support with iSulad - -Signed-off-by: holyfei ---- - src/agent/rustjail/src/cgroups/fs/mod.rs | 2 +- - src/runtime/containerd-shim-v2/container.go | 9 +++ - src/runtime/containerd-shim-v2/service.go | 55 +++++++++++++++++++ - src/runtime/containerd-shim-v2/start.go | 10 ++++ - .../containerd/runtime/v2/shim/shim.go | 8 ++- - 5 files changed, 81 insertions(+), 3 deletions(-) - -diff --git a/src/agent/rustjail/src/cgroups/fs/mod.rs b/src/agent/rustjail/src/cgroups/fs/mod.rs -index 7f41cb4..6c3bb32 100644 ---- a/src/agent/rustjail/src/cgroups/fs/mod.rs -+++ b/src/agent/rustjail/src/cgroups/fs/mod.rs -@@ -369,7 +369,7 @@ fn set_memory_resources(cg: &cgroups::Cgroup, memory: &LinuxMemory, update: bool - if let Some(swappiness) = memory.swappiness { - if (0..=100).contains(&swappiness) { - mem_controller.set_swappiness(swappiness as u64)?; -- } else { -+ } else if swappiness != -1 { - return Err(anyhow!( - "invalid value:{}. valid memory swappiness range is 0-100", - swappiness -diff --git a/src/runtime/containerd-shim-v2/container.go b/src/runtime/containerd-shim-v2/container.go -index faea0e2..d563888 100644 ---- a/src/runtime/containerd-shim-v2/container.go -+++ b/src/runtime/containerd-shim-v2/container.go -@@ -7,10 +7,13 @@ package containerdshim - - import ( - "io" -+ "os" -+ "path" - "time" - - "github.com/containerd/containerd/api/types/task" - "github.com/containerd/containerd/errdefs" -+ cdshim "github.com/containerd/containerd/runtime/v2/shim" - taskAPI "github.com/containerd/containerd/runtime/v2/task" - "github.com/opencontainers/runtime-spec/specs-go" - -@@ -37,6 +40,8 @@ type container struct { - status task.Status - terminal bool - mounted bool -+ exitFifo string -+ exitFd *os.File - } - - func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) { -@@ -49,6 +54,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con - spec = &specs.Spec{} - } - -+ dir := os.Getenv(cdshim.ExitFifoDir) -+ exitFifo := path.Join(dir, r.ID, exitFifoName) -+ - c := &container{ - s: s, - spec: spec, -@@ -65,6 +73,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con - exitCh: make(chan uint32, 1), - stdinCloser: make(chan struct{}), - mounted: mounted, -+ exitFifo: exitFifo, - } - return c, nil - } -diff --git a/src/runtime/containerd-shim-v2/service.go b/src/runtime/containerd-shim-v2/service.go -index 1003f8e..e13283c 100644 ---- a/src/runtime/containerd-shim-v2/service.go -+++ b/src/runtime/containerd-shim-v2/service.go -@@ -6,13 +6,16 @@ - package containerdshim - - import ( -+ "bytes" - "context" -+ "encoding/binary" - "io/ioutil" - "os" - sysexec "os/exec" - "sync" - "syscall" - "time" -+ "unsafe" - - eventstypes "github.com/containerd/containerd/api/events" - "github.com/containerd/containerd/api/types/task" -@@ -51,6 +54,8 @@ const ( - // A time span used to wait for publish a containerd event, - // once it costs a longer time than timeOut, it will be canceld. - timeOut = 5 * time.Second -+ -+ exitFifoName = "exit_fifo" - ) - - var ( -@@ -1019,6 +1024,10 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI. - func (s *service) processExits() { - for e := range s.ec { - s.checkProcesses(e) -+ -+ if os.Getenv(cdshim.ExitFifoDir) != "" { -+ s.closeExitFifo(e) -+ } - } - } - -@@ -1070,3 +1079,49 @@ func (s *service) getContainerStatus(containerID string) (task.Status, error) { - - return status, nil - } -+ -+func isBigEndian() (ret bool) { -+ i := int(0x1) -+ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i)) -+ return bs[0] == 0 -+} -+ -+func (s *service) closeExitFifo(e exit) { -+ if e.execid != "" { -+ // not a container, no need to close exit fifo -+ return -+ } -+ -+ var ret uint32 -+ var nativeEndian binary.ByteOrder -+ -+ s.mu.Lock() -+ c, err := s.getContainer(e.id) -+ s.mu.Unlock() -+ -+ if err != nil { -+ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id) -+ return -+ } -+ -+ ret = <-c.exitCh -+ // refill the exitCh with the container process's exit code in case -+ // there were other waits on this process. -+ c.exitCh <- ret -+ -+ if isBigEndian() { -+ nativeEndian = binary.BigEndian -+ } else { -+ nativeEndian = binary.LittleEndian -+ } -+ -+ bytesBuffer := bytes.NewBuffer([]byte{}) -+ binary.Write(bytesBuffer, nativeEndian, &ret) -+ -+ _, err = c.exitFd.Write(bytesBuffer.Bytes()) -+ if err != nil { -+ logrus.WithError(err).Error("write exit fifo failed") -+ } -+ -+ c.exitFd.Close() -+} -diff --git a/src/runtime/containerd-shim-v2/start.go b/src/runtime/containerd-shim-v2/start.go -index 72420e4..e89dc48 100644 ---- a/src/runtime/containerd-shim-v2/start.go -+++ b/src/runtime/containerd-shim-v2/start.go -@@ -8,8 +8,11 @@ package containerdshim - import ( - "context" - "fmt" -+ "golang.org/x/sys/unix" -+ "os" - - "github.com/containerd/containerd/api/types/task" -+ cdshim "github.com/containerd/containerd/runtime/v2/shim" - "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils" - ) - -@@ -59,6 +62,13 @@ func startContainer(ctx context.Context, s *service, c *container) error { - - c.status = task.StatusRunning - -+ if os.Getenv(cdshim.ExitFifoDir) != "" { -+ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) -+ if err != nil { -+ return err -+ } -+ } -+ - stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id) - if err != nil { - return err -diff --git a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go -index d60d496..946c386 100644 ---- a/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go -+++ b/src/runtime/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go -@@ -84,6 +84,8 @@ var ( - action string - ) - -+var ExitFifoDir = "EXIT_FIFO_DIR" -+ - func parseFlags() { - flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") - flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") -@@ -198,8 +200,10 @@ func run(id string, initFunc Init, config Config) error { - } - return nil - default: -- if err := setLogger(ctx, idFlag); err != nil { -- return err -+ if os.Getenv("EXIT_FIFO_DIR") == "" { -+ if err := setLogger(ctx, idFlag); err != nil { -+ return err -+ } - } - client := NewShimClient(ctx, service, signals) - return client.Serve() --- -2.23.0 - diff --git a/patches/0008-kata-containers-adpat-with-iSulad.patch b/patches/0008-kata-containers-adpat-with-iSulad.patch deleted file mode 100644 index 5cf487b810bedcf422b94bbdf1747d009b800378..0000000000000000000000000000000000000000 --- a/patches/0008-kata-containers-adpat-with-iSulad.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 5ee8e64c8c620f02cb580f1f3349ae63660ca34c Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Mon, 27 Sep 2021 11:03:58 +0800 -Subject: [PATCH] kata-containers: adpat with iSulad - -reason: chmod the exec fifo to 644, isula start container -and need the permission - -Signed-off-by: holyfei ---- - src/agent/rustjail/src/container.rs | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/agent/rustjail/src/container.rs b/src/agent/rustjail/src/container.rs -index 748ee486..5113a482 100644 ---- a/src/agent/rustjail/src/container.rs -+++ b/src/agent/rustjail/src/container.rs -@@ -822,7 +822,7 @@ impl BaseContainer for LinuxContainer { - if stat::stat(fifo_file.as_str()).is_ok() { - return Err(anyhow!("exec fifo exists")); - } -- unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o622).unwrap())?; -+ unistd::mkfifo(fifo_file.as_str(), Mode::from_bits(0o644).unwrap())?; - - fifofd = fcntl::open( - fifo_file.as_str(), --- -2.23.0 - diff --git a/patches/0009-kata-containers-fix-kata-runtime-hungs-when-qemu-proces.patch b/patches/0009-kata-containers-fix-kata-runtime-hungs-when-qemu-proces.patch deleted file mode 100644 index 4c780e69af28fb887b54b6551438a79b66734912..0000000000000000000000000000000000000000 --- a/patches/0009-kata-containers-fix-kata-runtime-hungs-when-qemu-proces.patch +++ /dev/null @@ -1,27 +0,0 @@ -From c563b455e3bb0cebfbe2a77c8d5ebac36aac3c76 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Thu, 4 Nov 2021 19:58:33 +0800 -Subject: [PATCH] kata-runtime: fix kata-runtime hungs when qemu process is D/T - state - -Signed-off-by: jikui ---- - src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -index 97e9245..325250d 100644 ---- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -@@ -717,6 +717,8 @@ func QMPStart(ctx context.Context, socket string, cfg QMPConfig, disconnectedCh - if q.version == nil { - return nil, nil, fmt.Errorf("failed to find QMP version information") - } -+ case <-time.After(15 * time.Second): -+ return nil, nil, fmt.Errorf("qmp start time out") - } - - return q, q.version, nil --- -2.25.1 - diff --git a/patches/0010-kata-containers-fix-kata-runtime-skip-read-lines-in-pro.patch b/patches/0010-kata-containers-fix-kata-runtime-skip-read-lines-in-pro.patch deleted file mode 100644 index ed0504506ad61cf6b2efc1ec25a73b6e6d4656a4..0000000000000000000000000000000000000000 --- a/patches/0010-kata-containers-fix-kata-runtime-skip-read-lines-in-pro.patch +++ /dev/null @@ -1,115 +0,0 @@ -From c355523761598154653466033a1d88643d3fd3df Mon Sep 17 00:00:00 2001 -From: jikui -Date: Thu, 4 Nov 2021 20:27:22 +0800 -Subject: [PATCH] kata-runtime: fix kata-runtime skip read lines in - /proc/mounts file problem - -Signed-off-by: jikui ---- - .../virtcontainers/utils/utils_linux.go | 61 +++++++++++-------- - 1 file changed, 36 insertions(+), 25 deletions(-) - -diff --git a/src/runtime/virtcontainers/utils/utils_linux.go b/src/runtime/virtcontainers/utils/utils_linux.go -index 3c14e0c..c6dbd0a 100644 ---- a/src/runtime/virtcontainers/utils/utils_linux.go -+++ b/src/runtime/virtcontainers/utils/utils_linux.go -@@ -7,15 +7,18 @@ package utils - - import ( - "bufio" -+ "bytes" - "crypto/rand" - "fmt" - "io" -+ "io/ioutil" - "math/big" - "os" - "strings" - "syscall" - "unsafe" - -+ "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - ) - -@@ -93,6 +96,7 @@ const ( - procMountsFile = "/proc/mounts" - - fieldsPerLine = 6 -+ maxRetryTimes = 5 - vfioAPSysfsDir = "vfio_ap" - ) - -@@ -110,37 +114,44 @@ func GetDevicePathAndFsType(mountPoint string) (devicePath, fsType string, err e - return - } - -- var file *os.File -+ var retry int = 0 - -- file, err = os.Open(procMountsFile) -- if err != nil { -- return -- } -- -- defer file.Close() -- -- reader := bufio.NewReader(file) -- for { -- var line string -- -- line, err = reader.ReadString('\n') -- if err == io.EOF { -- err = fmt.Errorf("Mount %s not found", mountPoint) -+ for retry <= maxRetryTimes { -+ var content []byte -+ content, err = ioutil.ReadFile(procMountsFile) -+ if err != nil { - return - } -- -- fields := strings.Fields(line) -- if len(fields) != fieldsPerLine { -- err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) -- return -+ bytesReader := bytes.NewReader(content) -+ reader := bufio.NewReader(bytesReader) -+ -+ for { -+ var line string -+ -+ line, err = reader.ReadString('\n') -+ if err == io.EOF { -+ err = fmt.Errorf("Mount %s not found", mountPoint) -+ break -+ } -+ -+ fields := strings.Fields(line) -+ if len(fields) != fieldsPerLine { -+ err = fmt.Errorf("Incorrect no of fields (expected %d, got %d)) :%s", fieldsPerLine, len(fields), line) -+ return -+ } -+ -+ if mountPoint == fields[procPathIndex] { -+ devicePath = fields[procDeviceIndex] -+ fsType = fields[procTypeIndex] -+ return -+ } - } -- -- if mountPoint == fields[procPathIndex] { -- devicePath = fields[procDeviceIndex] -- fsType = fields[procTypeIndex] -- return -+ retry = retry + 1 -+ if retry <= maxRetryTimes { -+ logrus.Warnf("can not find %s in %s, retry %d times again......", mountPoint, procMountsFile, retry) - } - } -+ return "", "", fmt.Errorf("retry %d times fail to get devicePath adn fs type", maxRetryTimes) - } - - // IsAPVFIOMediatedDevice decides whether a device is a VFIO-AP device --- -2.25.1 - diff --git a/patches/0011-kata-containers-keep-the-process-name-of-qemu-same-as-c.patch b/patches/0011-kata-containers-keep-the-process-name-of-qemu-same-as-c.patch deleted file mode 100644 index b67ab04926499856bddfcb72fc31c0e44d1cda91..0000000000000000000000000000000000000000 --- a/patches/0011-kata-containers-keep-the-process-name-of-qemu-same-as-c.patch +++ /dev/null @@ -1,52 +0,0 @@ -From 0797bf1fec9c40b67f2770bc8778e8eaee1657c8 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 11:35:25 +0800 -Subject: [PATCH] kata-runtime: keep the process name of qemu same as - configured path - -Signed-off-by: jikui ---- - src/runtime/pkg/katautils/config.go | 7 ++++++- - src/runtime/pkg/katautils/config_test.go | 4 ++-- - 2 files changed, 8 insertions(+), 3 deletions(-) - -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index 718677b..e0ebc84 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -172,7 +172,12 @@ func (h hypervisor) path() (string, error) { - p = defaultHypervisorPath - } - -- return ResolvePath(p) -+ absolutePath, err := filepath.Abs(p) -+ if err != nil { -+ return "", err -+ } -+ -+ return absolutePath, nil - } - - func (h hypervisor) ctlpath() (string, error) { -diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go -index 0d02534..84b7843 100644 ---- a/src/runtime/pkg/katautils/config_test.go -+++ b/src/runtime/pkg/katautils/config_test.go -@@ -983,12 +983,12 @@ func TestHypervisorDefaultsHypervisor(t *testing.T) { - assert.NoError(err) - assert.Equal(p, defaultHypervisorPath, "default hypervisor path wrong") - -- // test path resolution -+ // test path resolution, just return the absolute path instead of resolved path - defaultHypervisorPath = testHypervisorLinkPath - h = hypervisor{} - p, err = h.path() - assert.NoError(err) -- assert.Equal(p, testHypervisorPath) -+ assert.Equal(p, testHypervisorLinkPath) - } - - func TestHypervisorDefaultsKernel(t *testing.T) { --- -2.25.1 - diff --git a/patches/0012-kata-containers-modify-kernel-and-image-path-in-conf.patch b/patches/0012-kata-containers-modify-kernel-and-image-path-in-conf.patch deleted file mode 100644 index 67e6c054c2154e34df864a22eab5925dbf354492..0000000000000000000000000000000000000000 --- a/patches/0012-kata-containers-modify-kernel-and-image-path-in-conf.patch +++ /dev/null @@ -1,56 +0,0 @@ -From e8e8e05538bf2c7bd8feebc44d4a960f453d21e1 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Sat, 20 Nov 2021 17:04:39 +0800 -Subject: [PATCH] kata-containers: modify kernel and image path in - configuration.toml - -Signed-off-by: holyfei ---- - src/runtime/Makefile | 10 +++++----- - 1 file changed, 5 insertions(+), 5 deletions(-) - -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index 745bcc1..68e60ee 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -49,6 +49,8 @@ BINLIBEXECLIST := - BIN_PREFIX = $(PROJECT_TYPE) - PROJECT_DIR = $(PROJECT_TAG) - IMAGENAME = $(PROJECT_TAG).img -+KERNEL_PATH = /var/lib/kata/kernel -+IMAGE_PATH = /var/lib/kata/kata-containers-initrd.img - - TARGET = $(BIN_PREFIX)-runtime - TARGET_OUTPUT = $(CURDIR)/$(TARGET) -@@ -108,7 +110,7 @@ PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR) - - KERNELDIR := $(PKGDATADIR) - --IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME) -+IMAGEPATH := $(IMAGE_PATH) - FIRMWAREPATH := - - # Name of default configuration file the runtime will use. -@@ -248,8 +250,7 @@ ifneq (,$(QEMUCMD)) - DEFBLOCKSTORAGEDRIVER_QEMU := virtio-scsi - DEFNETWORKMODEL_QEMU := tcfilter - KERNELTYPE = uncompressed -- KERNELNAME = $(call MAKE_KERNEL_NAME,$(KERNELTYPE)) -- KERNELPATH = $(KERNELDIR)/$(KERNELNAME) -+ KERNELPATH = $(KERNEL_PATH) - endif - - ifneq (,$(CLHCMD)) -@@ -294,8 +295,7 @@ ifneq (,$(STRATOVIRTCMD)) - DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio - DEFNETWORKMODEL_STRATOVIRT := none - KENRELTYPE_STRATOVIRT = uncompressed -- KERNEL_NAME_STRATOVIRT = $(call MAKE_KERNEL_NAME,$(KENRELTYPE_STRATOVIRT)) -- KERNELPATH_STRATOVIRT = $(KERNELDIR)/$(KERNEL_NAME_STRATOVIRT) -+ KERNELPATH_STRATOVIRT = $(KERNEL_PATH) - endif - - ifneq (,$(FCCMD)) --- -2.27.0 - diff --git a/patches/0013-kata-containers-increase-delete-cgroup-retry-times.patch b/patches/0013-kata-containers-increase-delete-cgroup-retry-times.patch deleted file mode 100644 index b84917897a33f651213edf219c95c9789862dd9a..0000000000000000000000000000000000000000 --- a/patches/0013-kata-containers-increase-delete-cgroup-retry-times.patch +++ /dev/null @@ -1,63 +0,0 @@ -From 13aec526360797b3bce776f35b7e5f5976961b47 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 11:46:17 +0800 -Subject: [PATCH] kata-runtime: increase delete cgroup retry times - -Signed-off-by: jikui ---- - .../vendor/github.com/containerd/cgroups/cgroup.go | 4 ++-- - .../vendor/github.com/containerd/cgroups/utils.go | 9 ++++++--- - 2 files changed, 8 insertions(+), 5 deletions(-) - -diff --git a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go -index 5386668..69612b0 100644 ---- a/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go -+++ b/src/runtime/vendor/github.com/containerd/cgroups/cgroup.go -@@ -223,7 +223,7 @@ func (c *cgroup) Delete() error { - return err - } - if err := d.Delete(sp); err != nil { -- errors = append(errors, string(s.Name())) -+ errors = append(errors, fmt.Sprintf("delete %s get error: %v", string(s.Name()), err.Error())) - } - continue - } -@@ -234,7 +234,7 @@ func (c *cgroup) Delete() error { - } - path := p.Path(sp) - if err := remove(path); err != nil { -- errors = append(errors, path) -+ errors = append(errors, fmt.Sprintf("remove path %s get error: %v", path, err.Error())) - } - } - } -diff --git a/src/runtime/vendor/github.com/containerd/cgroups/utils.go b/src/runtime/vendor/github.com/containerd/cgroups/utils.go -index 8a97d04..82dbe2d 100644 ---- a/src/runtime/vendor/github.com/containerd/cgroups/utils.go -+++ b/src/runtime/vendor/github.com/containerd/cgroups/utils.go -@@ -99,16 +99,19 @@ func defaults(root string) ([]Subsystem, error) { - // retrying the remove after a exp timeout - func remove(path string) error { - delay := 10 * time.Millisecond -- for i := 0; i < 5; i++ { -+ var err error -+ var count int = 0 -+ for i := 0; i < 10; i++ { - if i != 0 { - time.Sleep(delay) - delay *= 2 - } -- if err := os.RemoveAll(path); err == nil { -+ if err = os.RemoveAll(path); err == nil { - return nil - } -+ count++ - } -- return fmt.Errorf("cgroups: unable to remove path %q", path) -+ return fmt.Errorf("cgroups: unable to remove path %q, err: %v, count:%d", path, err, count) - } - - // readPids will read all the pids of processes in a cgroup by the provided path --- -2.25.1 - diff --git a/patches/0014-kata-containers-fix-umount-container-rootfs-dir-return-.patch b/patches/0014-kata-containers-fix-umount-container-rootfs-dir-return-.patch deleted file mode 100644 index 0868201660177763629393483663dc618ac4e781..0000000000000000000000000000000000000000 --- a/patches/0014-kata-containers-fix-umount-container-rootfs-dir-return-.patch +++ /dev/null @@ -1,33 +0,0 @@ -From b654687f88281d1dfbbb6c8ede65aa53ec105122 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 11:55:24 +0800 -Subject: [PATCH] kata-runtime: fix umount container rootfs dir return invalid - argument error - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/container.go | 8 ++++++-- - 1 file changed, 6 insertions(+), 2 deletions(-) - -diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go -index 6e8e1ba..224f096 100644 ---- a/src/runtime/virtcontainers/container.go -+++ b/src/runtime/virtcontainers/container.go -@@ -1035,8 +1035,12 @@ func (c *Container) stop(ctx context.Context, force bool) error { - return err - } - -- if err := bindUnmountContainerRootfs(ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { -- return err -+ // umount container rootfs dir only if container use 9p -+ // to bind mount host container rootfs to 9p shared dir -+ if c.state.BlockDeviceID == "" { -+ if err := bindUnmountContainerRootfs(c.ctx, getMountPath(c.sandbox.id), c.id); err != nil && !force { -+ return err -+ } - } - - if err := c.detachDevices(ctx); err != nil && !force { --- -2.25.1 - diff --git a/patches/0015-kata-containers-truncate-the-log.json-file-before-kata-.patch b/patches/0015-kata-containers-truncate-the-log.json-file-before-kata-.patch deleted file mode 100644 index adcba1a54be4ad7fc81c09554c52bd0dd92a2d36..0000000000000000000000000000000000000000 --- a/patches/0015-kata-containers-truncate-the-log.json-file-before-kata-.patch +++ /dev/null @@ -1,33 +0,0 @@ -From fe490b4d63871f91c17ad72afef38a3227c8b4d3 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 12:06:11 +0800 -Subject: [PATCH] kata-runtime: truncate the log.json file before kata-runtime - subcommand executed - -Signed-off-by: jikui ---- - src/runtime/cli/main.go | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/runtime/cli/main.go b/src/runtime/cli/main.go -index 27757c0..7c77764 100644 ---- a/src/runtime/cli/main.go -+++ b/src/runtime/cli/main.go -@@ -252,6 +252,14 @@ func beforeSubcommands(c *cli.Context) error { - ignoreConfigLogs = true - } else { - if path := c.GlobalString("log"); path != "" { -+ // since we have redirect the kata-runtime log to /var/log/messages, and avoid the -+ // path of log.json file to be large in the tmpfs, so we truncate the log.json file -+ // every time before subcommand is executed. -+ if path != "/dev/null" && katautils.FileExists(path) { -+ if err := os.Truncate(path, 0); err != nil { -+ return err -+ } -+ } - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640) - if err != nil { - return err --- -2.25.1 - diff --git a/patches/0016-kata-containers-validate-sandbox-cpu-and-memory-size.patch b/patches/0016-kata-containers-validate-sandbox-cpu-and-memory-size.patch deleted file mode 100644 index 66ef95e62b8b1cecd39846d396c733dcf4e04517..0000000000000000000000000000000000000000 --- a/patches/0016-kata-containers-validate-sandbox-cpu-and-memory-size.patch +++ /dev/null @@ -1,248 +0,0 @@ -From 420cb51b47e19556c35423354102fdc3a4d041f0 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 16:29:40 +0800 -Subject: [PATCH] kata-runtime: validate sandbox cpu and memory size - -Signed-off-by: jikui ---- - src/runtime/pkg/katautils/config.go | 39 +++++++++++++++++-- - src/runtime/pkg/katautils/config_test.go | 5 +-- - src/runtime/virtcontainers/pkg/oci/utils.go | 10 ++--- - src/runtime/virtcontainers/utils/utils.go | 42 +++++++++++++++++++++ - 4 files changed, 85 insertions(+), 11 deletions(-) - -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index e0ebc84..e523ed3 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -11,7 +11,6 @@ import ( - "fmt" - "io/ioutil" - "path/filepath" -- goruntime "runtime" - "strings" - - "github.com/BurntSushi/toml" -@@ -311,7 +310,7 @@ func (h hypervisor) GetEntropySource() string { - } - - func (h hypervisor) defaultVCPUs() uint32 { -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - if h.NumVCPUs < 0 || h.NumVCPUs > int32(numCPUs) { - return uint32(numCPUs) -@@ -323,8 +322,22 @@ func (h hypervisor) defaultVCPUs() uint32 { - return uint32(h.NumVCPUs) - } - -+func (h hypervisor) checkVCPUs() error { -+ numCPUs := utils.GetPhysicalCPUNumber() -+ -+ if h.NumVCPUs <= 0 { -+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must larger than 0") -+ } -+ -+ if h.NumVCPUs > int32(numCPUs) { -+ return fmt.Errorf("invalid vcpus in configuration.toml! vcpus must smaller than max CPUs: %d in machine", numCPUs) -+ } -+ -+ return nil -+} -+ - func (h hypervisor) defaultMaxVCPUs() uint32 { -- numcpus := uint32(goruntime.NumCPU()) -+ numcpus := uint32(utils.GetPhysicalCPUNumber()) - maxvcpus := vc.MaxQemuVCPUs() - reqVCPUs := h.DefaultMaxVCPUs - -@@ -350,6 +363,18 @@ func (h hypervisor) defaultMemSz() uint32 { - return h.MemorySize - } - -+func (h hypervisor) checkMemSz() error { -+ if h.MemorySize < utils.MinMemorySizeInMB { -+ return fmt.Errorf("invalid memory size! Memory size must larger than %d MB", utils.MinMemorySizeInMB) -+ } -+ -+ if h.MemorySize > utils.MaxMemorySizeInMB { -+ return fmt.Errorf("invalid memory size, memory size must smaller than %d MB", utils.MaxMemorySizeInMB) -+ } -+ -+ return nil -+} -+ - func (h hypervisor) defaultMemSlots() uint32 { - slots := h.MemSlots - if slots == 0 { -@@ -665,6 +690,14 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - return vc.HypervisorConfig{}, err - } - -+ if err = h.checkVCPUs(); err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ -+ if err = h.checkMemSz(); err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ - rxRateLimiterMaxRate := h.getRxRateLimiterCfg() - txRateLimiterMaxRate := h.getTxRateLimiterCfg() - -diff --git a/src/runtime/pkg/katautils/config_test.go b/src/runtime/pkg/katautils/config_test.go -index 84b7843..3782268 100644 ---- a/src/runtime/pkg/katautils/config_test.go -+++ b/src/runtime/pkg/katautils/config_test.go -@@ -14,7 +14,6 @@ import ( - "path" - "path/filepath" - "reflect" -- goruntime "runtime" - "strings" - "syscall" - "testing" -@@ -155,7 +154,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), - HypervisorMachineType: machineType, - NumVCPUs: defaultVCPUCount, -- DefaultMaxVCPUs: uint32(goruntime.NumCPU()), -+ DefaultMaxVCPUs: uint32(utils.GetPhysicalCPUNumber()), - MemorySize: defaultMemSize, - DisableBlockDeviceUse: disableBlockDevice, - BlockDeviceDriver: defaultBlockDeviceDriver, -@@ -918,7 +917,7 @@ func TestNewClhHypervisorConfig(t *testing.T) { - func TestHypervisorDefaults(t *testing.T) { - assert := assert.New(t) - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - h := hypervisor{} - -diff --git a/src/runtime/virtcontainers/pkg/oci/utils.go b/src/runtime/virtcontainers/pkg/oci/utils.go -index efaee4a..ea46ab7 100644 ---- a/src/runtime/virtcontainers/pkg/oci/utils.go -+++ b/src/runtime/virtcontainers/pkg/oci/utils.go -@@ -12,15 +12,12 @@ import ( - "fmt" - "path/filepath" - "regexp" -- goruntime "runtime" - "strconv" - "strings" - "syscall" - - criContainerdAnnotations "github.com/containerd/cri-containerd/pkg/annotations" - crioAnnotations "github.com/cri-o/cri-o/pkg/annotations" -- specs "github.com/opencontainers/runtime-spec/specs-go" -- "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/api/resource" - - vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" -@@ -29,6 +26,9 @@ import ( - vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" - dockershimAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" -+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" -+ specs "github.com/opencontainers/runtime-spec/specs-go" -+ "github.com/sirupsen/logrus" - ) - - type annotationContainerType struct { -@@ -656,7 +656,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - return fmt.Errorf("Error encountered parsing annotation default_vcpus: %v, please specify numeric value", err) - } - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - - if uint32(vcpus) > uint32(numCPUs) { - return fmt.Errorf("Number of cpus %d specified in annotation default_vcpus is greater than the number of CPUs %d on the system", vcpus, numCPUs) -@@ -671,7 +671,7 @@ func addHypervisorCPUOverrides(ocispec specs.Spec, sbConfig *vc.SandboxConfig) e - return fmt.Errorf("Error encountered parsing annotation for default_maxvcpus: %v, please specify positive numeric value", err) - } - -- numCPUs := goruntime.NumCPU() -+ numCPUs := utils.GetPhysicalCPUNumber() - max := uint32(maxVCPUs) - - if max > uint32(numCPUs) { -diff --git a/src/runtime/virtcontainers/utils/utils.go b/src/runtime/virtcontainers/utils/utils.go -index e49f55a..7023318 100644 ---- a/src/runtime/virtcontainers/utils/utils.go -+++ b/src/runtime/virtcontainers/utils/utils.go -@@ -6,12 +6,14 @@ - package utils - - import ( -+ "bufio" - "crypto/rand" - "errors" - "fmt" - "os" - "os/exec" - "path/filepath" -+ "strings" - "syscall" - "time" - -@@ -28,11 +30,26 @@ const fileMode0755 = os.FileMode(0755) - // MibToBytesShift the number to shift needed to convert MiB to Bytes - const MibToBytesShift = 20 - -+const ( -+ // Min needed memory size to start a Kata VM -+ MinMemorySizeInMB = 300 -+ MinMemorySizeInByte = MinMemorySizeInMB << MibToBytesShift -+ -+ // Max support memory size in the Kata VM -+ MaxMemorySizeInMB = 512 * 1024 -+ MaxMemorySizeInByte = MaxMemorySizeInMB << MibToBytesShift -+) -+ - // MaxSocketPathLen is the effective maximum Unix domain socket length. - // - // See unix(7). - const MaxSocketPathLen = 107 - -+const ( -+ procCPUInfoPath = "/proc/cpuinfo" -+ processorIdentifier = "processor" -+) -+ - // VHostVSockDevicePath path to vhost-vsock device - var VHostVSockDevicePath = "/dev/vhost-vsock" - -@@ -390,3 +407,28 @@ outer: - - return nil - } -+ -+// GetPhysicalCPUNumber return the number of the CPUs in the physical machine -+func GetPhysicalCPUNumber() int { -+ f, err := os.Open(procCPUInfoPath) -+ if err != nil { -+ return 0 -+ } -+ defer f.Close() -+ -+ cpuNum := 0 -+ s := bufio.NewScanner(f) -+ for s.Scan() { -+ if err := s.Err(); err != nil { -+ return 0 -+ } -+ -+ fields := strings.Fields(s.Text()) -+ if len(fields) > 0 { -+ if fields[0] == processorIdentifier { -+ cpuNum++ -+ } -+ } -+ } -+ return cpuNum -+} --- -2.25.1 - diff --git a/patches/0017-kata-containers-fix-delete-sandbox-failed-problem.patch b/patches/0017-kata-containers-fix-delete-sandbox-failed-problem.patch deleted file mode 100644 index d42e9f853feadd9158b5e774844f18b13892776e..0000000000000000000000000000000000000000 --- a/patches/0017-kata-containers-fix-delete-sandbox-failed-problem.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 514a0a19e5f37458d1a7e44c8b2fb4bff3b12d5b Mon Sep 17 00:00:00 2001 -From: jikui -Date: Fri, 5 Nov 2021 16:40:05 +0800 -Subject: [PATCH] kata-runtime: fix delete sandbox failed problem - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/sandbox.go | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go -index 33ae6f0..09cb2eb 100644 ---- a/src/runtime/virtcontainers/sandbox.go -+++ b/src/runtime/virtcontainers/sandbox.go -@@ -1114,6 +1114,8 @@ func (s *Sandbox) CreateContainer(ctx context.Context, contConfig ContainerConfi - if len(s.config.Containers) > 0 { - // delete container config - s.config.Containers = s.config.Containers[:len(s.config.Containers)-1] -+ // need to flush change to persist storage -+ _ = s.storeSandbox(ctx) - } - } - }() --- -2.25.1 - diff --git a/patches/0018-kata-containers-check-VFIO-when-create-device.patch b/patches/0018-kata-containers-check-VFIO-when-create-device.patch deleted file mode 100644 index f858aab0863712f71d8a26ef91d4a6d3ab5b985b..0000000000000000000000000000000000000000 --- a/patches/0018-kata-containers-check-VFIO-when-create-device.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 8251be558d6824fa1dce37836b7f1d6ec6be6e9f Mon Sep 17 00:00:00 2001 -From: jikui -Date: Tue, 30 Nov 2021 10:36:27 +0800 -Subject: [PATCH] kata-runtime: check VFIO when create device - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/device/manager/manager.go | 7 ++++++- - 1 file changed, 6 insertions(+), 1 deletion(-) - -diff --git a/src/runtime/virtcontainers/device/manager/manager.go b/src/runtime/virtcontainers/device/manager/manager.go -index 4515609..3afc148 100644 ---- a/src/runtime/virtcontainers/device/manager/manager.go -+++ b/src/runtime/virtcontainers/device/manager/manager.go -@@ -10,6 +10,7 @@ import ( - "context" - "encoding/hex" - "errors" -+ "fmt" - "sync" - - "github.com/sirupsen/logrus" -@@ -116,7 +117,11 @@ func (dm *deviceManager) createDevice(devInfo config.DeviceInfo) (dev api.Device - }() - - if existingDev := dm.findDeviceByMajorMinor(devInfo.Major, devInfo.Minor); existingDev != nil { -- return existingDev, nil -+ if isVFIO(devInfo.HostPath) { -+ return nil, fmt.Errorf("device %s is replicated in the same Pod!", devInfo.ContainerPath) -+ } else { -+ return existingDev, nil -+ } - } - - // device ID must be generated by manager instead of device itself --- -2.25.1 - diff --git a/patches/0019-kata-containers-do-not-delete-the-exist-tap-device-in-t.patch b/patches/0019-kata-containers-do-not-delete-the-exist-tap-device-in-t.patch deleted file mode 100644 index f2824a23c15767524c3e76321fc4c21c8b98f8c6..0000000000000000000000000000000000000000 --- a/patches/0019-kata-containers-do-not-delete-the-exist-tap-device-in-t.patch +++ /dev/null @@ -1,76 +0,0 @@ -From 51e17297b61fdb9ec560bae2f6c36003c8ebdfc0 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Tue, 30 Nov 2021 11:29:50 +0800 -Subject: [PATCH] kata-runtime: do not delete the exist tap device in the host - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/tap_endpoint.go | 16 +++++++++++++--- - src/runtime/virtcontainers/veth_endpoint.go | 3 +++ - 2 files changed, 16 insertions(+), 3 deletions(-) - -diff --git a/src/runtime/virtcontainers/tap_endpoint.go b/src/runtime/virtcontainers/tap_endpoint.go -index 9617945..5eafd15 100644 ---- a/src/runtime/virtcontainers/tap_endpoint.go -+++ b/src/runtime/virtcontainers/tap_endpoint.go -@@ -80,7 +80,7 @@ func (endpoint *TapEndpoint) Detach(ctx context.Context, netNsCreated bool, netN - - networkLogger().WithField("endpoint-type", TapEndpointType).Info("Detaching endpoint") - return doNetNS(netNsPath, func(_ ns.NetNS) error { -- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) -+ return unTapNetwork(endpoint) - }) - } - -@@ -94,6 +94,9 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error - - if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error attach tap ep") -+ if errUnTap := unTapNetwork(endpoint); errUnTap != nil { -+ networkLogger().WithError(errUnTap).Errorf("Error rollback tap %s", endpoint.TapInterface.TAPIface.Name) -+ } - return err - } - return nil -@@ -103,7 +106,7 @@ func (endpoint *TapEndpoint) HotAttach(ctx context.Context, h hypervisor) error - func (endpoint *TapEndpoint) HotDetach(ctx context.Context, h hypervisor, netNsCreated bool, netNsPath string) error { - networkLogger().Info("Hot detaching tap endpoint") - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { -- return unTapNetwork(endpoint.TapInterface.TAPIface.Name) -+ return unTapNetwork(endpoint) - }); err != nil { - networkLogger().WithError(err).Warn("Error un-bridging tap ep") - } -@@ -174,7 +177,14 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err - return nil - } - --func unTapNetwork(name string) error { -+func unTapNetwork(endpoint *TapEndpoint) error { -+ // length of VMFDs == 0 means that the endpoint is already exist in the host, -+ // no created by kata, so we don't need to remove it when detach -+ if len(endpoint.TapInterface.VMFds) == 0 { -+ return nil -+ } -+ -+ name := endpoint.TapInterface.TAPIface.Name - netHandle, err := netlink.NewHandle() - if err != nil { - return err -diff --git a/src/runtime/virtcontainers/veth_endpoint.go b/src/runtime/virtcontainers/veth_endpoint.go -index f93ca21..5f2435c 100644 ---- a/src/runtime/virtcontainers/veth_endpoint.go -+++ b/src/runtime/virtcontainers/veth_endpoint.go -@@ -124,6 +124,9 @@ func (endpoint *VethEndpoint) HotAttach(ctx context.Context, h hypervisor) error - - if _, err := h.hotplugAddDevice(ctx, endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error attach virtual ep") -+ if errDisconn := xDisconnectVMNetwork(endpoint); errDisconn != nil { -+ networkLogger().WithError(errDisconn).Error("Error rollback virtual ep") -+ } - return err - } - return nil --- -2.25.1 - diff --git a/patches/0020-kata-containers-do-not-ignore-updateInterface-return-er.patch b/patches/0020-kata-containers-do-not-ignore-updateInterface-return-er.patch deleted file mode 100644 index 26e407ef0f852ca9276e0816fb652aa3dce1f1ff..0000000000000000000000000000000000000000 --- a/patches/0020-kata-containers-do-not-ignore-updateInterface-return-er.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 39ac2f6929aea7e404c4597683e43fba7949964c Mon Sep 17 00:00:00 2001 -From: jikui -Date: Tue, 30 Nov 2021 11:47:36 +0800 -Subject: [PATCH] kata-runtime: do not ignore updateInterface return error - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/kata_agent.go | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go -index 607f1c8..19d09bb 100644 ---- a/src/runtime/virtcontainers/kata_agent.go -+++ b/src/runtime/virtcontainers/kata_agent.go -@@ -620,7 +620,15 @@ func (k *kataAgent) updateInterface(ctx context.Context, ifc *pbTypes.Interface) - "interface-requested": fmt.Sprintf("%+v", ifc), - "resulting-interface": fmt.Sprintf("%+v", resultingInterface), - }).WithError(err).Error("update interface request failed") -+ return nil, err -+ } -+ -+ // need to judege resultingInterface is not nil, otherwise may cause -+ // deference nil pointer panic problem -+ if resultingInterface == nil { -+ return nil, fmt.Errorf("resultingInterface should not be nil") - } -+ - if resultInterface, ok := resultingInterface.(*pbTypes.Interface); ok { - return resultInterface, err - } --- -2.25.1 - diff --git a/patches/0021-kata-containers-fix-the-block-device-not-remove-in-devM.patch b/patches/0021-kata-containers-fix-the-block-device-not-remove-in-devM.patch deleted file mode 100644 index 151dd6f600fb1297e4f934e38a0a28203a80e5d8..0000000000000000000000000000000000000000 --- a/patches/0021-kata-containers-fix-the-block-device-not-remove-in-devM.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 1bb584ac783d057b64058b02cc96c1c67dd7bf30 Mon Sep 17 00:00:00 2001 -From: jikui -Date: Tue, 30 Nov 2021 12:02:42 +0800 -Subject: [PATCH] kata-runtime: fix the block device not remove in devManager - -Signed-off-by: jikui ---- - src/runtime/virtcontainers/container.go | 11 ++++++++++- - 1 file changed, 10 insertions(+), 1 deletion(-) - -diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go -index 224f096..dd4317e 100644 ---- a/src/runtime/virtcontainers/container.go -+++ b/src/runtime/virtcontainers/container.go -@@ -1302,6 +1302,7 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error { - } - - if c.checkBlockDeviceSupport(ctx) && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { -+ var err error - b, err := c.sandbox.devManager.NewDevice(config.DeviceInfo{ - HostPath: devicePath, - ContainerPath: filepath.Join(kataGuestSharedDir(), c.id), -@@ -1313,10 +1314,18 @@ func (c *Container) plugDevice(ctx context.Context, devicePath string) error { - return fmt.Errorf("device manager failed to create rootfs device for %q: %v", devicePath, err) - } - -+ defer func() { -+ if err != nil { -+ if newErr := c.sandbox.devManager.RemoveDevice(b.DeviceID()); newErr != nil { -+ c.Logger().WithError(newErr).Error("fail rollback to remove block device") -+ } -+ } -+ }() -+ - c.state.BlockDeviceID = b.DeviceID() - - // attach rootfs device -- if err := c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil { -+ if err = c.sandbox.devManager.AttachDevice(ctx, b.DeviceID(), c.sandbox); err != nil { - return err - } - } --- -2.25.1 - diff --git a/patches/0022-kata-containers-modify-stratovirt-config-file.patch b/patches/0022-kata-containers-modify-stratovirt-config-file.patch deleted file mode 100644 index d8dc01dbe034c2882aed2d517b72859fcbac0ef9..0000000000000000000000000000000000000000 --- a/patches/0022-kata-containers-modify-stratovirt-config-file.patch +++ /dev/null @@ -1,40 +0,0 @@ -From c9e8df6902ecd72223a3837de953233f00f94093 Mon Sep 17 00:00:00 2001 -From: holyfei -Date: Fri, 10 Dec 2021 09:43:30 +0800 -Subject: [PATCH] kata-containers: modify stratovirt config file - -Signed-off-by: holyfei ---- - src/runtime/Makefile | 2 +- - src/runtime/cli/config/configuration-stratovirt.toml.in | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index 68e60ee..ff93ab3 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -293,7 +293,7 @@ ifneq (,$(STRATOVIRTCMD)) - - # stratovirt-specific options (all should be suffixed by "_STRATOVIRT") - DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio -- DEFNETWORKMODEL_STRATOVIRT := none -+ DEFNETWORKMODEL_STRATOVIRT := tcfilter - KENRELTYPE_STRATOVIRT = uncompressed - KERNELPATH_STRATOVIRT = $(KERNEL_PATH) - endif -diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in -index b557b71..753e3dc 100644 ---- a/src/runtime/cli/config/configuration-stratovirt.toml.in -+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in -@@ -13,7 +13,7 @@ - [hypervisor.stratovirt] - path = "@STRATOVIRTPATH@" - kernel = "@KERNELPATH_STRATOVIRT@" --image = "@IMAGEPATH@" -+initrd = "@IMAGEPATH@" - - # List of valid annotation names for the hypervisor - # Each member of the list is a regular expression, which is the base name --- -2.27.0 - diff --git a/patches/0023-stratovirt-update-configuration-toml-file.patch b/patches/0023-stratovirt-update-configuration-toml-file.patch deleted file mode 100644 index 1206c3c628131c84dfcd66a1b9698f910dfbda25..0000000000000000000000000000000000000000 --- a/patches/0023-stratovirt-update-configuration-toml-file.patch +++ /dev/null @@ -1,243 +0,0 @@ -From 1f83147653208f01effab0cf89209b8454d15f03 Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Mon, 10 Jan 2022 10:49:44 +0800 -Subject: [PATCH 1/5] stratovirt: update configuration toml file - -1.Adapt to default machine type as microvm. -2.Add more configuration items. -3.Modify toml file format. - -Signed-off-by: Xinle.Guo ---- - src/runtime/Makefile | 2 + - .../config/configuration-stratovirt.toml.in | 67 ++++++++++++------- - 2 files changed, 46 insertions(+), 23 deletions(-) - -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index ff93ab3..bade196 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -292,6 +292,7 @@ ifneq (,$(STRATOVIRTCMD)) - CONFIGS += $(CONFIG_STRATOVIRT) - - # stratovirt-specific options (all should be suffixed by "_STRATOVIRT") -+ DEFMACHINETYPE_STRATOVIRT := microvm - DEFBLOCKSTORAGEDRIVER_STRATOVIRT := virtio-mmio - DEFNETWORKMODEL_STRATOVIRT := tcfilter - KENRELTYPE_STRATOVIRT = uncompressed -@@ -437,6 +438,7 @@ USER_VARS += FIRMWAREPATH - USER_VARS += MACHINEACCELERATORS - USER_VARS += CPUFEATURES - USER_VARS += DEFMACHINETYPE_CLH -+USER_VARS += DEFMACHINETYPE_STRATOVIRT - USER_VARS += KERNELPARAMS - USER_VARS += LIBEXECDIR - USER_VARS += LOCALSTATEDIR -diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in -index 753e3dc..db46665 100644 ---- a/src/runtime/cli/config/configuration-stratovirt.toml.in -+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in -@@ -14,6 +14,8 @@ - path = "@STRATOVIRTPATH@" - kernel = "@KERNELPATH_STRATOVIRT@" - initrd = "@IMAGEPATH@" -+#image = "/var/lib/kata/kata-containers-rootfs.img" -+machine_type = "@DEFMACHINETYPE_STRATOVIRT@" - - # List of valid annotation names for the hypervisor - # Each member of the list is a regular expression, which is the base name -@@ -29,12 +31,12 @@ valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ - # Path for the ozone specific to stratovirt - # If the ozone path is set, stratovirt will be launched in - # ozone secure environment. It is disabled by default. --# ozone_path = "@STRATOVIRTOZONEPATH@" -+#ozone_path = "@STRATOVIRTOZONEPATH@" - - # List of valid ozone path values for the hypervisor - # Each member of the list can be a regular expression - # The default if not set is empty (all annotations rejected.) --# valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@ -+#valid_jailer_paths = @STRATOVIRTVALIDOZONEPATHS@ - - # Optional space-separated list of options to pass to the guest kernel. - # For example, use `kernel_params = "vsyscall=emulate"` if you are having -@@ -87,18 +89,18 @@ default_bridges = @DEFBRIDGES@ - # Default memory size in MiB for SB/VM. - # If unspecified then it will be set @DEFMEMSZ@ MiB. - default_memory = @DEFMEMSZ@ --# -+ - # Default memory slots per SB/VM. - # If unspecified then it will be set @DEFMEMSLOTS@. - # This is will determine the times that memory will be hotadded to sandbox/VM. --# memory_slots = @DEFMEMSLOTS@ -+#memory_slots = @DEFMEMSLOTS@ - - # The size in MiB will be plused to max memory of hypervisor. - # It is the memory address space for the NVDIMM devie. - # If set block storage driver (block_device_driver) to "nvdimm", - # should set memory_offset to the size of block device. - # Default 0 --# memory_offset = 0 -+#memory_offset = 0 - - # Disable block device from being used for a container's rootfs. - # In case of a storage driver like devicemapper where a container's -@@ -108,6 +110,14 @@ default_memory = @DEFMEMSZ@ - # 9pfs is used instead to pass the rootfs. - disable_block_device_use = @DEFDISABLEBLOCK@ - -+# Shared file system type: -+# - virtio-fs (default) -+# - virtio-9p -+shared_fs = "virtio-fs" -+ -+# Path to vhost-user-fs daemon. -+virtio_fs_daemon = "/usr/bin/vhost_user_fs" -+ - # Block storage driver to be used for the hypervisor in case the container - # rootfs is backed by a block device. This is virtio-scsi, virtio-blk - # or nvdimm. -@@ -120,12 +130,17 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" - # Specifies cache-related options for block devices. - # Denotes whether use of O_DIRECT (bypass the host page cache) is enabled. - # Default false --# block_device_cache_direct = true -+#block_device_cache_direct = true - - # Specifies cache-related options for block devices. - # Denotes whether flush requests for the device are ignored. - # Default false --# block_device_cache_noflush = true -+#block_device_cache_noflush = true -+ -+# Enable iothreads to be used. This causes IO to be -+# handled in a separate IO thread. This is currently only implemented -+# for virtio blk. -+#enable_iothreads = true - - # Enable pre allocation of VM RAM, default false - # Enabling this will result in lower container density -@@ -134,7 +149,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" - # upfront or in the cases where you want memory latencies - # to be very predictable - # Default false --# enable_mem_prealloc = true -+#enable_mem_prealloc = true - - # Enable huge pages for VM RAM, default false - # Enabling this will result in the VM memory -@@ -142,42 +157,48 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" - # This is useful when you want to use vhost-user network - # stacks within the container. This will automatically - # result in memory pre allocation --# enable_hugepages = true -+#enable_hugepages = true - - # Enable vIOMMU, default false - # Enabling this will result in the VM having a vIOMMU device - # This will also add the following options to the kernel's - # command line: intel_iommu=on,iommu=pt --# enable_iommu = true -+#enable_iommu = true - - # Enable swap of vm memory. Default false. - # The behaviour is undefined if mem_prealloc is also set to true --# enable_swap = true -+#enable_swap = true - - # This option changes the default hypervisor and kernel parameters - # to enable debug output where available. - # - # Default false --# enable_debug = true -+#enable_debug = true - - # Disable the customizations done in the runtime when it detects - # that it is running on top a VMM. This will result in the runtime - # behaving as it would when running on bare metal. - # --# disable_nesting_checks = true -+#disable_nesting_checks = true - - # This is the msize used for 9p shares. It is the number of bytes - # used for 9p packet payload. --# msize_9p = -+#msize_9p = - - # VFIO devices are hotplugged on a bridge by default. - # Enable hotplugging on root bus. This may be required for devices with - # a large PCI bar, as this is a current limitation with hotplugging on - # a bridge. - # Default false --# hotplug_vfio_on_root_bus = true -+#hotplug_vfio_on_root_bus = true -+ -+# Before hot plugging a PCIe device, you need to add a pcie_root_port device. -+# Use this parameter when using some large PCI bar devices, such as Nvidia GPU -+# The value means the number of pcie_root_port -+# This value is valid when hotplug_vfio_on_root_bus is true and machine_type is "q35" -+# Default 0 -+pcie_root_port = 2 - --# - # Default entropy source. - # The path to a host source of entropy (including a real hardware RNG) - # /dev/urandom and /dev/random are two main options. -@@ -187,7 +208,7 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_STRATOVIRT@" - # The source of entropy /dev/urandom is non-blocking and provides a - # generally acceptable source of entropy. It should work well for pretty much - # all practical purposes. --# entropy_source= "" -+entropy_source= "@DEFENTROPYSOURCE@" - - # List of valid annotations values for entropy_source - # The default if not set is empty (all annotations rejected.) -@@ -209,7 +230,7 @@ valid_entropy_sources = @DEFVALIDENTROPYSOURCES@ - # https://github.com/opencontainers/runtime-spec/blob/v1.0.1/config.md#posix-platform-hooks - # Warnings will be logged if any error is encountered will scanning for hooks, - # but it will not abort container execution. --# guest_hook_path = "/usr/share/oci/hooks" -+#guest_hook_path = "/usr/share/oci/hooks" - - [factory] - # VM templating support. Once enabled, new VMs are created from template -@@ -312,14 +333,14 @@ path = "@NETMONPATH@" - # Uses tc filter rules to redirect traffic from the network interface - # provided by plugin to a tap interface connected to the VM. - # --internetworking_model="@DEFNETWORKMODEL_STRATOVIRT@" -+internetworking_model = "@DEFNETWORKMODEL_STRATOVIRT@" - - # disable guest seccomp - # Determines whether container seccomp profiles are passed to the virtual - # machine and applied by the kata agent. If set to true, seccomp is not applied - # within the guest - # (default: true) --disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ -+disable_guest_seccomp = @DEFDISABLEGUESTSECCOMP@ - - # If enabled, the runtime will create opentracing.io traces and spans. - # (See https://www.jaegertracing.io/docs/getting-started). -@@ -352,15 +373,15 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@ - # The sandbox cgroup path is the parent cgroup of a container with the PodSandbox annotation. - # The sandbox cgroup is constrained if there is no container type annotation. - # See: https://godoc.org/github.com/kata-containers/runtime/virtcontainers#ContainerType --sandbox_cgroup_only=@DEFSANDBOXCGROUPONLY@ -+sandbox_cgroup_only = @DEFSANDBOXCGROUPONLY@ - - # Enabled experimental feature list, format: ["a", "b"]. - # Experimental features are features not stable enough for production, - # they may break compatibility, and are prepared for a big version bump. - # Supported experimental features: - # (default: []) --experimental=@DEFAULTEXPFEATURES@ -+experimental = @DEFAULTEXPFEATURES@ - - # If enabled, user can run pprof tools with shim v2 process through kata-monitor. - # (default: false) --# enable_pprof = true -+#enable_pprof = true --- -2.20.1.windows.1 - diff --git a/patches/0024-stratovirt-add-struct-vmConfig-and-methods-to-get-al.patch b/patches/0024-stratovirt-add-struct-vmConfig-and-methods-to-get-al.patch deleted file mode 100644 index 7497b0ab6fe2282ae5fb2e8ce504b4f311cde757..0000000000000000000000000000000000000000 --- a/patches/0024-stratovirt-add-struct-vmConfig-and-methods-to-get-al.patch +++ /dev/null @@ -1,235 +0,0 @@ -From 37c4a009a6ea9028e237f849e7b27a15c602113e Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Tue, 11 Jan 2022 10:44:53 +0800 -Subject: [PATCH] stratovirt: add struct `vmConfig` and methods to get all - parameters of VM - -Defines `vmConfig` struct to containes all configuration items that -virtual machine needs. Provides methods to get VM paramters, -including name, UUID, cpu, memory, kernel, devices, etc. - -Signed-off-by: Xinle.Guo ---- - src/runtime/virtcontainers/stratovirt.go | 197 ++++++++++++++++++++++- - 1 file changed, 189 insertions(+), 8 deletions(-) - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index e9b2ba8..7e32a8a 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -26,15 +26,196 @@ import ( - ) - - const ( -- apiSocket = "qmp.socket" -- debugSocket = "console.socket" -- ozoneBaseDir = "/srv/ozone/stratovirt" -- defaultDummyMac = "22:33:44:aa:bb:" -- mmioBlkCount = 4 -- mmioNetCount = 2 -- randomDevice = "/dev/urandom" -+ defaultStratoVirt = "/usr/bin/stratovirt" -+ ozoneBaseDir = "/srv/ozone/stratovirt" -+ defaultStratoVirtMachineType = "microvm" -+ defaultKernelParames = "console=hvc0 reboot=k panic=1 agent.use_vsock=true ramdom.trust_cpu=on rw" -+ defaultMicroVMParames = "pci=off iommu=off acpi=off" -+ apiSocket = "qmp.socket" -+ debugSocket = "console.socket" -+ virtiofsSocket = "virtiofs_kata.sock" -+ iothreadID = "iothread_block" -+ mmioBlkCount = 4 -+ mmioNetCount = 2 - ) - -+const ( -+ WaitSandboxTimeoutSecs = 15 -+ MachineTypeMicrovm = "microvm" -+) -+ -+// VirtioDev is the StratoVirt device interface. -+type VirtioDev interface { -+ getParams(config *vmConfig) []string -+} -+ -+type inComing struct { -+ path string -+ bootFromTemplate bool -+} -+ -+type Ozone struct { -+ ozoneRoot string -+ ozoneRes []string -+ consolePath string -+ kernelPath string -+ initrdPath string -+ pidFile string -+ logFile string -+ qmpSocketPath string -+} -+ -+// vmConfig keeps the custom settings and paramters to start virtual machine. -+type vmConfig struct { -+ name string -+ uuid string -+ machineType string -+ vmPath string -+ smp uint32 -+ memory uint64 -+ kernelPath string -+ params string -+ rootfsPath string -+ initrdPath string -+ rootBus types.Bridge -+ devices []VirtioDev -+ IOThread bool -+ PFlash []string -+ pidFile string -+ logFile string -+ qmpSocketPath govmmQemu.QMPSocket -+ consolePath string -+ fsSockPath string -+ incoming inComing -+ daemonize bool -+ useOzone bool -+ Ozone Ozone -+} -+ -+func (c *vmConfig) appendName(params *[]string) { -+ if c.name == "" { -+ return -+ } -+ *params = append(*params, "-name", c.name) -+} -+ -+func (c *vmConfig) appendUUID(params *[]string) { -+ if c.uuid == "" { -+ return -+ } -+ *params = append(*params, "-uuid", c.uuid) -+} -+ -+func (c *vmConfig) appendMachine(params *[]string) { -+ if c.machineType == "" { -+ return -+ } -+ *params = append(*params, "-machine", fmt.Sprintf("type=%s,dump-guest-core=off,mem-share=on", c.machineType)) -+} -+ -+func (c *vmConfig) appendCPU(params *[]string) { -+ if c.smp == 0 { -+ return -+ } -+ *params = append(*params, "-smp", strconv.Itoa(int(c.smp))) -+} -+ -+func (c *vmConfig) appendMemory(params *[]string) { -+ if c.memory == 0 { -+ return -+ } -+ *params = append(*params, "-m", strconv.Itoa(int(c.memory))) -+} -+ -+func (c *vmConfig) appendKernel(params *[]string) { -+ var ozone Ozone -+ if c.kernelPath == "" { -+ return -+ } -+ -+ if c.useOzone { -+ ozone = c.Ozone -+ *params = append(*params, "-kernel", ozone.kernelPath) -+ } else { -+ *params = append(*params, "-kernel", c.kernelPath) -+ } -+ -+ if c.initrdPath != "" { -+ if c.useOzone { -+ *params = append(*params, "-initrd", ozone.initrdPath) -+ } else { -+ *params = append(*params, "-initrd", c.initrdPath) -+ } -+ } -+ -+ if c.params != "" { -+ *params = append(*params, "-append", c.params) -+ } -+} -+ -+func (c *vmConfig) appendIOThreads(params *[]string) { -+ if c.IOThread { -+ *params = append(*params, "-object", fmt.Sprintf("iothread,id=%s", iothreadID)) -+ } -+} -+ -+func (c *vmConfig) appendDevices(params *[]string) { -+ for _, d := range c.devices { -+ *params = append(*params, d.getParams(c)...) -+ } -+} -+ -+func (c *vmConfig) appendPFlash(params *[]string) { -+ for _, p := range c.PFlash { -+ if p != "" { -+ *params = append(*params, "-drive", p) -+ } -+ } -+} -+ -+func (c *vmConfig) appendPidFile(params *[]string) { -+ if c.pidFile == "" { -+ return -+ } -+ -+ if c.useOzone { -+ *params = append(*params, "-pidfile", c.Ozone.pidFile) -+ } else { -+ *params = append(*params, "-pidfile", c.pidFile) -+ } -+} -+ -+func (c *vmConfig) appendLogFile(params *[]string) { -+ if c.logFile == "" { -+ return -+ } -+ -+ if c.useOzone { -+ *params = append(*params, "-D", c.Ozone.logFile) -+ } else { -+ *params = append(*params, "-D", c.logFile) -+ } -+} -+ -+func (c *vmConfig) appendQMPSocket(params *[]string) { -+ qmpInfo := c.qmpSocketPath -+ qmpParams := append([]string{}, fmt.Sprintf("%s:", qmpInfo.Type)) -+ if c.useOzone { -+ qmpParams = append(qmpParams, c.Ozone.qmpSocketPath) -+ } else { -+ qmpParams = append(qmpParams, qmpInfo.Name) -+ } -+ qmpParams = append(qmpParams, ",server") -+ qmpParams = append(qmpParams, ",nowait") -+ *params = append(*params, "-qmp", strings.Join(qmpParams, "")) -+} -+ -+func (c *vmConfig) appendIncoming(params *[]string) { -+ if c.incoming.path != "" && c.incoming.bootFromTemplate { -+ *params = append(*params, "-incoming", fmt.Sprintf("file:%s", c.incoming.path)) -+ } -+} -+ - type stratovirtDev struct { - dev interface{} - devType deviceType -@@ -208,7 +389,7 @@ func (s *stratovirt) createbaseParams() []string { - params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) - params = append(params, "-device", "virtio-serial-device") - params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") -- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", randomDevice)) -+ params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", s.config.EntropySource)) - params = append(params, "-device", "virtio-rng-device,rng=objrng0") - - // daemonize --- -2.20.1.windows.1 - diff --git a/patches/0025-stratovirt-refactor-hypervisor-type-stratovirt-and-i.patch b/patches/0025-stratovirt-refactor-hypervisor-type-stratovirt-and-i.patch deleted file mode 100644 index 6881cad094efb0c405d5d6e838f6b3bf59e97d2f..0000000000000000000000000000000000000000 --- a/patches/0025-stratovirt-refactor-hypervisor-type-stratovirt-and-i.patch +++ /dev/null @@ -1,1449 +0,0 @@ -From 21b077875f35b92335dfa2891542ec2e418c386c Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Thu, 13 Jan 2022 09:39:27 +0800 -Subject: [PATCH] stratovirt: refactor hypervisor type `stratovirt` and its - methods - -1.For more flexible management of VM state and config, move parameters -to `State` and `vmConfig` struct. -2.Update `createSandbox()` and `startSandbox()` for further -encapsulation functions. -3.Create device structs and methods to store deivce data. - -Signed-off-by: Xinle.Guo ---- - src/runtime/virtcontainers/stratovirt.go | 1096 ++++++++++++++++------ - 1 file changed, 784 insertions(+), 312 deletions(-) - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 7e32a8a..4fcfb94 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -7,6 +7,7 @@ import ( - "os" - "os/exec" - "path/filepath" -+ "runtime" - "strconv" - "strings" - "syscall" -@@ -18,6 +19,7 @@ import ( - - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" -+ "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" - "go.opentelemetry.io/otel" -@@ -42,6 +44,8 @@ const ( - const ( - WaitSandboxTimeoutSecs = 15 - MachineTypeMicrovm = "microvm" -+ MmioBus VirtioDriver = "mmio" -+ PciBus VirtioDriver = "pci" - ) - - // VirtioDev is the StratoVirt device interface. -@@ -49,6 +53,281 @@ type VirtioDev interface { - getParams(config *vmConfig) []string - } - -+type VirtioDriver string -+ -+func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver { -+ switch runtime.GOARCH { -+ case "amd64", "arm64": -+ if config != nil && config.machineType == MachineTypeMicrovm { -+ return MmioBus -+ } -+ return PciBus -+ default: -+ return MmioBus -+ } -+} -+ -+type blkDevice struct { -+ id string -+ filePath string -+ driver VirtioDriver -+ deviceID string -+ bus string -+ addr string -+ iothread string -+} -+ -+var blkDriver = map[VirtioDriver]string{ -+ MmioBus: "virtio-blk-device", -+ PciBus: "virtio-blk-pci", -+} -+ -+func (b blkDevice) isVaild() bool { -+ if b.id == "" || b.filePath == "" || b.deviceID == "" { -+ return false -+ } -+ return true -+} -+ -+func (b blkDevice) getParams(config *vmConfig) []string { -+ if !b.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var driveParams []Param -+ var devParams []Param -+ -+ driveParams = append(driveParams, Param{"id", b.id}) -+ driveParams = append(driveParams, Param{"file", b.filePath}) -+ -+ b.driver = b.driver.getDriver(config) -+ driver := blkDriver[b.driver] -+ devParams = append(devParams, Param{"drive", b.id}) -+ devParams = append(devParams, Param{"id", b.deviceID}) -+ if b.bus != "" { -+ devParams = append(devParams, Param{"bus", b.bus}) -+ devParams = append(devParams, Param{"addr", b.addr}) -+ } -+ if b.iothread != "" { -+ devParams = append(devParams, Param{"iothread", b.iothread}) -+ } -+ -+ params = append(params, "-drive", strings.Join(SerializeParams(driveParams, "="), ",")) -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ return params -+} -+ -+type netDevice struct { -+ devType string -+ id string -+ ifname string -+ vhost bool -+ FDs []*os.File -+ VhostFDs []*os.File -+ driver VirtioDriver -+ netdev string -+ deviceID string -+ bus string -+ addr string -+ mac string -+} -+ -+var netDriver = map[VirtioDriver]string{ -+ MmioBus: "virtio-net-device", -+ PciBus: "virtio-net-pci", -+} -+ -+func (n netDevice) isVaild() bool { -+ if n.id == "" || n.netdev == "" || n.deviceID == "" { -+ return false -+ } -+ return true -+} -+ -+func (n netDevice) getParams(config *vmConfig) []string { -+ if !n.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var netdevParams []Param -+ var devParams []Param -+ -+ netdevParams = append(netdevParams, Param{"id", n.id}) -+ if n.ifname != "" { -+ netdevParams = append(netdevParams, Param{"ifname", n.ifname}) -+ } -+ if n.vhost { -+ netdevParams = append(netdevParams, Param{"vhost", "on"}) -+ } -+ -+ n.driver = n.driver.getDriver(config) -+ driver := netDriver[n.driver] -+ devParams = append(devParams, Param{"netdev", n.id}) -+ devParams = append(devParams, Param{"id", n.deviceID}) -+ if n.bus != "" { -+ devParams = append(devParams, Param{"bus", n.bus}) -+ devParams = append(devParams, Param{"addr", n.addr}) -+ } -+ if n.mac != "" { -+ devParams = append(devParams, Param{"mac", n.mac}) -+ } -+ -+ params = append(params, "-netdev", fmt.Sprintf("%s,%s", n.devType, strings.Join(SerializeParams(netdevParams, "="), ","))) -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ return params -+} -+ -+type vhostVsock struct { -+ driver VirtioDriver -+ id string -+ guestID string -+ bus string -+ addr string -+} -+ -+var vsockDriver = map[VirtioDriver]string{ -+ MmioBus: "vhost-vsock-device", -+ PciBus: "vhost-vsock-pci", -+} -+ -+func (v vhostVsock) isVaild() bool { -+ if v.id == "" || v.guestID == "" { -+ return false -+ } -+ return true -+} -+ -+func (v vhostVsock) getParams(config *vmConfig) []string { -+ if !v.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var devParams []Param -+ -+ v.driver = v.driver.getDriver(config) -+ driver := vsockDriver[v.driver] -+ devParams = append(devParams, Param{"id", v.id}) -+ devParams = append(devParams, Param{"guest-cid", v.guestID}) -+ if v.bus != "" { -+ devParams = append(devParams, Param{"bus", v.bus}) -+ devParams = append(devParams, Param{"addr", v.addr}) -+ } -+ -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ return params -+} -+ -+type rngDevice struct { -+ id string -+ fileName string -+ driver VirtioDriver -+ deviceID string -+ rng string -+ bus string -+ addr string -+} -+ -+var rngDriver = map[VirtioDriver]string{ -+ MmioBus: "virtio-rng-device", -+ PciBus: "virtio-rng-pci", -+} -+ -+func (r rngDevice) isVaild() bool { -+ if r.id == "" || r.rng == "" || r.fileName == "" || r.deviceID == "" { -+ return false -+ } -+ return true -+} -+ -+func (r rngDevice) getParams(config *vmConfig) []string { -+ if !r.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var objParams []Param -+ var devParams []Param -+ -+ objParams = append(objParams, Param{"id", r.id}) -+ objParams = append(objParams, Param{"filename", r.fileName}) -+ -+ r.driver = r.driver.getDriver(config) -+ driver := rngDriver[r.driver] -+ devParams = append(devParams, Param{"rng", r.rng}) -+ devParams = append(devParams, Param{"id", r.deviceID}) -+ if r.bus != "" { -+ devParams = append(devParams, Param{"bus", r.bus}) -+ devParams = append(devParams, Param{"addr", r.addr}) -+ } -+ -+ params = append(params, "-object", fmt.Sprintf("rng-random,%s", strings.Join(SerializeParams(objParams, "="), ","))) -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ return params -+} -+ -+type consoleDevice struct { -+ driver VirtioDriver -+ id string -+ bus string -+ addr string -+ backend string -+ charID string -+ devType string -+ charDev string -+ deviceID string -+} -+ -+var consoleDriver = map[VirtioDriver]string{ -+ MmioBus: "virtio-serial-device", -+ PciBus: "virtio-serial-pci", -+} -+ -+func (c consoleDevice) isVaild() bool { -+ if c.id == "" || c.charDev == "" { -+ return false -+ } -+ return true -+} -+ -+func (c consoleDevice) getParams(config *vmConfig) []string { -+ if !c.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var devParams []Param -+ var charParams []Param -+ var conParams []Param -+ -+ c.driver = c.driver.getDriver(config) -+ driver := consoleDriver[c.driver] -+ if c.id != "" { -+ devParams = append(devParams, Param{"id", c.id}) -+ } -+ if c.bus != "" { -+ devParams = append(devParams, Param{"bus", c.bus}) -+ devParams = append(devParams, Param{"addr", c.addr}) -+ } -+ -+ charParams = append(charParams, Param{"id", c.charID}) -+ if config.useOzone { -+ charParams = append(charParams, Param{"path", config.Ozone.consolePath}) -+ } else { -+ charParams = append(charParams, Param{"path", config.consolePath}) -+ } -+ -+ conParams = append(conParams, Param{"chardev", c.charDev}) -+ conParams = append(conParams, Param{"id", c.deviceID}) -+ -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", c.backend, strings.Join(SerializeParams(charParams, "="), ","))) -+ params = append(params, "-device", fmt.Sprintf("%s,%s", c.devType, strings.Join(SerializeParams(conParams, "="), ","))) -+ return params -+} -+ - type inComing struct { - path string - bootFromTemplate bool -@@ -132,7 +411,7 @@ func (c *vmConfig) appendKernel(params *[]string) { - if c.kernelPath == "" { - return - } -- -+ - if c.useOzone { - ozone = c.Ozone - *params = append(*params, "-kernel", ozone.kernelPath) -@@ -216,35 +495,26 @@ func (c *vmConfig) appendIncoming(params *[]string) { - } - } - --type stratovirtDev struct { -- dev interface{} -- devType deviceType -+// State keeps StratoVirt device and pids state. -+type State struct { -+ mmioBlkSlots [mmioBlkCount]bool -+ mmioNetSlots [mmioNetCount]bool -+ pid int -+ virtiofsPid int - } - -+// stratovirt struct is an Hypervisor interface implementation for the StratoVirt VMM. - type stratovirt struct { -- id string -- ctx context.Context -- sandbox *Sandbox -- store persistapi.PersistDriver -- config HypervisorConfig -- rootfsPath string -- kernelPath string -- templatePath string -- pid int -- consolePath string -- socketPath string -- netNSPath string -- qmpMonitorCh qmpChannel -- ozoneRoot string -- ozoneRes []string -- useOzone bool -- useImage bool -- pidfile string -- logfile string -- devices []stratovirtDev -- HotpluggedVCPUs []CPUDevice -- mmioBlkSlots [mmioBlkCount]bool -- mmioNetSlots [mmioNetCount]bool -+ id string -+ path string -+ ctx context.Context -+ sandbox *Sandbox -+ config HypervisorConfig -+ qmpMonitorCh qmpChannel -+ vmConfig vmConfig -+ store persistapi.PersistDriver -+ state State -+ netNSPath string - } - - func (s *stratovirt) Logger() *logrus.Entry { -@@ -258,102 +528,371 @@ func (s *stratovirt) trace(parent context.Context, name string) (otelTrace.Span, - } - - tracer := otel.Tracer("kata") -- ctx, span := tracer.Start(parent, name, otelTrace.WithAttributes(otelLabel.String("source", "runtime"), otelLabel.String("package", "virtcontainers"), otelLabel.String("subsystem", "hypervisor"), otelLabel.String("type", "stratovirt"), otelLabel.String("sandbox_id", s.id))) -+ ctx, span := tracer.Start(parent, name) -+ span.SetAttributes([]otelLabel.KeyValue{otelLabel.Key("subsystem").String("hypervisor"), otelLabel.Key("type").String("stratovirt")}...) - - return span, ctx - } - --func (s *stratovirt) getKernelCmdLine() string { -+func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (string, error) { - var params []string - -- if s.useImage { -+ if machineType == "microvm" { -+ params = append(params, defaultMicroVMParames) -+ } -+ // Take the default parameters. -+ params = append(params, defaultKernelParames) -+ -+ if initrdPath == "" { - params = append(params, "root=/dev/vda") - } - -- params = append(params, "pci=off") -- params = append(params, "reboot=k") -- params = append(params, "panic=1") -- params = append(params, "iommu=off") -- params = append(params, "acpi=off") -- params = append(params, "quiet") -- params = append(params, "agent.use_vsock=true") -- params = append(params, "random.trust_cpu=on") -- params = append(params, "rw") -- params = append(params, SerializeParams(s.config.KernelParams, "=")...) -+ if s.config.Debug { -+ params = append(params, "debug") -+ } else { -+ params = append(params, "quiet") -+ } -+ // Add extra kernel parameters from configuration file. -+ params = append(params, strings.Join(SerializeParams(s.config.KernelParams, "="), " ")) -+ -+ return strings.Join(params, " "), nil -+} -+ -+func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket { -+ socketPath := filepath.Join(vmPath, apiSocket) - -- return strings.Join(params, " ") -+ s.qmpMonitorCh = qmpChannel{ -+ ctx: s.ctx, -+ path: socketPath, -+ } -+ -+ return govmmQemu.QMPSocket{ -+ Type: "unix", -+ Name: s.qmpMonitorCh.path, -+ Server: true, -+ NoWait: true, -+ } - } - --func (s *stratovirt) hypervisorConfig() HypervisorConfig { -- return s.config -+func (s *stratovirt) createDevices() []VirtioDev { -+ var devices []VirtioDev -+ ctx := s.ctx -+ -+ // Set random device. -+ devices = s.appendRng(ctx, devices) -+ // Set serial console device. -+ devices = s.appendConsole(ctx, devices) -+ -+ if s.vmConfig.initrdPath == "" { -+ devices = s.appendBlock(ctx, devices) -+ if s.vmConfig.machineType == MachineTypeMicrovm { -+ s.state.mmioBlkSlots[0] = true -+ } -+ } -+ -+ return devices - } - --func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { -- s.ctx = ctx -+func (s *stratovirt) appendBlock(ctx context.Context, devices []VirtioDev) []VirtioDev { -+ var bus string -+ var addr uint32 -+ var err error -+ var iothread string - -- var span otelTrace.Span -- span, _ = s.trace(ctx, "createSandbox") -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, "rootfs") -+ if err != nil { -+ return devices -+ } -+ } -+ -+ if s.vmConfig.IOThread { -+ iothread = iothreadID -+ } -+ -+ devices = append(devices, blkDevice{ -+ id: "rootfs", -+ filePath: s.vmConfig.rootfsPath, -+ deviceID: "virtio-blk0", -+ bus: bus, -+ addr: fmt.Sprintf("%d", addr), -+ iothread: iothread, -+ }) -+ -+ return devices -+} -+ -+func (s *stratovirt) appendRng(ctx context.Context, devices []VirtioDev) []VirtioDev { -+ var bus string -+ var addr uint32 -+ var err error -+ -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, "objrng0") -+ if err != nil { -+ return devices -+ } -+ } -+ -+ devices = append(devices, rngDevice{ -+ id: "objrng0", -+ fileName: s.config.EntropySource, -+ rng: "objrng0", -+ deviceID: "virtio-rng0", -+ bus: bus, -+ addr: fmt.Sprintf("%d", addr), -+ }) -+ -+ return devices -+} -+ -+func (s *stratovirt) appendConsole(ctx context.Context, devices []VirtioDev) []VirtioDev { -+ var bus string -+ var addr uint32 -+ var err error -+ -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, "virtio-serial0") -+ if err != nil { -+ return devices -+ } -+ } -+ -+ devices = append(devices, consoleDevice{ -+ id: "virtio-serial0", -+ backend: "socket", -+ charID: "charconsole0", -+ devType: "virtconsole", -+ charDev: "charconsole0", -+ deviceID: "virtio-console0", -+ bus: bus, -+ addr: fmt.Sprintf("%d", addr), -+ }) -+ -+ return devices -+} -+ -+func (s *stratovirt) appendVhostVsock(ctx context.Context, devices []VirtioDev, vsock types.VSock) []VirtioDev { -+ var bus string -+ var addr uint32 -+ var err error -+ -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, "vsock-id") -+ if err != nil { -+ return devices -+ } -+ } -+ -+ devices = append(devices, vhostVsock{ -+ id: "vsock-id", -+ guestID: fmt.Sprintf("%d", vsock.ContextID), -+ bus: bus, -+ addr: fmt.Sprintf("%d", addr), -+ }) -+ -+ return devices -+} -+ -+func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, endpoint Endpoint) []VirtioDev { -+ var bus string -+ var addr uint32 -+ var err error -+ -+ name := endpoint.Name() -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, name) -+ if err != nil { -+ return devices -+ } -+ } -+ -+ devices = append(devices, netDevice{ -+ devType: "tap", -+ id: name, -+ ifname: endpoint.NetworkPair().TapInterface.TAPIface.Name, -+ vhost: false, -+ FDs: nil, -+ VhostFDs: nil, -+ netdev: name, -+ deviceID: name, -+ bus: bus, -+ mac: endpoint.HardwareAddr(), -+ addr: fmt.Sprintf("%d", addr), -+ }) -+ -+ return devices -+} -+ -+func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error { -+ span, _ := s.trace(s.ctx, "setStratoVirtUp") - defer span.End() - -+ if err := hypervisorConfig.valid(); err != nil { -+ return err -+ } -+ - s.id = id - s.config = *hypervisorConfig -- if (s.config.OzonePath == "") || s.config.BootToBeTemplate { -- s.useOzone = false -- s.pidfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "pid") -- s.logfile = filepath.Join(s.store.RunVMStoragePath(), s.id, "/stratovirt.log") -- s.socketPath = filepath.Join(s.store.RunVMStoragePath(), id, apiSocket) -- s.consolePath = filepath.Join(s.store.RunVMStoragePath(), id, debugSocket) -- } else { -- s.useOzone = true -- s.ozoneRoot = filepath.Join(ozoneBaseDir, s.id) -- s.pidfile = filepath.Join(s.ozoneRoot, "pid") -- s.logfile = filepath.Join(s.ozoneRoot, "stratovirt.log") -- s.socketPath = filepath.Join(s.ozoneRoot, apiSocket) -- s.consolePath = filepath.Join(s.ozoneRoot, debugSocket) -+ -+ machineType := strings.ToLower(s.config.HypervisorMachineType) -+ if machineType == "" { -+ machineType = defaultStratoVirtMachineType -+ } -+ -+ initrdPath, err := s.config.InitrdAssetPath() -+ if err != nil { -+ return err -+ } -+ -+ imagePath, err := s.config.ImageAssetPath() -+ if err != nil { -+ return err -+ } -+ -+ KernelPath, err := s.config.KernelAssetPath() -+ if err != nil { -+ return err -+ } -+ -+ params, err := s.getKernelParams(machineType, initrdPath) -+ if err != nil { -+ return err -+ } -+ -+ var PFlash []string -+ -+ vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id) -+ qmpSocket := s.createQMPSocket(vmPath) -+ -+ s.vmConfig = vmConfig{ -+ name: fmt.Sprintf("sandbox-%s", id), -+ uuid: uuid.Generate().String(), -+ machineType: machineType, -+ vmPath: vmPath, -+ smp: uint32(s.config.NumVCPUs), -+ memory: uint64(s.config.MemorySize), -+ kernelPath: KernelPath, -+ params: params, -+ rootfsPath: imagePath, -+ initrdPath: initrdPath, -+ rootBus: types.NewBridge(types.PCIE, "pcie.0", make(map[uint32]string), 0), -+ IOThread: s.config.EnableIOThreads, -+ PFlash: PFlash, -+ pidFile: filepath.Join(vmPath, "pid"), -+ qmpSocketPath: qmpSocket, -+ consolePath: filepath.Join(vmPath, debugSocket), -+ fsSockPath: filepath.Join(vmPath, virtiofsSocket), -+ daemonize: true, -+ } -+ -+ s.vmConfig.devices = s.createDevices() -+ -+ // Set incoming parameters if VM starts from template mode. -+ if s.config.BootFromTemplate || s.config.BootToBeTemplate { -+ s.vmConfig.incoming = inComing{ -+ path: strings.Replace(s.config.DevicesStatePath, "/state", "", -1), -+ bootFromTemplate: s.config.BootFromTemplate, -+ } -+ } -+ -+ if hypervisorConfig.Debug { -+ s.vmConfig.logFile = filepath.Join(vmPath, "stratovirt.log") -+ } -+ return nil -+} -+ -+// When running with ozone environment, all resources need to be under a -+// specific path (e.g. /srv/ozone/stratovirt/). -+func (s *stratovirt) setOzone() error { -+ if s.config.OzonePath == "" || s.vmConfig.machineType != MachineTypeMicrovm { -+ s.vmConfig.useOzone = false -+ return nil - } - -- if s.config.VMid != "" && s.useOzone { -- // Make sure the symlinks do not exist -- os.RemoveAll(s.ozoneRoot) -+ ozoneRoot := filepath.Join(ozoneBaseDir, s.id) -+ s.vmConfig.useOzone = true -+ s.vmConfig.consolePath = filepath.Join(ozoneRoot, debugSocket) -+ s.vmConfig.logFile = filepath.Join(ozoneRoot, "stratovirt.log") -+ s.vmConfig.pidFile = filepath.Join(ozoneRoot, "pid") -+ s.vmConfig.qmpSocketPath.Name = filepath.Join(ozoneRoot, apiSocket) -+ s.qmpMonitorCh.path = filepath.Join(ozoneRoot, apiSocket) -+ -+ if s.config.VMid != "" { -+ os.RemoveAll(ozoneRoot) - ozoneVmRoot := filepath.Join(ozoneBaseDir, s.config.VMid) -- if err := os.Symlink(ozoneVmRoot, s.ozoneRoot); err != nil { -+ if err := os.Symlink(ozoneVmRoot, ozoneRoot); err != nil { - return err - } - } - -- if s.config.BootFromTemplate || s.config.BootToBeTemplate { -- s.templatePath = strings.Replace(s.config.DevicesStatePath, "/state", "", -1) -+ var ozoneRes []string -+ ozoneRes = append(ozoneRes, s.vmConfig.kernelPath) -+ if s.vmConfig.initrdPath != "" { -+ ozoneRes = append(ozoneRes, s.vmConfig.initrdPath) -+ } else { -+ ozoneRes = append(ozoneRes, s.vmConfig.rootfsPath) - } - -- s.netNSPath = networkNS.NetNsPath -- s.qmpMonitorCh = qmpChannel{ -- ctx: s.ctx, -- path: s.socketPath, -+ s.vmConfig.Ozone = Ozone{ -+ ozoneRoot: ozoneRoot, -+ ozoneRes: ozoneRes, -+ consolePath: filepath.Base(s.vmConfig.consolePath), -+ kernelPath: filepath.Base(s.vmConfig.kernelPath), -+ initrdPath: filepath.Base(s.vmConfig.initrdPath), -+ pidFile: filepath.Base(s.vmConfig.pidFile), -+ logFile: filepath.Base(s.vmConfig.logFile), -+ qmpSocketPath: filepath.Base(s.vmConfig.qmpSocketPath.Name), - } -+ return nil -+} -+ -+func (s *stratovirt) hypervisorConfig() HypervisorConfig { -+ return s.config -+} - -- if kernelPath, err := s.config.KernelAssetPath(); err == nil { -- s.kernelPath = kernelPath -- s.ozoneRes = append(s.ozoneRes, s.kernelPath) -+// Get StratoVirt binary path. -+func (s *stratovirt) binPath() (string, error) { -+ path, err := s.config.HypervisorAssetPath() -+ if err != nil { -+ return "", err - } - -- initrdPath, err := s.config.InitrdAssetPath() -+ if path == "" { -+ path = defaultStratoVirt -+ } -+ -+ if _, err = os.Stat(path); os.IsNotExist(err) { -+ return "", fmt.Errorf("StratoVirt path (%s) does not exist", path) -+ } -+ return path, nil -+} -+ -+func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { -+ var span otelTrace.Span -+ span, _ = s.trace(ctx, "createSandbox") -+ defer span.End() -+ -+ s.ctx = ctx -+ err := s.setVMConfig(id, hypervisorConfig) - if err != nil { - return err - } - -- if initrdPath == "" { -- imagePath, err := s.config.ImageAssetPath() -- if err != nil { -- return err -- } -- s.useImage = true -- s.rootfsPath = imagePath -- } else { -- s.useImage = false -- s.rootfsPath = initrdPath -+ if err := s.setOzone(); err != nil { -+ return err -+ } -+ -+ if s.path, err = s.binPath(); err != nil { -+ return err - } -- s.ozoneRes = append(s.ozoneRes, s.rootfsPath) - -+ s.netNSPath = networkNS.NetNsPath - return nil - } - -@@ -366,7 +905,7 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error { - } - - if int(time.Since(timeStart).Seconds()) > timeout { -- return fmt.Errorf("Failed to connect to StratoVirt instance (timeout %ds): %v", timeout, err) -+ return fmt.Errorf("failed to connect to stratovirt instance (timeout %ds): %v", timeout, err) - } - - time.Sleep(time.Duration(50) * time.Millisecond) -@@ -376,205 +915,114 @@ func (s *stratovirt) waitSandBoxStarted(timeout int) error { - s.Logger().WithError(err).Error(qmpCapErrMsg) - return err - } -- - return nil - } - --func (s *stratovirt) createbaseParams() []string { -- var params []string -- -- params = append(params, "-name", fmt.Sprintf("sandbox-%s", s.id)) -- params = append(params, "-append", s.getKernelCmdLine()) -- params = append(params, "-smp", fmt.Sprintf("%d", s.config.NumVCPUs)) -- params = append(params, "-m", fmt.Sprintf("%d", uint64(s.config.MemorySize))) -- params = append(params, "-device", "virtio-serial-device") -- params = append(params, "-device", "virtconsole,chardev=charconsole0,id=virtioconsole0") -- params = append(params, "-object", fmt.Sprintf("rng-random,id=objrng0,filename=%s", s.config.EntropySource)) -- params = append(params, "-device", "virtio-rng-device,rng=objrng0") -- -- // daemonize -- params = append(params, "-daemonize") -- -- return params -+func (s *stratovirt) createBaseParams(config vmConfig, params *[]string) { -+ config.appendName(params) -+ config.appendUUID(params) -+ config.appendMachine(params) -+ config.appendCPU(params) -+ config.appendMemory(params) -+ config.appendKernel(params) -+ config.appendPFlash(params) -+ config.appendQMPSocket(params) -+ config.appendPidFile(params) -+ config.appendLogFile(params) -+ config.appendIOThreads(params) -+ config.appendDevices(params) -+ config.appendIncoming(params) -+ -+ if config.daemonize { -+ *params = append(*params, "-daemonize") -+ } -+ *params = append(*params, "-disable-seccomp") - } - --func (s *stratovirt) createOzoneParams(params []string) ([]string, error) { -- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", apiSocket)) -- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", debugSocket)) -- params = append(params, "-kernel", filepath.Base(s.kernelPath)) -- params = append(params, "-pidfile", filepath.Base(s.pidfile)) -- -- // append logfile only on debug -- if s.config.Debug { -- params = append(params, "-D", filepath.Base(s.logfile)) -- } -+func (s *stratovirt) createOzoneParams(params []string, config vmConfig) ([]string, error) { -+ var err error - -- if s.useImage { -- s.mmioBlkSlots[0] = true -- params = append(params, "-device", "virtio-blk-device,drive=rootfs") -- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", filepath.Base(s.rootfsPath))) -- } else { -- params = append(params, "-initrd", filepath.Base(s.rootfsPath)) -- } -- -- // handle boot from template -- if s.config.BootFromTemplate { -- s.ozoneRes = append(s.ozoneRes, s.templatePath) -- params = append(params, "-incoming", fmt.Sprintf("file:%s", filepath.Base(s.templatePath))) -- } -- -- // add devices to cmdline -- for _, d := range s.devices { -- switch v := d.dev.(type) { -- case Endpoint: -- name := v.Name() -- mac := v.HardwareAddr() -- tapName := v.NetworkPair().TapInterface.TAPIface.Name -- params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) -- params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) -- case config.BlockDrive: -- id := v.ID -- path := v.File -- s.ozoneRes = append(s.ozoneRes, path) -- params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) -- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, filepath.Base(path))) -- case types.VSock: -- v.VhostFd.Close() -- params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) -- default: -- s.Logger().Error("Adding device type is unsupported") -- } -+ ozoneParams := []string{ -+ "-exec-file", s.path, -+ "-name", s.id, -+ "-gid", "0", -+ "-uid", "0", - } - -- return params, nil --} -- --func (s *stratovirt) createParams(params []string) ([]string, error) { -- params = append(params, "-qmp", fmt.Sprintf("unix:%s,server,nowait", s.socketPath)) -- params = append(params, "-chardev", fmt.Sprintf("socket,id=charconsole0,path=%s,server,nowait", s.consolePath)) -- params = append(params, "-kernel", s.kernelPath) -- params = append(params, "-pidfile", s.pidfile) -- -- // append logfile only on debug -- if s.config.Debug { -- params = append(params, "-D", s.logfile) -+ if s.netNSPath != "" { -+ ozoneParams = append(ozoneParams, "-netns", s.netNSPath) - } -- -- if s.useImage { -- s.mmioBlkSlots[0] = true -- params = append(params, "-device", "virtio-blk-device,drive=rootfs") -- params = append(params, "-drive", fmt.Sprintf("id=rootfs,file=%s,direct=off", s.rootfsPath)) -- } else { -- params = append(params, "-initrd", s.rootfsPath) -- } -- -- // handle boot from template -- if s.config.BootFromTemplate { -- params = append(params, "-incoming", fmt.Sprintf("file:%s", s.templatePath)) -- } -- -- // add devices to cmdline -- for _, d := range s.devices { -- switch v := d.dev.(type) { -- case Endpoint: -- name := v.Name() -- mac := v.HardwareAddr() -- tapName := v.NetworkPair().TapInterface.TAPIface.Name -- params = append(params, "-device", fmt.Sprintf("virtio-net-device,netdev=%s,id=%s,mac=%s", name, name, mac)) -- params = append(params, "-netdev", fmt.Sprintf("tap,id=%s,ifname=%s", name, tapName)) -- case config.BlockDrive: -- id := v.ID -- path := v.File -- params = append(params, "-device", fmt.Sprintf("virtio-blk-device,drive=%s", id)) -- params = append(params, "-drive", fmt.Sprintf("id=%s,file=%s", id, path)) -- case types.VSock: -- v.VhostFd.Close() -- params = append(params, "-device", fmt.Sprintf("vhost-vsock-device,id=vsock-id,guest-cid=%d", v.ContextID)) -- default: -- s.Logger().Error("Adding device type is unsupported") -+ ozoneParams = append(ozoneParams, "-source") -+ ozoneParams = append(ozoneParams, s.vmConfig.Ozone.ozoneRes...) -+ defer func() { -+ if err != nil { -+ ozoneParams = append(ozoneParams, "-clean-resource") -+ cmd := exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) -+ if errRun := cmd.Run(); errRun != nil { -+ s.Logger().WithError(errRun).Error("Failed to clean up ozone dir %s", s.vmConfig.Ozone.ozoneRoot) -+ } - } -- } -- -- return params, nil -+ }() -+ ozoneParams = append(ozoneParams, "--") -+ ozoneParams = append(ozoneParams, params...) -+ return ozoneParams, nil - } - - func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { - span, _ := s.trace(ctx, "startSandbox") - defer span.End() - -- var err error -- var cmd *exec.Cmd -- -- params := s.createbaseParams() -- -- stratovirtBinPath, err := s.config.HypervisorAssetPath() -+ err := os.MkdirAll(s.vmConfig.vmPath, DirMode) - if err != nil { - return err - } - -- if s.useOzone { -- var ozoneParams []string -- extend_params, err := s.createOzoneParams(params) -+ defer func() { - if err != nil { -- return err -- } -- ozoneParams = append(ozoneParams, "-exec-file", stratovirtBinPath) -- ozoneParams = append(ozoneParams, "-name", s.id) -- ozoneParams = append(ozoneParams, "-gid", "0") -- ozoneParams = append(ozoneParams, "-uid", "0") -- if s.netNSPath != "" { -- ozoneParams = append(ozoneParams, "-netns", s.netNSPath) -- } -- -- ozoneParams = append(ozoneParams, "-source") -- ozoneParams = append(ozoneParams, s.ozoneRes...) -+ if s.state.virtiofsPid != 0 { -+ syscall.Kill(s.state.virtiofsPid, syscall.SIGILL) -+ } - -- defer func() { -- if err != nil { -- ozoneParams = append(ozoneParams, "-clean-resource") -- cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) -- if err := cmd.Run(); err != nil { -- s.Logger().WithError(err).Error("Failed to clean up ozone dir %s", s.ozoneRoot) -- } -+ link, errLink := filepath.EvalSymlinks(s.vmConfig.vmPath) -+ if errLink != nil { -+ s.Logger().WithError(errLink).Warnf("Failed to get evaluation of any symbolic links.") - } -- }() - -- ozoneParams = append(ozoneParams, "--") -- ozoneParams = append(ozoneParams, extend_params...) -- cmd = exec.CommandContext(s.ctx, s.config.OzonePath, ozoneParams...) -- s.Logger().Info("StratoVirt/Ozone start with params: ", cmd) -- } else { -- params, err = s.createParams(params) -- if err != nil { -- return err -- } -+ if errRemove := os.RemoveAll(s.vmConfig.vmPath); errRemove != nil { -+ s.Logger().WithError(errRemove).Warnf("Failed to clean up vm dir %s", s.vmConfig.vmPath) -+ } - -- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -- err = os.MkdirAll(dir, DirMode) -- if err != nil { -- return err -- } -- defer func() { -- if err != nil { -- if err := os.RemoveAll(dir); err != nil { -- s.Logger().WithError(err).Error("Fail to clean up vm dir %s", dir) -+ if link != s.vmConfig.vmPath && link != "" { -+ if errRemove := os.RemoveAll(link); errRemove != nil { -+ s.Logger().WithError(errRemove).WithField("link", link).Warn("Failed to remove vm path link %s", link) - } - } -- }() -+ } -+ }() - -- cmd = exec.CommandContext(s.ctx, stratovirtBinPath, params...) -- s.Logger().Info("StratoVirt start with params: ", cmd) -+ var params []string -+ s.createBaseParams(s.vmConfig, ¶ms) -+ -+ var cmd *exec.Cmd -+ if s.vmConfig.useOzone { -+ var ozoneParams []string -+ if ozoneParams, err = s.createOzoneParams(params, s.vmConfig); err != nil { -+ return nil -+ } -+ cmd = exec.Command(s.config.OzonePath, ozoneParams...) -+ } else { -+ cmd = exec.Command(s.path, params...) - } -+ s.Logger().Info("StratoVirt start with cmd: ", cmd) - -- if err := cmd.Start(); err != nil { -+ if err = cmd.Run(); err != nil { - s.Logger().WithField("Error starting hypervisor, please check the params", err).Error() -- return err -+ return fmt.Errorf("failed to lunch stratovirt: %q", err) - } - - if err = s.waitSandBoxStarted(timeout); err != nil { - return err - } -- - return nil - } - -@@ -583,34 +1031,41 @@ func (s *stratovirt) stopSandbox(ctx context.Context, force bool) error { - defer span.End() - - defer func() { -- dir := filepath.Join(s.store.RunVMStoragePath(), s.id) -- link, _ := filepath.EvalSymlinks(dir) -+ link, errLink := filepath.EvalSymlinks(s.vmConfig.vmPath) -+ if errLink != nil { -+ s.Logger().WithError(errLink).Warnf("Failed to get evaluation of any symbolic links.") -+ } - -- if err := os.RemoveAll(dir); err != nil { -- s.Logger().WithError(err).Warnf("Failed to clean up vm dir %s", dir) -+ if errRemove := os.RemoveAll(s.vmConfig.vmPath); errRemove != nil { -+ s.Logger().WithError(errRemove).Warnf("Failed to clean up vm dir %s", s.vmConfig.vmPath) - } - -- if link != dir && link != "" { -- if err := os.RemoveAll(link); err != nil { -- s.Logger().WithError(err).WithField("link", link).Warn("Failed to remove vm path link %s", link) -+ if link != s.vmConfig.vmPath && link != "" { -+ if errRemove := os.RemoveAll(link); errRemove != nil { -+ s.Logger().WithError(errRemove).WithField("link", link).Warn("Failed to remove vm path link %s", link) - } - } - }() - - if !force { -- err := s.qmpSetup() -- if err != nil { -- return err -+ if errQmp := s.qmpSetup(); errQmp != nil { -+ return errQmp - } - -- err = s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) -- if err != nil { -- s.Logger().WithError(err).Error("Fail to execute qmp: QUIT") -- return err -+ errQuit := s.qmpMonitorCh.qmp.ExecuteQuit(s.qmpMonitorCh.ctx) -+ if errQuit != nil { -+ s.Logger().WithError(errQuit).Error("Fail to execute qmp: QUIT") -+ return errQuit -+ } -+ -+ if s.state.virtiofsPid != 0 { -+ syscall.Kill(s.state.virtiofsPid, syscall.SIGILL) - } - } else { -- if s.pid > 0 { -- syscall.Kill(s.pid, syscall.SIGKILL) -+ for _, p := range s.getPids() { -+ if p > 0 { -+ syscall.Kill(p, syscall.SIGKILL) -+ } - } - } - return nil -@@ -624,8 +1079,6 @@ func (s *stratovirt) pauseSandbox(ctx context.Context) error { - } - - func (s *stratovirt) saveSandbox() error { -- s.Logger().Info("save sandbox") -- - err := s.qmpSetup() - if err != nil { - return err -@@ -634,13 +1087,12 @@ func (s *stratovirt) saveSandbox() error { - // BootToBeTemplate sets the VM to be a template that other VMs can can clone from. - // We would want to bypass shared memory when saving VM to local file through migrate. - if s.config.BootToBeTemplate { -- err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.templatePath)) -+ err = s.qmpMonitorCh.qmp.ExecSetMigrateArguments(s.qmpMonitorCh.ctx, fmt.Sprintf("file:%s", s.vmConfig.incoming.path)) - if err != nil { - s.Logger().WithError(err).Error("exec migration") - return err - } - } -- - return nil - } - -@@ -665,7 +1117,6 @@ func (s *stratovirt) togglePauseSandbox(ctx context.Context, pause bool) error { - } else { - s.qmpMonitorCh.qmp.ExecuteCont(s.qmpMonitorCh.ctx) - } -- - return nil - } - -@@ -673,12 +1124,19 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType - span, _ := s.trace(ctx, "addDevice") - defer span.End() - -- dev := stratovirtDev{ -- dev: devInfo, -- devType: devType, -+ switch v := devInfo.(type) { -+ case types.Socket: -+ s.vmConfig.devices = s.appendConsole(ctx, s.vmConfig.devices) -+ case types.VSock: -+ v.VhostFd.Close() -+ s.vmConfig.devices = s.appendVhostVsock(ctx, s.vmConfig.devices, v) -+ case Endpoint: -+ s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v) -+ case config.BlockDrive: -+ s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices) -+ default: -+ s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type") - } -- s.devices = append(s.devices, dev) -- - return nil - } - -@@ -696,10 +1154,10 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { - return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) - } - -- if !isPut && s.mmioNetSlots[idx] { -+ if !isPut && s.state.mmioNetSlots[idx] { - return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) - } -- s.mmioNetSlots[idx] = !isPut -+ s.state.mmioNetSlots[idx] = !isPut - - return idx, nil - } else if strings.HasPrefix(Name, "vd") { -@@ -711,10 +1169,10 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { - char := []rune(charStr) - idx := int(char[0] - 'a') - -- if !isPut && s.mmioBlkSlots[idx] { -+ if !isPut && s.state.mmioBlkSlots[idx] { - return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) - } -- s.mmioBlkSlots[idx] = !isPut -+ s.state.mmioBlkSlots[idx] = !isPut - - return idx, nil - } -@@ -806,8 +1264,11 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err - switch op { - case addDevice: - driver := "virtio-blk-mmio" -- if s.useOzone { -+ if s.vmConfig.useOzone { - filePath, err = s.updateOzoneRes(drive.File, true) -+ if err != nil { -+ return fmt.Errorf("Failed to update ozone resources") -+ } - } else { - filePath = drive.File - } -@@ -826,7 +1287,7 @@ func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err err - return err - } - case removeDevice: -- if s.useOzone { -+ if s.vmConfig.useOzone { - s.updateOzoneRes(drive.File, false) - } - if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { -@@ -872,30 +1333,28 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ - } - } - --func (s *stratovirt) resizeMemory(ctx context.Context, reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { -+func (s *stratovirt) resizeMemory(ctx context.Context, memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { - return 0, memoryDevice{}, nil - } - --func (s *stratovirt) resizeVCPUs(ctx context.Context, reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { -+func (s *stratovirt) resizeVCPUs(ctx context.Context, vcpus uint32) (uint32, uint32, error) { - return 0, 0, nil - } - --func (s *stratovirt) getSandboxConsole(ctx context.Context, id string) (string, string, error) { -+func (s *stratovirt) getSandboxConsole(ctx context.Context, sandboxID string) (string, string, error) { - span, _ := s.trace(ctx, "getSandboxConsole") - defer span.End() - - var consolePath string - if s.config.Debug { -- consolePath = s.consolePath -- } else { -- consolePath = "" -+ consolePath = s.vmConfig.consolePath - } - consoleURL, err := utils.BuildSocketPath(consolePath) - if err != nil { - return consoleProtoUnix, "", err - } -- return consoleProtoUnix, consoleURL, nil - -+ return consoleProtoUnix, consoleURL, nil - } - - func (s *stratovirt) disconnect(ctx context.Context) { -@@ -925,10 +1384,14 @@ func (s *stratovirt) qmpTeardown() { - } - - func (s *stratovirt) qmpSetup() error { -- s.qmpTeardown() -+ s.qmpMonitorCh.Lock() -+ defer s.qmpMonitorCh.Unlock() - -- cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} -+ if s.qmpMonitorCh.qmp != nil { -+ return nil -+ } - -+ cfg := govmmQemu.QMPConfig{Logger: newQMPLogger()} - // Auto-closed by QMPStart(). - disconnectCh := make(chan struct{}) - -@@ -938,6 +1401,12 @@ func (s *stratovirt) qmpSetup() error { - return err - } - -+ if err := qmp.ExecuteQMPCapabilities(s.qmpMonitorCh.ctx); err != nil { -+ s.qmpMonitorCh.qmp.Shutdown() -+ s.Logger().WithError(err).Error(qmpCapErrMsg) -+ return err -+ } -+ - s.qmpMonitorCh.qmp = qmp - s.qmpMonitorCh.disconn = disconnectCh - -@@ -969,27 +1438,33 @@ func (s *stratovirt) getThreadIDs(ctx context.Context) (vcpuThreadIDs, error) { - } - - func (s *stratovirt) updateOzoneRes(src string, add bool) (string, error) { -- dst := filepath.Join(s.ozoneRoot, filepath.Base(src)) -+ dst := filepath.Join(s.vmConfig.Ozone.ozoneRoot, filepath.Base(src)) - if add { - if err := bindMount(context.Background(), src, dst, false, "slave"); err != nil { - s.Logger().WithField("bindMount failed", err).Error() - return "", err - } - } else { -- syscall.Unmount(dst, syscall.MNT_DETACH) -+ if err := syscall.Unmount(dst, syscall.MNT_DETACH); err != nil { -+ s.Logger().WithField("Unmount failed", err).Error() -+ return "", err -+ } - } - return filepath.Base(src), nil - } - - func (s *stratovirt) cleanOzoneRes() { - // Umount all resource in ozoneRoot -- if dir, err := ioutil.ReadDir(s.ozoneRoot); err == nil { -+ ozone := s.vmConfig.Ozone -+ if dir, err := ioutil.ReadDir(ozone.ozoneRoot); err == nil { - for _, file := range dir { -- syscall.Unmount(filepath.Join(s.ozoneRoot, file.Name()), syscall.MNT_DETACH) -+ if err := syscall.Unmount(filepath.Join(ozone.ozoneRoot, file.Name()), syscall.MNT_DETACH); err != nil { -+ s.Logger().WithField("Unmount failed", err).Error() -+ } - } - } - -- if err := os.RemoveAll(s.ozoneRoot); err != nil { -+ if err := os.RemoveAll(ozone.ozoneRoot); err != nil { - s.Logger().WithField("cleanup Ozone failed", err).Error() - } - -@@ -1007,7 +1482,8 @@ func (s *stratovirt) cleanup(ctx context.Context) error { - defer span.End() - - s.qmpTeardown() -- if s.useOzone { -+ -+ if s.vmConfig.useOzone { - s.cleanOzoneRes() - } - -@@ -1016,10 +1492,8 @@ func (s *stratovirt) cleanup(ctx context.Context) error { - - func (s *stratovirt) getPids() []int { - var pids []int -- if s.pid != 0 { -- pids = append(pids, s.pid) -- } else { -- pid, err := ioutil.ReadFile(s.pidfile) -+ if s.state.pid == 0 { -+ pid, err := ioutil.ReadFile(s.vmConfig.pidFile) - if err != nil { - s.Logger().WithError(err).Error("Read pid file failed.") - return []int{0} -@@ -1031,15 +1505,12 @@ func (s *stratovirt) getPids() []int { - return []int{0} - } - -- pids = append(pids, p) -- s.pid = p -+ s.state.pid = p - } - -- return pids --} -+ pids = append(pids, s.state.pid) - --func (s *stratovirt) getVirtioFsPid() *int { -- return nil -+ return pids - } - - func (s *stratovirt) fromGrpc(ctx context.Context, hypervisorConfig *HypervisorConfig, j []byte) error { -@@ -1051,8 +1522,12 @@ func (s *stratovirt) toGrpc(ctx context.Context) ([]byte, error) { - } - - func (s *stratovirt) check() error { -- if err := syscall.Kill(s.pid, syscall.Signal(0)); err != nil { -- return errors.Wrapf(err, "Failed to ping StratoVirt process") -+ if err := s.qmpSetup(); err != nil { -+ return err -+ } -+ -+ if _, err := s.qmpMonitorCh.qmp.ExecuteQueryStatus(s.qmpMonitorCh.ctx); err != nil { -+ return err - } - - return nil -@@ -1074,12 +1549,9 @@ func (s *stratovirt) save() (p persistapi.HypervisorState) { - } - - func (s *stratovirt) load(p persistapi.HypervisorState) { -- s.pid = p.Pid -- -- return -+ s.state.pid = p.Pid - } - - func (s *stratovirt) setSandbox(sandbox *Sandbox) { - s.sandbox = sandbox -- return - } --- -2.20.1.windows.1 - diff --git a/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch b/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch deleted file mode 100644 index 6de9b6a738f75ba859a08d14cfe71063c7ac1a00..0000000000000000000000000000000000000000 --- a/patches/0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch +++ /dev/null @@ -1,771 +0,0 @@ -From 8c0dd0f91366910b3029cb982a3cb807bdbb34cf Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Fri, 14 Jan 2022 17:07:37 +0800 -Subject: [PATCH] stratovirt: add a standard virtual machine sandbox type to - kata container - -Because stratovirt supports both microVM and standardVM -architectures, adapts two sandbox types for kata container. -Besides basic features, it also support virtio-fs, hotplug -VFIO, support more devices. - -Signed-off-by: Xinle.Guo ---- - src/runtime/pkg/katautils/config.go | 1 + - src/runtime/virtcontainers/stratovirt.go | 482 ++++++++++++++++++++--- - 2 files changed, 438 insertions(+), 45 deletions(-) - -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index e523ed3..b04cdee 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -1007,6 +1007,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - BlockDeviceCacheDirect: h.BlockDeviceCacheDirect, - BlockDeviceCacheNoflush: h.BlockDeviceCacheNoflush, - EnableIOThreads: h.EnableIOThreads, -+ PCIeRootPort: h.PCIeRootPort, - DisableVhostNet: h.DisableVhostNet, - EnableVhostUserStore: h.EnableVhostUserStore, - VhostUserStorePath: h.vhostUserStorePath(), -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 4fcfb94..ffe8965 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -14,14 +14,14 @@ import ( - "time" - - govmmQemu "github.com/kata-containers/govmm/qemu" -- "github.com/pkg/errors" -- "github.com/sirupsen/logrus" -- - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" -+ vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" -+ "github.com/pkg/errors" -+ "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel" - otelLabel "go.opentelemetry.io/otel/label" - otelTrace "go.opentelemetry.io/otel/trace" -@@ -44,6 +44,11 @@ const ( - const ( - WaitSandboxTimeoutSecs = 15 - MachineTypeMicrovm = "microvm" -+ MachineTypeQ35 = "q35" -+ MachineTypeVirt = "virt" -+ RootPortPrefix = "pcie" -+ Q35PFlashCode = "/usr/share/edk2/ovmf/OVMF_CODE.fd" -+ VirtPFlashCode = "/usr/share/edk2/aarch64/QEMU_EFI-pflash.raw" - MmioBus VirtioDriver = "mmio" - PciBus VirtioDriver = "pci" - ) -@@ -67,6 +72,42 @@ func (d VirtioDriver) getDriver(config *vmConfig) VirtioDriver { - } - } - -+type rootPortDevice struct { -+ id string -+ port string -+ bus string -+ slot int -+ plugged bool -+ addedDev string -+} -+ -+func (r rootPortDevice) isVaild() bool { -+ if r.id == "" || r.port == "" { -+ return false -+ } -+ return true -+} -+ -+func (r rootPortDevice) getParams(config *vmConfig) []string { -+ if !r.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var devParams []Param -+ devParams = append(devParams, Param{"id", r.id}) -+ devParams = append(devParams, Param{"port", r.port}) -+ if r.bus == "" { -+ r.bus = "pcie.0" -+ } -+ devParams = append(devParams, Param{"bus", r.bus}) -+ devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)}) -+ -+ driver := "pcie-root-port" -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -+ return params -+} -+ - type blkDevice struct { - id string - filePath string -@@ -179,6 +220,56 @@ func (n netDevice) getParams(config *vmConfig) []string { - return params - } - -+type virtioFs struct { -+ driver VirtioDriver -+ backend string -+ charID string -+ charDev string -+ tag string -+ deviceID string -+ bus string -+ addr string -+} -+ -+var virtiofsDriver = map[VirtioDriver]string{ -+ MmioBus: "vhost-user-fs-device", -+ PciBus: "vhost-user-fs-pci", -+} -+ -+func (v virtioFs) isVaild() bool { -+ if v.charID == "" || v.charDev == "" || v.deviceID == "" { -+ return false -+ } -+ return true -+} -+ -+func (v virtioFs) getParams(config *vmConfig) []string { -+ if !v.isVaild() { -+ return nil -+ } -+ -+ var params []string -+ var charParams []Param -+ var fsParams []Param -+ -+ charParams = append(charParams, Param{"id", v.charID}) -+ charParams = append(charParams, Param{"path", config.fsSockPath}) -+ -+ v.driver = v.driver.getDriver(config) -+ driver := virtiofsDriver[v.driver] -+ fsParams = append(fsParams, Param{"chardev", v.charDev}) -+ fsParams = append(fsParams, Param{"tag", v.tag}) -+ fsParams = append(fsParams, Param{"id", v.deviceID}) -+ if v.bus != "" { -+ fsParams = append(fsParams, Param{"bus", v.bus}) -+ fsParams = append(fsParams, Param{"addr", v.addr}) -+ } -+ -+ params = append(params, "-chardev", fmt.Sprintf("%s,%s,server,nowait", v.backend, strings.Join(SerializeParams(charParams, "="), ","))) -+ params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(fsParams, "="), ","))) -+ return params -+} -+ - type vhostVsock struct { - driver VirtioDriver - id string -@@ -442,6 +533,12 @@ func (c *vmConfig) appendDevices(params *[]string) { - for _, d := range c.devices { - *params = append(*params, d.getParams(c)...) - } -+ -+ if c.machineType == MachineTypeMicrovm { -+ return -+ } -+ // Add flag to unplug devices from their root port faster. -+ *params = append(*params, "-global", "pcie-root-port.fast-unplug=1") - } - - func (c *vmConfig) appendPFlash(params *[]string) { -@@ -499,6 +596,8 @@ func (c *vmConfig) appendIncoming(params *[]string) { - type State struct { - mmioBlkSlots [mmioBlkCount]bool - mmioNetSlots [mmioNetCount]bool -+ // The list of RootPorts that can be hot-added. -+ rootPort []rootPortDevice - pid int - virtiofsPid int - } -@@ -558,6 +657,26 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str - return strings.Join(params, " "), nil - } - -+func (s *stratovirt) getPFlash(machineType string) ([]string, error) { -+ if s.config.PFlash != nil { -+ return s.config.PFlash, nil -+ } -+ -+ var PFlash []string -+ switch machineType { -+ case MachineTypeQ35: -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode)) -+ case MachineTypeVirt: -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode)) -+ case MachineTypeMicrovm: -+ return nil, nil -+ default: -+ return nil, fmt.Errorf("failed to match machine type %s", machineType) -+ } -+ -+ return PFlash, nil -+} -+ - func (s *stratovirt) createQMPSocket(vmPath string) govmmQemu.QMPSocket { - socketPath := filepath.Join(vmPath, apiSocket) - -@@ -590,6 +709,34 @@ func (s *stratovirt) createDevices() []VirtioDev { - } - } - -+ // Create root port for all devices that need to be hot-added. -+ if s.vmConfig.machineType != MachineTypeMicrovm && s.config.PCIeRootPort > 0 { -+ devices = s.appendRootPort(ctx, devices) -+ } -+ -+ return devices -+} -+ -+func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev { -+ number := s.config.PCIeRootPort -+ -+ for i := uint32(1); i < number+1; i++ { -+ addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i)) -+ if err != nil { -+ return devices -+ } -+ -+ rp := rootPortDevice{ -+ id: fmt.Sprintf("%s.%d", RootPortPrefix, i), -+ port: fmt.Sprintf("%d", i), -+ bus: defaultBridgeBus, -+ slot: int(addr), -+ addedDev: "", -+ } -+ s.state.rootPort = append(s.state.rootPort, rp) -+ devices = append(devices, rp) -+ } -+ - return devices - } - -@@ -729,6 +876,38 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end - return devices - } - -+func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, volume types.Volume) []VirtioDev { -+ if s.config.SharedFS != config.VirtioFS { -+ return devices -+ } -+ -+ var bus string -+ var addr uint32 -+ var err error -+ name := "virtio_fs" -+ -+ if s.vmConfig.machineType != MachineTypeMicrovm { -+ bus = "pcie.0" -+ addr, err = s.vmConfig.rootBus.AddDevice(ctx, name) -+ if err != nil { -+ return devices -+ } -+ } -+ -+ devices = append(devices, virtioFs{ -+ backend: "socket", -+ // Virtio-fs must be bound to unique charDev, it uses the same name. -+ charID: name, -+ charDev: name, -+ tag: volume.MountTag, -+ deviceID: "virtio-fs0", -+ bus: bus, -+ addr: fmt.Sprintf("%d", addr), -+ }) -+ -+ return devices -+} -+ - func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) error { - span, _ := s.trace(s.ctx, "setStratoVirtUp") - defer span.End() -@@ -765,7 +944,10 @@ func (s *stratovirt) setVMConfig(id string, hypervisorConfig *HypervisorConfig) - return err - } - -- var PFlash []string -+ PFlash, err := s.getPFlash(machineType) -+ if err != nil { -+ return err -+ } - - vmPath := filepath.Join(s.store.RunVMStoragePath(), s.id) - qmpSocket := s.createQMPSocket(vmPath) -@@ -852,8 +1034,33 @@ func (s *stratovirt) setOzone() error { - return nil - } - --func (s *stratovirt) hypervisorConfig() HypervisorConfig { -- return s.config -+// Virtio fs daemon is a shared file system that lets VM access a directory -+// tree on the host. -+func (s *stratovirt) setupVirtioFs() error { -+ if !s.config.DisableBlockDeviceUse || s.config.SharedFS != config.VirtioFS { -+ return nil -+ } -+ -+ if _, err := os.Stat(s.config.VirtioFSDaemon); os.IsNotExist(err) { -+ return fmt.Errorf("virtiofsd path (%s) does not exist", s.config.VirtioFSDaemon) -+ } -+ -+ args := []string{ -+ "-socket-path", filepath.Join(s.vmConfig.vmPath, "virtiofs_kata.sock"), -+ "-source", getSharePath(s.id)} -+ if len(s.config.VirtioFSExtraArgs) != 0 { -+ args = append(args, s.config.VirtioFSExtraArgs...) -+ } -+ -+ cmd := exec.Command(s.config.VirtioFSDaemon, args...) -+ s.Logger().Info("Virtiofsd start with cmd: ", cmd) -+ -+ if err := cmd.Start(); err != nil { -+ return fmt.Errorf("failed to strat virtiofsd: %v", cmd) -+ } -+ -+ s.state.virtiofsPid = cmd.Process.Pid -+ return nil - } - - // Get StratoVirt binary path. -@@ -873,6 +1080,10 @@ func (s *stratovirt) binPath() (string, error) { - return path, nil - } - -+func (s *stratovirt) hypervisorConfig() HypervisorConfig { -+ return s.config -+} -+ - func (s *stratovirt) createSandbox(ctx context.Context, id string, networkNS NetworkNamespace, hypervisorConfig *HypervisorConfig) error { - var span otelTrace.Span - span, _ = s.trace(ctx, "createSandbox") -@@ -1000,6 +1211,10 @@ func (s *stratovirt) startSandbox(ctx context.Context, timeout int) error { - } - }() - -+ if err = s.setupVirtioFs(); err != nil { -+ return err -+ } -+ - var params []string - s.createBaseParams(s.vmConfig, ¶ms) - -@@ -1134,28 +1349,29 @@ func (s *stratovirt) addDevice(ctx context.Context, devInfo interface{}, devType - s.vmConfig.devices = s.appendNetwork(ctx, s.vmConfig.devices, v) - case config.BlockDrive: - s.vmConfig.devices = s.appendBlock(ctx, s.vmConfig.devices) -+ case types.Volume: -+ s.vmConfig.devices = s.appendVirtioFs(ctx, s.vmConfig.devices, v) - default: - s.Logger().WithField("dev-type", v).Warn("Could not append device: unsupported device type") - } - return nil - } - --func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { -+func (s *stratovirt) setupMmioSlot(Name string, isPut bool) (int, error) { - Name = filepath.Base(strings.ToLower(Name)) -- - if strings.HasPrefix(Name, "eth") { - idxStr := strings.TrimPrefix(Name, "eth") - if idxStr == Name { -- return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ return 0, fmt.Errorf("could not parse idx from name %q", Name) - } - - idx, err := strconv.Atoi(idxStr) - if err != nil { -- return 0, fmt.Errorf("Could not convert to int from Str %q", idxStr) -+ return 0, fmt.Errorf("could not convert to int from str %q", idxStr) - } - - if !isPut && s.state.mmioNetSlots[idx] { -- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", idxStr) -+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", idxStr) - } - s.state.mmioNetSlots[idx] = !isPut - -@@ -1163,25 +1379,80 @@ func (s *stratovirt) getDevSlot(Name string, isPut bool) (slot int, err error) { - } else if strings.HasPrefix(Name, "vd") { - charStr := strings.TrimPrefix(Name, "vd") - if charStr == Name { -- return 0, fmt.Errorf("Could not parse idx from Name %q", Name) -+ return 0, fmt.Errorf("could not parse idx from name %q", Name) - } - - char := []rune(charStr) - idx := int(char[0] - 'a') - - if !isPut && s.state.mmioBlkSlots[idx] { -- return 0, fmt.Errorf("GetDevSlot failed, slot is being used %q", charStr) -+ return 0, fmt.Errorf("failed to setup mmio slot, slot is being used %q", charStr) - } - s.state.mmioBlkSlots[idx] = !isPut - - return idx, nil - } - -- return 0, fmt.Errorf("GetDevSlot failed, Name is invalid %q", Name) -+ return 0, fmt.Errorf("failed to setup mmio slot , Name is invalid %q", Name) -+} -+ -+func (s *stratovirt) setupPciSlot(Name string, isPut bool) (string, int, error) { -+ rootports := &s.state.rootPort -+ if len(*rootports) == 0 { -+ return "", 0, fmt.Errorf("failed to get available address from bridges") -+ } -+ -+ for i, rootport := range *rootports { -+ if !isPut && !rootport.plugged && rootport.addedDev == "" { -+ (*rootports)[i].plugged = true -+ (*rootports)[i].addedDev = Name -+ return rootport.id, rootport.slot, nil -+ } else if isPut && rootport.plugged && rootport.addedDev == Name { -+ (*rootports)[i].plugged = false -+ (*rootports)[i].addedDev = "" -+ return rootport.id, rootport.slot, nil -+ } -+ } -+ -+ return "", 0, fmt.Errorf("no more bridge slots available") -+} -+ -+func (s *stratovirt) getDevSlot(Name string) (string, int, error) { -+ if s.config.HypervisorMachineType == MachineTypeMicrovm { -+ slot, err := s.setupMmioSlot(Name, false) -+ if err != nil { -+ return "", 0, err -+ } -+ -+ return "", slot, nil -+ } -+ -+ bus, slot, err := s.setupPciSlot(Name, false) -+ if err != nil { -+ return "pcie.0", 0, err -+ } -+ -+ return bus, slot, nil -+} -+ -+func (s *stratovirt) delDevSlot(Name string) error { -+ if s.vmConfig.machineType == MachineTypeMicrovm { -+ if _, err := s.setupMmioSlot(Name, true); err != nil { -+ return err -+ } -+ -+ return nil -+ } -+ -+ if _, _, err := s.setupPciSlot(Name, true); err != nil { -+ return err -+ } -+ -+ return nil - } - --func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) (err error) { -- err = s.qmpSetup() -+func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op operation) error { -+ err := s.qmpSetup() - if err != nil { - return err - } -@@ -1198,6 +1469,14 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera - return fmt.Errorf("Endpoint is not supported") - } - -+ defer func() { -+ if err != nil { -+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } -+ } -+ }() -+ - switch op { - case addDevice: - var ( -@@ -1220,27 +1499,51 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera - VhostFdNames = append(VhostFdNames, fdName) - } - -- slot, err := s.getDevSlot(endpoint.Name(), false) -+ bus, slot, err := s.getDevSlot(endpoint.Name()) - if err != nil { -- return fmt.Errorf("Could not get unused slot for %q", endpoint.Name()) -+ return fmt.Errorf("could not get unused slot for %q", endpoint.Name()) - } - - if len(VMFdNames) != 0 || len(VhostFdNames) != 0 { - if err := s.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(s.qmpMonitorCh.ctx, "tap", tap.ID, VMFdNames, VhostFdNames); err != nil { -- s.getDevSlot(endpoint.Name(), true) - return err - } - } else { - if err := s.qmpMonitorCh.qmp.ExecuteNetdevAdd(s.qmpMonitorCh.ctx, "tap", tap.ID, tap.TAPIface.Name, "no", "no", 0); err != nil { -- s.getDevSlot(endpoint.Name(), true) - return err - } - } -- if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.Name, tap.ID, endpoint.HardwareAddr(), fmt.Sprintf("%d", slot), "", "", 0, false); err != nil { -- s.getDevSlot(endpoint.Name(), true) -+ -+ // The slot of net device that hotplugged to the root port -+ // must be zero. -+ devAddr := "0x0.0x0" -+ if s.vmConfig.machineType == MachineTypeMicrovm { -+ devAddr = fmt.Sprintf("%d", slot) -+ } else { -+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot) -+ if err != nil { -+ return err -+ } -+ -+ devSlot, err := vcTypes.PciSlotFromString("0") -+ if err != nil { -+ return err -+ } -+ -+ pciPath, err := vcTypes.PciPathFromSlots(bridgeSlot, devSlot) -+ if err != nil { -+ return err -+ } -+ endpoint.SetPciPath(pciPath) -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil { - return err - } - case removeDevice: -+ if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } - if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, tap.ID); err != nil { - return err - } -@@ -1248,58 +1551,134 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera - return err - } - default: -- return fmt.Errorf("Operation is not supported") -+ return fmt.Errorf("operation is not supported") - } - - return nil - } - --func (s *stratovirt) hotplugBlk(drive *config.BlockDrive, op operation) (err error) { -- var filePath string -- err = s.qmpSetup() -+func (s *stratovirt) hotplugBlk(ctx context.Context, drive *config.BlockDrive, op operation) error { -+ err := s.qmpSetup() - if err != nil { - return err - } - -+ driver := "virtio-blk-pci" -+ if s.vmConfig.machineType == MachineTypeMicrovm { -+ driver = "virtio-blk-mmio" -+ } -+ -+ defer func() { -+ if err != nil { -+ s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID) -+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } -+ } -+ }() -+ - switch op { - case addDevice: -- driver := "virtio-blk-mmio" -+ filePath := drive.File - if s.vmConfig.useOzone { - filePath, err = s.updateOzoneRes(drive.File, true) -- if err != nil { -- return fmt.Errorf("Failed to update ozone resources") -- } -- } else { -- filePath = drive.File -- } -- slot, err := s.getDevSlot(drive.VirtPath, false) -- if err != nil { -- return fmt.Errorf("Could not get unused slot for %q", drive.VirtPath) - } - - if err := s.qmpMonitorCh.qmp.ExecuteBlockdevAdd(s.qmpMonitorCh.ctx, filePath, drive.ID, false); err != nil { -- s.getDevSlot(drive.VirtPath, true) - return err - } - -- if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, fmt.Sprintf("%d", slot), "", "", 0, true, false); err != nil { -- s.getDevSlot(drive.VirtPath, true) -+ bus, slot, err := s.getDevSlot(drive.VirtPath) -+ if err != nil { -+ return err -+ } -+ -+ // The slot of block device that hotplugged to the root port -+ // must be zero. -+ devAddr := "0x0.0x0" -+ if s.vmConfig.machineType == MachineTypeMicrovm { -+ devAddr = fmt.Sprintf("%d", slot) -+ } else { -+ bridgeSlot, err := vcTypes.PciSlotFromInt(slot) -+ if err != nil { -+ return err -+ } -+ -+ devSlot, err := vcTypes.PciSlotFromString("0") -+ if err != nil { -+ return err -+ } -+ -+ drive.PCIPath, err = vcTypes.PciPathFromSlots(bridgeSlot, devSlot) -+ if err != nil { -+ return err -+ } -+ } -+ -+ if err := s.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(s.qmpMonitorCh.ctx, drive.ID, drive.ID, driver, devAddr, bus, "", 0, false, false); err != nil { - return err - } - case removeDevice: - if s.vmConfig.useOzone { - s.updateOzoneRes(drive.File, false) - } -+ -+ if errDel := s.delDevSlot(drive.VirtPath); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } - if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { - return err - } - if err := s.qmpMonitorCh.qmp.ExecuteBlockdevDel(s.qmpMonitorCh.ctx, drive.ID); err != nil { - return err - } -+ default: -+ return fmt.Errorf("operation is not supported %d", op) -+ } -+ -+ return nil -+} -+ -+func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op operation) error { -+ err := s.qmpSetup() -+ if err != nil { -+ return err -+ } -+ -+ defer func() { -+ if err != nil { -+ if errDel := s.delDevSlot(device.ID); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } -+ } -+ }() - -- s.getDevSlot(drive.VirtPath, true) -+ switch op { -+ case addDevice: -+ var bus string -+ // The slot of block device that hotplugged to the root port -+ // must be zero. -+ devAddr := "0x0.0x0" -+ // The vfio device BDF format is 0000:1a:00.3 -+ device.BDF = "0000:" + device.BDF -+ -+ bus, _, err = s.getDevSlot(device.ID) -+ if err != nil { -+ return err -+ } -+ -+ if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil { -+ return err -+ } -+ case removeDevice: -+ if errDel := s.delDevSlot(device.ID); errDel != nil { -+ s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -+ } -+ if err := s.qmpMonitorCh.qmp.ExecuteDeviceDel(s.qmpMonitorCh.ctx, device.ID); err != nil { -+ return err -+ } - default: -- return fmt.Errorf("Operation is not supported") -+ return fmt.Errorf("operation is not supported %d", op) - } - - return nil -@@ -1313,9 +1692,11 @@ func (s *stratovirt) hotplugAddDevice(ctx context.Context, devInfo interface{}, - case netDev: - return nil, s.hotplugNet(ctx, devInfo.(Endpoint), addDevice) - case blockDev: -- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), addDevice) -+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), addDevice) -+ case vfioDev: -+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), addDevice) - default: -- return nil, fmt.Errorf("Hotplug add device failed: unsupported device type '%v'", devType) -+ return nil, fmt.Errorf("hotplug add device failed: unsupported device type '%v'", devType) - } - } - -@@ -1327,9 +1708,11 @@ func (s *stratovirt) hotplugRemoveDevice(ctx context.Context, devInfo interface{ - case netDev: - return nil, s.hotplugNet(ctx, devInfo.(Endpoint), removeDevice) - case blockDev: -- return nil, s.hotplugBlk(devInfo.(*config.BlockDrive), removeDevice) -+ return nil, s.hotplugBlk(ctx, devInfo.(*config.BlockDrive), removeDevice) -+ case vfioDev: -+ return nil, s.hotplugVFIO(ctx, devInfo.(*config.VFIODev), removeDevice) - default: -- return nil, fmt.Errorf("Hotplug remove device: unsupported device type '%v'", devType) -+ return nil, fmt.Errorf("hotplug remove device: unsupported device type '%v'", devType) - } - } - -@@ -1371,6 +1754,10 @@ func (s *stratovirt) capabilities(ctx context.Context) types.Capabilities { - var caps types.Capabilities - caps.SetBlockDeviceHotplugSupport() - -+ if s.config.DisableBlockDeviceUse && s.config.SharedFS == config.VirtioFS { -+ caps.SetFsSharingSupport() -+ } -+ - return caps - } - -@@ -1510,6 +1897,9 @@ func (s *stratovirt) getPids() []int { - - pids = append(pids, s.state.pid) - -+ if s.state.virtiofsPid != 0 { -+ pids = append(pids, s.state.virtiofsPid) -+ } - return pids - } - -@@ -1544,12 +1934,14 @@ func (s *stratovirt) isRateLimiterBuiltin() bool { - func (s *stratovirt) save() (p persistapi.HypervisorState) { - pids := s.getPids() - p.Pid = pids[0] -+ p.VirtiofsdPid = s.state.virtiofsPid - p.Type = string(StratovirtHypervisor) - return - } - - func (s *stratovirt) load(p persistapi.HypervisorState) { - s.state.pid = p.Pid -+ s.state.virtiofsPid = p.VirtiofsdPid - } - - func (s *stratovirt) setSandbox(sandbox *Sandbox) { --- -2.20.1.windows.1 - diff --git a/patches/0027-stratovirt-fix-the-problem-that-fails-to-plug-net-de.patch b/patches/0027-stratovirt-fix-the-problem-that-fails-to-plug-net-de.patch deleted file mode 100644 index d9e6188f4a15feacf66b6da08bc29c0d064f31be..0000000000000000000000000000000000000000 --- a/patches/0027-stratovirt-fix-the-problem-that-fails-to-plug-net-de.patch +++ /dev/null @@ -1,46 +0,0 @@ -From 9ac9c120a51663fa98ef25d721c321e5d16b3859 Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Sat, 26 Feb 2022 17:25:11 +0800 -Subject: [PATCH] stratovirt: fix the problem that fails to plug net device - -In kernel, hot plug pcie device needs to sleep 100ms to avoid -misoperation. Kata 1.x version agent monitor the plug event to -make sure plug device success. But kata 2.x does not provide -method. It may failed to get net device after plug the device. - -Signed-off-by: Xinle.Guo ---- - src/runtime/virtcontainers/stratovirt.go | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index ffe8965..27b45b7 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -39,6 +39,7 @@ const ( - iothreadID = "iothread_block" - mmioBlkCount = 4 - mmioNetCount = 2 -+ hotPlugDelayTime = 200 - ) - - const ( -@@ -1540,6 +1541,7 @@ func (s *stratovirt) hotplugNet(ctx context.Context, endpoint Endpoint, op opera - if err := s.qmpMonitorCh.qmp.ExecuteNetPCIDeviceAdd(s.qmpMonitorCh.ctx, tap.ID, tap.ID, endpoint.HardwareAddr(), devAddr, bus, "", 0, false); err != nil { - return err - } -+ time.Sleep(time.Millisecond * hotPlugDelayTime) - case removeDevice: - if errDel := s.delDevSlot(endpoint.Name()); errDel != nil { - s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") -@@ -1670,6 +1672,7 @@ func (s *stratovirt) hotplugVFIO(ctx context.Context, device *config.VFIODev, op - if err = s.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(s.qmpMonitorCh.ctx, device.ID, device.BDF, devAddr, bus, ""); err != nil { - return err - } -+ time.Sleep(time.Millisecond * hotPlugDelayTime) - case removeDevice: - if errDel := s.delDevSlot(device.ID); errDel != nil { - s.Logger().WithError(errDel).Warnf("Failed to delete device slot.") --- -2.20.1.windows.1 - diff --git a/patches/0028-stratovirt-provide-a-way-to-dynomically-obtain-firmw.patch b/patches/0028-stratovirt-provide-a-way-to-dynomically-obtain-firmw.patch deleted file mode 100644 index a849127e875b8bb749c62c294857c9f166fa82e1..0000000000000000000000000000000000000000 --- a/patches/0028-stratovirt-provide-a-way-to-dynomically-obtain-firmw.patch +++ /dev/null @@ -1,78 +0,0 @@ -From 769af40220ea3d7b87173f11c135e651076e14ee Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Wed, 2 Mar 2022 09:41:49 +0800 -Subject: [PATCH] stratovirt: provide a way to dynomically obtain firmware - -For stratovirt, it requires firmware to boot. The default -path is `/usr/share/edk2/xxx/pflash`. Now, we provides a -way to set path in configuration.toml file - -Signed-off-by: Xinle.Guo ---- - src/runtime/cli/config/configuration-stratovirt.toml.in | 4 ++++ - src/runtime/pkg/katautils/config.go | 6 ++++++ - src/runtime/virtcontainers/stratovirt.go | 7 ++++--- - 3 files changed, 14 insertions(+), 3 deletions(-) - -diff --git a/src/runtime/cli/config/configuration-stratovirt.toml.in b/src/runtime/cli/config/configuration-stratovirt.toml.in -index db46665..519c390 100644 ---- a/src/runtime/cli/config/configuration-stratovirt.toml.in -+++ b/src/runtime/cli/config/configuration-stratovirt.toml.in -@@ -28,6 +28,10 @@ enable_annotations = @DEFENABLEANNOTATIONS@ - # Your distribution recommends: @STRATOVIRTVALIDHYPERVISORPATHS@ - valid_hypervisor_paths = @STRATOVIRTVALIDHYPERVISORPATHS@ - -+# Path to the firmware. -+# If you want that qemu uses the default firmware leave this option empty -+firmware = "@FIRMWAREPATH@" -+ - # Path for the ozone specific to stratovirt - # If the ozone path is set, stratovirt will be launched in - # ozone secure environment. It is disabled by default. -diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go -index b04cdee..cdc9d7a 100644 ---- a/src/runtime/pkg/katautils/config.go -+++ b/src/runtime/pkg/katautils/config.go -@@ -951,6 +951,11 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - errors.New("either image or initrd must be defined in the configuration file") - } - -+ firmware, err := h.firmware() -+ if err != nil { -+ return vc.HypervisorConfig{}, err -+ } -+ - kernelParams := h.kernelParams() - machineType := h.machineType() - -@@ -979,6 +984,7 @@ func newStratovirtHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { - KernelPath: kernel, - InitrdPath: initrd, - ImagePath: image, -+ FirmwarePath: firmware, - OzonePath: ozone, - KernelParams: vc.DeserializeParams(strings.Fields(kernelParams)), - HypervisorMachineType: machineType, -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 27b45b7..d2b2233 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -659,11 +659,12 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str - } - - func (s *stratovirt) getPFlash(machineType string) ([]string, error) { -- if s.config.PFlash != nil { -- return s.config.PFlash, nil -+ var PFlash []string -+ if s.config.FirmwarePath != "" { -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", s.config.FirmwarePath)) -+ return PFlash, nil - } - -- var PFlash []string - switch machineType { - case MachineTypeQ35: - PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode)) --- -2.20.1.windows.1 - diff --git a/patches/0029-stratovirt-fix-the-problem-that-add-more-than-16-roo.patch b/patches/0029-stratovirt-fix-the-problem-that-add-more-than-16-roo.patch deleted file mode 100644 index bffd30641779309a62c60a4640db938358505009..0000000000000000000000000000000000000000 --- a/patches/0029-stratovirt-fix-the-problem-that-add-more-than-16-roo.patch +++ /dev/null @@ -1,95 +0,0 @@ -From c455ef6f406ba317af76b48ff79db131d0594bc1 Mon Sep 17 00:00:00 2001 -From: "Xinle.Guo" -Date: Fri, 18 Mar 2022 10:49:35 +0800 -Subject: [PATCH] stratovirt: fix the problem that add more than 16 root port - devices - -It will failed to start StratoVirt sandbox if pcie root prot is set -more than 16. The reason is that StratoVirt can only distinguish -hexadecimal device address number. - -Signed-off-by: Xinle.Guo ---- - src/runtime/virtcontainers/stratovirt.go | 17 ++++++++++------- - 1 file changed, 10 insertions(+), 7 deletions(-) - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index d2b2233..98a702a 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -102,7 +102,7 @@ func (r rootPortDevice) getParams(config *vmConfig) []string { - r.bus = "pcie.0" - } - devParams = append(devParams, Param{"bus", r.bus}) -- devParams = append(devParams, Param{"addr", fmt.Sprintf("%d", r.slot)}) -+ devParams = append(devParams, Param{"addr", fmt.Sprintf("0x%x", r.slot)}) - - driver := "pcie-root-port" - params = append(params, "-device", fmt.Sprintf("%s,%s", driver, strings.Join(SerializeParams(devParams, "="), ","))) -@@ -721,6 +721,9 @@ func (s *stratovirt) createDevices() []VirtioDev { - - func (s *stratovirt) appendRootPort(ctx context.Context, devices []VirtioDev) []VirtioDev { - number := s.config.PCIeRootPort -+ if number > 20 { -+ number = 20 -+ } - - for i := uint32(1); i < number+1; i++ { - addr, err := s.vmConfig.rootBus.AddDevice(ctx, fmt.Sprintf("%s.%d", RootPortPrefix, i)) -@@ -765,7 +768,7 @@ func (s *stratovirt) appendBlock(ctx context.Context, devices []VirtioDev) []Vir - filePath: s.vmConfig.rootfsPath, - deviceID: "virtio-blk0", - bus: bus, -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - iothread: iothread, - }) - -@@ -791,7 +794,7 @@ func (s *stratovirt) appendRng(ctx context.Context, devices []VirtioDev) []Virti - rng: "objrng0", - deviceID: "virtio-rng0", - bus: bus, -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - }) - - return devices -@@ -818,7 +821,7 @@ func (s *stratovirt) appendConsole(ctx context.Context, devices []VirtioDev) []V - charDev: "charconsole0", - deviceID: "virtio-console0", - bus: bus, -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - }) - - return devices -@@ -841,7 +844,7 @@ func (s *stratovirt) appendVhostVsock(ctx context.Context, devices []VirtioDev, - id: "vsock-id", - guestID: fmt.Sprintf("%d", vsock.ContextID), - bus: bus, -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - }) - - return devices -@@ -872,7 +875,7 @@ func (s *stratovirt) appendNetwork(ctx context.Context, devices []VirtioDev, end - deviceID: name, - bus: bus, - mac: endpoint.HardwareAddr(), -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - }) - - return devices -@@ -904,7 +907,7 @@ func (s *stratovirt) appendVirtioFs(ctx context.Context, devices []VirtioDev, vo - tag: volume.MountTag, - deviceID: "virtio-fs0", - bus: bus, -- addr: fmt.Sprintf("%d", addr), -+ addr: fmt.Sprintf("0x%x", addr), - }) - - return devices --- -2.20.1.windows.1 - diff --git a/patches/0030-use-host_device-blockdev-adding.patch b/patches/0030-use-host_device-blockdev-adding.patch deleted file mode 100644 index deaad571b0675cb1f1609c3f503992e54952bfef..0000000000000000000000000000000000000000 --- a/patches/0030-use-host_device-blockdev-adding.patch +++ /dev/null @@ -1,38 +0,0 @@ -From 2da8be5657f219de41e11917cb916895934749b8 Mon Sep 17 00:00:00 2001 -From: chengzrz -Date: Mon, 22 Aug 2022 19:19:52 +0800 -Subject: [PATCH] use host_device blockdev adding - -ExecuteBlockdevAdd() and ExecuteBlockdevAddWithCache() both appear to be -intended to create block devices in the guest which backend onto a block -device in the host. That seems to be the way that Kata always uses it. -However blockdevAddBaseArgs(), used by both those functions always uses the -"file" driver, which is only intended for use with regular file backends. - -Use of the "file" driver for host block devices was deprecated in qemu-3.0, -and has been removed entirely in qemu-6.0 (commit 8d17adf34f5). We should -be using the "host_device" driver instead. - -reference: https://github.com/kata-containers/kata-containers/commit/1b021929864fa45b643d9603d6615cc4b86235d7 - -Signed-off-by: chengzrz ---- - src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -index 97e924559..0e0337dbf 100644 ---- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -@@ -775,7 +775,7 @@ func (q *QMP) blockdevAddBaseArgs(device, blockdevID string, ro bool) (map[strin - "driver": "raw", - "read-only": ro, - "file": map[string]interface{}{ -- "driver": "file", -+ "driver": "host_device", - "filename": device, - }, - } --- -2.25.1 - diff --git a/patches/0031-add-explicit-on-after-kernel_irqchip.patch b/patches/0031-add-explicit-on-after-kernel_irqchip.patch deleted file mode 100644 index b27f444c23b85bcefc279b1d4016257e429ec83a..0000000000000000000000000000000000000000 --- a/patches/0031-add-explicit-on-after-kernel_irqchip.patch +++ /dev/null @@ -1,33 +0,0 @@ -From 0c9c5ca20b6aeae5e550decfd3540b389fb02cb5 Mon Sep 17 00:00:00 2001 -From: chengzrz -Date: Tue, 23 Aug 2022 15:30:05 +0800 -Subject: [PATCH] add explicit on after kernel_irqchip -Kata uses the 'kernel_irqchip' machine option to qemu. By default it -uses it in what qemu calls the "short-form boolean" with no parameter. -That style was deprecated by qemu between 5.2 and 6.0 (commit -ccd3b3b8112b) and effectively removed entirely between 6.0 and 6.1 -(commit d8fb7d0969d5). - -reference:https://github.com/kata-containers/kata-containers/commit/316509566966e4c9b3fd9ba3521554b384fdbf88 - -Signed-off-by: chengzrz ---- - src/runtime/virtcontainers/qemu_amd64.go | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go -index 1a045fae0..bc598c653 100644 ---- a/src/runtime/virtcontainers/qemu_amd64.go -+++ b/src/runtime/virtcontainers/qemu_amd64.go -@@ -27,7 +27,7 @@ const ( - - defaultQemuMachineType = QemuPC - -- defaultQemuMachineOptions = "accel=kvm,kernel_irqchip" -+ defaultQemuMachineOptions = "accel=kvm,kernel_irqchip=on" - - qmpMigrationWaitTimeout = 5 * time.Second - ) --- -2.25.1 - diff --git a/patches/0032-qmp-Don-t-use-deprecated-props-field-for-object-add.patch b/patches/0032-qmp-Don-t-use-deprecated-props-field-for-object-add.patch deleted file mode 100644 index db112536a97196777e291581a5e775f6c08f10d5..0000000000000000000000000000000000000000 --- a/patches/0032-qmp-Don-t-use-deprecated-props-field-for-object-add.patch +++ /dev/null @@ -1,67 +0,0 @@ -From e3296a68f7fa270d72605dfd4eb40b1081a79969 Mon Sep 17 00:00:00 2001 -From: chengzrz -Date: Fri, 2 Sep 2022 15:10:26 +0800 -Subject: [PATCH] qmp: Don't use deprecated 'props' field for object-add - -Use of the 'props' argument to 'object-add' has been deprecated since QEMU -5.0 (commit 5f07c4d60d09) in favor of flattening the properties directly -into the 'object-add' arguments. Support for 'props' is removed entirely -in qemu 6.0 (commit 50243407457a). - -reference:https://github.com/kata-containers/kata-containers/commit/d27256f8635d3fa382d6cbd9f3a60f601773c4dc - -Signed-off-by: chengzrz ---- - .../kata-containers/govmm/qemu/qmp.go | 18 +++++++----------- - 1 file changed, 7 insertions(+), 11 deletions(-) - -diff --git a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -index 97e9245..91dd732 100644 ---- a/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -+++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go -@@ -1387,17 +1387,16 @@ func (q *QMP) ExecQueryCpusFast(ctx context.Context) ([]CPUInfoFast, error) { - - // ExecMemdevAdd adds size of MiB memory device to the guest - func (q *QMP) ExecMemdevAdd(ctx context.Context, qomtype, id, mempath string, size int, share bool, driver, driverID string) error { -- props := map[string]interface{}{"size": uint64(size) << 20} - args := map[string]interface{}{ - "qom-type": qomtype, - "id": id, -- "props": props, -+ "size": uint64(size) << 20, - } - if mempath != "" { -- props["mem-path"] = mempath -+ args["mem-path"] = mempath - } - if share { -- props["share"] = true -+ args["share"] = true - } - err := q.executeCommand(ctx, "object-add", args, nil) - if err != nil { -@@ -1439,17 +1438,14 @@ func (q *QMP) ExecuteNVDIMMDeviceAdd(ctx context.Context, id, mempath string, si - args := map[string]interface{}{ - "qom-type": "memory-backend-file", - "id": "nvdimmbackmem" + id, -- "props": map[string]interface{}{ -- "mem-path": mempath, -- "size": size, -- "share": true, -- }, -+ "mem-path": mempath, -+ "size": size, -+ "share": true, - } - - if q.version.Major > 4 || (q.version.Major == 4 && q.version.Minor >= 1) { - if pmem != nil { -- props := args["props"].(map[string]interface{}) -- props["pmem"] = *pmem -+ args["pmem"] = *pmem - } - } - --- -2.25.1 - diff --git a/patches/0033-optimize-compile-options.patch b/patches/0033-optimize-compile-options.patch deleted file mode 100644 index 397901e4d9b34626547ce9cb4122fcef2c167495..0000000000000000000000000000000000000000 --- a/patches/0033-optimize-compile-options.patch +++ /dev/null @@ -1,57 +0,0 @@ -From 443e65326eeccb3a4ad8170670d56c32931215f9 Mon Sep 17 00:00:00 2001 -From: Vanient -Date: Thu, 8 Sep 2022 21:44:13 +0800 -Subject: [PATCH] optimize compile options - -Signed-off-by: Vanient ---- - src/runtime/Makefile | 20 ++++++++++++++++---- - 1 file changed, 16 insertions(+), 4 deletions(-) - -diff --git a/src/runtime/Makefile b/src/runtime/Makefile -index bade196..ad421d9 100644 ---- a/src/runtime/Makefile -+++ b/src/runtime/Makefile -@@ -566,7 +566,11 @@ monitor: $(MONITOR_OUTPUT) - netmon: $(NETMON_TARGET_OUTPUT) - - $(NETMON_TARGET_OUTPUT): $(SOURCES) VERSION -- $(QUIET_BUILD)(cd $(NETMON_DIR) && go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION)" $(KATA_LDFLAGS)) -+ $(QUIET_BUILD)(cd $(NETMON_DIR) && \ -+ CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \ -+ CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \ -+ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ -+ go build $(BUILDFLAGS) -o $@ -ldflags "-linkmode=external -X main.version=$(VERSION)" $(KATA_LDFLAGS)) - - runtime: $(TARGET_OUTPUT) $(CONFIGS) - .DEFAULT: default -@@ -605,15 +609,23 @@ GENERATED_FILES += $(GENERATED_CONFIG) - GENERATED_FILES += pkg/katautils/config-settings.go - - $(TARGET_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) | show-summary -- $(QUIET_BUILD)(cd $(CLI_DIR) && go build $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .) -+ $(QUIET_BUILD)(cd $(CLI_DIR) && \ -+ CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \ -+ CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \ -+ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ -+ go build -ldflags "-linkmode=external" $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .) - - $(SHIMV2_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) - $(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && ln -fs $(GENERATED_CONFIG)) -- $(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && go build $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .) -+ $(QUIET_BUILD)(cd $(SHIMV2_DIR)/ && \ -+ CGO_CFLAGS="-fstack-protector-strong -fPIE -D_FORTIFY_SOURCE=2 -O2" \ -+ CGO_LDFLAGS_ALLOW="-Wl,-z,relro,-z,now" \ -+ CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \ -+ go build -ldflags "-linkmode=external" $(KATA_LDFLAGS) $(BUILDFLAGS) -o $@ .) - - $(MONITOR_OUTPUT): $(SOURCES) $(GENERATED_FILES) $(MAKEFILE_LIST) .git-commit - $(QUIET_BUILD)(cd $(MONITOR_DIR)/ && CGO_ENABLED=0 go build \ -- --ldflags "-X main.GitCommit=$(shell cat .git-commit)" $(BUILDFLAGS) -buildmode=exe -o $@ .) -+ --ldflags "-linkmode=external -X main.GitCommit=$(shell cat .git-commit)" $(BUILDFLAGS) -o $@ .) - - .PHONY: \ - check \ --- -2.27.0 - diff --git a/patches/0034-stratovirt-Append-readonly-option-when-get-pflash-cm.patch b/patches/0034-stratovirt-Append-readonly-option-when-get-pflash-cm.patch deleted file mode 100644 index 419152ea991708e173c5a2b56c639cd3969373f9..0000000000000000000000000000000000000000 --- a/patches/0034-stratovirt-Append-readonly-option-when-get-pflash-cm.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 59cf9bfb95386f123190eff58d50e99ec1ec5ea7 Mon Sep 17 00:00:00 2001 -From: Keqian Zhu -Date: Tue, 20 Dec 2022 14:14:46 +0800 -Subject: [PATCH] stratovirt: Append readonly option when get pflash cmdline - -All Stratovirt VM shares the same pflash file by default, and file can only be -shared readonly for safety. - -Signed-off-by: Keqian Zhu ---- - src/runtime/virtcontainers/stratovirt.go | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - -diff --git a/src/runtime/virtcontainers/stratovirt.go b/src/runtime/virtcontainers/stratovirt.go -index 98a702a..7b01c76 100644 ---- a/src/runtime/virtcontainers/stratovirt.go -+++ b/src/runtime/virtcontainers/stratovirt.go -@@ -661,15 +661,15 @@ func (s *stratovirt) getKernelParams(machineType string, initrdPath string) (str - func (s *stratovirt) getPFlash(machineType string) ([]string, error) { - var PFlash []string - if s.config.FirmwarePath != "" { -- PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", s.config.FirmwarePath)) -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0,readonly=true", s.config.FirmwarePath)) - return PFlash, nil - } - - switch machineType { - case MachineTypeQ35: -- PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", Q35PFlashCode)) -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0,readonly=true", Q35PFlashCode)) - case MachineTypeVirt: -- PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0", VirtPFlashCode)) -+ PFlash = append(PFlash, fmt.Sprintf("file=%s,if=pflash,unit=0,readonly=true", VirtPFlashCode)) - case MachineTypeMicrovm: - return nil, nil - default: --- -2.33.0 - diff --git a/patches/0035-update-cgroup-rs-crate.patch b/patches/0035-update-cgroup-rs-crate.patch deleted file mode 100644 index 9676f0826679ee922bb2f4e33d18d9e0234fb550..0000000000000000000000000000000000000000 --- a/patches/0035-update-cgroup-rs-crate.patch +++ /dev/null @@ -1,111 +0,0 @@ -From b5ce74b04b457d54104113c6fc7e507ead3ac26d Mon Sep 17 00:00:00 2001 -From: Vanient -Date: Mon, 4 Sep 2023 15:36:45 +0800 -Subject: [PATCH] update cgroup-rs crate - -update cgroup-rs crate to fix "unable to write to a control group file" - -Signed-off-by: Vanient ---- - src/agent/Cargo.lock | 25 +++++++++++++++++++------ - src/agent/Cargo.toml | 2 +- - src/agent/rustjail/Cargo.toml | 2 +- - 3 files changed, 21 insertions(+), 8 deletions(-) - -diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock -index 87807a3..799eec1 100644 ---- a/src/agent/Cargo.lock -+++ b/src/agent/Cargo.lock -@@ -1,5 +1,7 @@ - # This file is automatically @generated by Cargo. - # It is not intended for manual editing. -+version = 3 -+ - [[package]] - name = "addr2line" - version = "0.14.0" -@@ -147,13 +149,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - - [[package]] - name = "cgroups-rs" --version = "0.2.5" -+version = "0.2.11" - source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "d4cec688ee0fcd143ffd7893ce2c9857bfc656eb1f2a27202244b72f08f5f8ed" -+checksum = "3845d8ddaca63e9975f07b7a32262afe284561c2f0f620aa968913a65f671fd2" - dependencies = [ - "libc", - "log", -- "nix 0.20.0", -+ "nix 0.24.3", - "regex", - ] - -@@ -510,9 +512,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - - [[package]] - name = "libc" --version = "0.2.91" -+version = "0.2.147" - source = "registry+https://github.com/rust-lang/crates.io-index" --checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" -+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" - - [[package]] - name = "libflate" -@@ -727,6 +729,17 @@ dependencies = [ - "libc", - ] - -+[[package]] -+name = "nix" -+version = "0.24.3" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" -+dependencies = [ -+ "bitflags", -+ "cfg-if 1.0.0", -+ "libc", -+] -+ - [[package]] - name = "ntapi" - version = "0.3.6" -@@ -877,7 +890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "059a34f111a9dee2ce1ac2826a68b24601c4298cfeb1a587c3cb493d5ab46f52" - dependencies = [ - "libc", -- "nix 0.19.1", -+ "nix 0.20.0", - ] - - [[package]] -diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml -index 195b53b..3405827 100644 ---- a/src/agent/Cargo.toml -+++ b/src/agent/Cargo.toml -@@ -45,7 +45,7 @@ tempfile = "3.1.0" - prometheus = { version = "0.9.0", features = ["process"] } - procfs = "0.7.9" - anyhow = "1.0.32" --cgroups = { package = "cgroups-rs", version = "0.2.5" } -+cgroups = { package = "cgroups-rs", version = "0.2.11" } - - [workspace] - members = [ -diff --git a/src/agent/rustjail/Cargo.toml b/src/agent/rustjail/Cargo.toml -index 55e59f4..7d59d2b 100644 ---- a/src/agent/rustjail/Cargo.toml -+++ b/src/agent/rustjail/Cargo.toml -@@ -23,7 +23,7 @@ scan_fmt = "0.2" - regex = "1.1" - path-absolutize = "1.2.0" - anyhow = "1.0.32" --cgroups = { package = "cgroups-rs", version = "0.2.5" } -+cgroups = { package = "cgroups-rs", version = "0.2.11" } - tempfile = "3.1.0" - rlimit = "0.5.3" - --- -2.33.0 - diff --git a/series.conf b/series.conf index 79a6e9926d60893f47beeeef7c30e3181628af6f..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/series.conf +++ b/series.conf @@ -1,35 +0,0 @@ -0001-runtime-add-support-of-new-sandbox-hypervisor-type-S.patch -0002-agent-add-support-of-new-sandbox-hypervisor-kind-Str.patch -0003-runtime-implement-updateInterfaceHwAddrByName-interf.patch -0004-configuration-add-configuration-generator-for-hyperv.patch -0005-runtime-add-the-secure-component-ozone-support-for-h.patch -0006-factory-add-the-template-factory-support-for-hypervi.patch -0007-kata-containers-support-with-iSulad.patch -0008-kata-containers-adpat-with-iSulad.patch -0009-kata-containers-fix-kata-runtime-hungs-when-qemu-proces.patch -0010-kata-containers-fix-kata-runtime-skip-read-lines-in-pro.patch -0011-kata-containers-keep-the-process-name-of-qemu-same-as-c.patch -0012-kata-containers-modify-kernel-and-image-path-in-conf.patch -0013-kata-containers-increase-delete-cgroup-retry-times.patch -0014-kata-containers-fix-umount-container-rootfs-dir-return-.patch -0015-kata-containers-truncate-the-log.json-file-before-kata-.patch -0016-kata-containers-validate-sandbox-cpu-and-memory-size.patch -0017-kata-containers-fix-delete-sandbox-failed-problem.patch -0018-kata-containers-check-VFIO-when-create-device.patch -0019-kata-containers-do-not-delete-the-exist-tap-device-in-t.patch -0020-kata-containers-do-not-ignore-updateInterface-return-er.patch -0021-kata-containers-fix-the-block-device-not-remove-in-devM.patch -0022-kata-containers-modify-stratovirt-config-file.patch -0023-stratovirt-update-configuration-toml-file.patch -0024-stratovirt-add-struct-vmConfig-and-methods-to-get-al.patch -0025-stratovirt-refactor-hypervisor-type-stratovirt-and-i.patch -0026-stratovirt-add-a-standard-virtual-machine-sandbox-ty.patch -0027-stratovirt-fix-the-problem-that-fails-to-plug-net-de.patch -0028-stratovirt-provide-a-way-to-dynomically-obtain-firmw.patch -0029-stratovirt-fix-the-problem-that-add-more-than-16-roo.patch -0030-use-host_device-blockdev-adding.patch -0031-add-explicit-on-after-kernel_irqchip.patch -0032-qmp-Don-t-use-deprecated-props-field-for-object-add.patch -0033-optimize-compile-options.patch -0034-stratovirt-Append-readonly-option-when-get-pflash-cm.patch -0035-update-cgroup-rs-crate.patch diff --git a/vendor.tar.gz b/vendor.tar.gz deleted file mode 100644 index 784b9c102c282a8ee70d8d4db9d5c2b60261d5eb..0000000000000000000000000000000000000000 Binary files a/vendor.tar.gz and /dev/null differ diff --git a/version.rs b/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..47038ad5653a7f8d9445f6ba1cccc40d1afb951e --- /dev/null +++ b/version.rs @@ -0,0 +1,18 @@ +// Copyright (c) 2020 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +// +// WARNING: This file is auto-generated - DO NOT EDIT! +// + +#![allow(dead_code)] + +pub const AGENT_VERSION: &str = "3.2.0"; +pub const API_VERSION: &str = "0.0.1"; +pub const VERSION_COMMIT: &str = "3.2.0-15e85b8ce740faf026e7d72d0d950cd9e0bbfc1d"; +pub const GIT_COMMIT: &str = "15e85b8ce740faf026e7d72d0d950cd9e0bbfc1d"; +pub const AGENT_NAME: &str = "kata-agent"; +pub const AGENT_DIR: &str = "/usr/bin"; +pub const AGENT_PATH: &str = "/usr/bin/kata-agent";