From e861f426c9e6702e820348ddc61b18013c853402 Mon Sep 17 00:00:00 2001
From: jiangpengfei <jiangpengfei9@huawei.com>
Date: Thu, 13 Aug 2020 11:24:58 +0800
Subject: [PATCH 24/50] kata-runtime: support hotplug tap interface into kata
 VM

reason: support hotplug exist tap interface or a new created tap
interface into kata VM.

Signed-off-by: jiangpengfei <jiangpengfei9@huawei.com>
---
 cli/network.go                    |  45 ++++++++++++++---
 virtcontainers/kata_agent.go      |  12 ++++-
 virtcontainers/network.go         | 100 +++++++++++++++++++++++---------------
 virtcontainers/pkg/types/types.go |  16 +++---
 virtcontainers/qemu.go            |  11 +++--
 virtcontainers/sandbox.go         |  24 +++++++--
 virtcontainers/tap_endpoint.go    |  23 +++++++--
 7 files changed, 161 insertions(+), 70 deletions(-)

diff --git a/cli/network.go b/cli/network.go
index 881a2358..7e7791f1 100644
--- a/cli/network.go
+++ b/cli/network.go
@@ -26,6 +26,8 @@ const (
 	routeType
 )
 
+const defaultLinkType = "tap"
+
 var kataNetworkCLICommand = cli.Command{
 	Name:  "kata-network",
 	Usage: "manage interfaces and routes for container",
@@ -42,10 +44,22 @@ var kataNetworkCLICommand = cli.Command{
 }
 
 var addIfaceCommand = cli.Command{
-	Name:      "add-iface",
-	Usage:     "add an interface to a container",
-	ArgsUsage: `add-iface <container-id> file or - for stdin`,
-	Flags:     []cli.Flag{},
+	Name:  "add-iface",
+	Usage: "add an interface to a container",
+	ArgsUsage: `add-iface <container-id> file or - for stdin
+	file or stdin for example:
+       {
+		   "device":"<device-name>",
+		   "name":"<interface-name>",
+		   "IPAddresses":[{"address":"<ip>","mask":"<mask>"}],
+		   "mtu":<mtu>,
+		   "hwAddr":"<mac>",
+		   "linkType":"tap",
+		   "vhostUserSocket":"<path>"
+       }
+       device,name,mtu,hwAddr are required, IPAddresses and vhostUserSocket are optional.
+	`,
+	Flags: []cli.Flag{},
 	Action: func(context *cli.Context) error {
 		ctx, err := cliContextToContext(context)
 		if err != nil {
@@ -57,10 +71,20 @@ var addIfaceCommand = cli.Command{
 }
 
 var delIfaceCommand = cli.Command{
-	Name:      "del-iface",
-	Usage:     "delete an interface from a container",
-	ArgsUsage: `del-iface <container-id> file or - for stdin`,
-	Flags:     []cli.Flag{},
+	Name:  "del-iface",
+	Usage: "delete an interface from a container",
+	ArgsUsage: `del-iface <container-id> file or - for stdin
+    file or stdin for example:
+       {
+		   "device":"",
+		   "name":"<interface-name>",
+		   "IPAddresses":[],
+		   "mtu":0,
+		   "hwAddr":""
+       }
+       Only the "name" field is required.
+	`,
+	Flags: []cli.Flag{},
 	Action: func(context *cli.Context) error {
 		ctx, err := cliContextToContext(context)
 		if err != nil {
@@ -156,6 +180,11 @@ func networkModifyCommand(ctx context.Context, containerID, input string, opType
 		if err = json.NewDecoder(f).Decode(&inf); err != nil {
 			return err
 		}
+
+		if len(inf.LinkType) == 0 {
+			inf.LinkType = defaultLinkType
+		}
+
 		if add {
 			resultingInf, err = vci.AddInterface(ctx, sandboxID, inf)
 			if err != nil {
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 8e073339..dfdd263b 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -600,8 +600,16 @@ func (k *kataAgent) updateInterface(ifc *vcTypes.Interface) (*vcTypes.Interface,
 			"resulting-interface": fmt.Sprintf("%+v", resultingInterface),
 		}).WithError(err).Error("update interface request failed")
 	}
-	if resultInterface, ok := resultingInterface.(*vcTypes.Interface); ok {
-		return resultInterface, err
+	if resultInterface, ok := resultingInterface.(*aTypes.Interface); ok {
+		iface := &vcTypes.Interface{
+			Device:      resultInterface.Device,
+			Name:        resultInterface.Name,
+			IPAddresses: k.convertToIPAddresses(resultInterface.IPAddresses),
+			Mtu:         resultInterface.Mtu,
+			HwAddr:      resultInterface.HwAddr,
+			PciAddr:     resultInterface.PciAddr,
+		}
+		return iface, err
 	}
 	return nil, err
 }
diff --git a/virtcontainers/network.go b/virtcontainers/network.go
index d70c5360..e909a822 100644
--- a/virtcontainers/network.go
+++ b/virtcontainers/network.go
@@ -117,6 +117,7 @@ type NetlinkIface struct {
 // NetworkInfo gathers all information related to a network interface.
 // It can be used to store the description of the underlying network.
 type NetworkInfo struct {
+	Device string
 	Iface  NetlinkIface
 	Addrs  []netlink.Addr
 	Routes []netlink.Route
@@ -303,6 +304,20 @@ func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Lin
 	var newLink netlink.Link
 	var fds []*os.File
 
+	// check if tapname exists, if true, use existing one instead of create new one
+	if name != "" {
+		retLink, err := netlink.LinkByName(name)
+		// link exist, use it
+		if err == nil {
+			networkLogger().Debugf("exist tap device is found, instead of creating new one")
+			tuntapLink, ok := retLink.(*netlink.Tuntap)
+			if ok {
+				fds = tuntapLink.Fds
+			}
+			return retLink, nil, nil
+		}
+	}
+
 	switch expectedLink.Type() {
 	case (&netlink.Tuntap{}).Type():
 		flags := netlink.TUNTAP_VNET_HDR
@@ -1156,7 +1171,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
 
 	// Check if interface is a physical interface. Do not create
 	// tap interface/bridge if it is.
-	isPhysical, err := isPhysicalIface(netInfo.Iface.Name)
+	isPhysical, err := isPhysicalIface(netInfo.Device)
 	if err != nil {
 		return nil, err
 	}
@@ -1164,49 +1179,54 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
 	if isPhysical {
 		networkLogger().WithField("interface", netInfo.Iface.Name).Info("Physical network interface found")
 		endpoint, err = createPhysicalEndpoint(netInfo)
-	} else {
-		var socketPath string
+		return endpoint, err
+	}
 
-		// Check if this is a dummy interface which has a vhost-user socket associated with it
-		socketPath, err = vhostUserSocketPath(netInfo)
-		if err != nil {
-			return nil, err
-		}
+	// Check if this is a dummy interface which has a vhost-user socket associated with it
+	socketPath, err := vhostUserSocketPath(netInfo)
+	if err != nil {
+		return nil, err
+	}
 
-		if socketPath != "" {
-			networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
-			endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
-		} else if netInfo.Iface.Type == "macvlan" {
-			networkLogger().Infof("macvlan interface found")
-			endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
-		} else if netInfo.Iface.Type == "macvtap" {
-			networkLogger().Infof("macvtap interface found")
-			endpoint, err = createMacvtapNetworkEndpoint(netInfo)
-		} else if netInfo.Iface.Type == "tap" {
-			networkLogger().Info("tap interface found")
-			endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name)
-		} else if netInfo.Iface.Type == "tuntap" {
-			if link != nil {
-				switch link.(*netlink.Tuntap).Mode {
-				case 0:
-					// mount /sys/class/net to get links
-					return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
-				case 1:
-					return nil, fmt.Errorf("tun networking device not yet supported")
-				case 2:
-					networkLogger().Info("tuntap tap interface found")
-					endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
-				default:
-					return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
-				}
+	if socketPath != "" {
+		networkLogger().WithField("interface", netInfo.Iface.Name).Info("VhostUser network interface found")
+		endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
+		return endpoint, err
+	}
+
+	// We should create tap interface/bridge of other interface type.
+	networkLogger().Infof("%s interface found", netInfo.Iface.Type)
+	switch netInfo.Iface.Type {
+	case "macvlan":
+		networkLogger().Infof("macvlan interface found")
+		endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
+	case "macvtap":
+		networkLogger().Infof("macvtap interface found")
+		endpoint, err = createMacvtapNetworkEndpoint(netInfo)
+	case "tap":
+		networkLogger().Info("tap interface found")
+		endpoint, err = createTapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Device)
+	case "tuntap":
+		if link != nil {
+			switch link.(*netlink.Tuntap).Mode {
+			case 0:
+				// mount /sys/class/net to get links
+				return nil, fmt.Errorf("Network device mode not determined correctly. Mount sysfs in caller")
+			case 1:
+				return nil, fmt.Errorf("tun networking device not yet supported")
+			case 2:
+				networkLogger().Info("tuntap tap interface found")
+				endpoint, err = createTuntapNetworkEndpoint(idx, netInfo.Iface.Name, netInfo.Iface.HardwareAddr, model)
+			default:
+				return nil, fmt.Errorf("tuntap network %v mode unsupported", link.(*netlink.Tuntap).Mode)
 			}
-		} else if netInfo.Iface.Type == "veth" {
-			endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
-		} else if netInfo.Iface.Type == "ipvlan" {
-			endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
-		} else {
-			return nil, fmt.Errorf("Unsupported network interface: %s", netInfo.Iface.Type)
 		}
+	case "veth":
+		endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model)
+	case "ipvlan":
+		endpoint, err = createIPVlanNetworkEndpoint(idx, netInfo.Iface.Name)
+	default:
+		err = fmt.Errorf("Unsupported network interface, %s", netInfo.Iface.Type)
 	}
 
 	return endpoint, err
diff --git a/virtcontainers/pkg/types/types.go b/virtcontainers/pkg/types/types.go
index 0d4a9cfa..fcc63d84 100644
--- a/virtcontainers/pkg/types/types.go
+++ b/virtcontainers/pkg/types/types.go
@@ -14,21 +14,21 @@ type IPAddress struct {
 
 // Interface describes a network interface.
 type Interface struct {
-	Device      string
-	Name        string
-	IPAddresses []*IPAddress
-	Mtu         uint64
-	RawFlags    uint32
-	HwAddr      string
+	Device      string       `json:"device,omitempty"`
+	Name        string       `json:"name,omitempty"`
+	IPAddresses []*IPAddress `json:"IPAddresses,omitempty"`
+	Mtu         uint64       `json:"mtu,omitempty"`
+	RawFlags    uint32       `json:"rawFlags,omitempty"`
+	HwAddr      string       `json:"hwAddr,omitempty"`
 	// pciAddr is the PCI address in the format  "bridgeAddr/deviceAddr".
 	// Here, bridgeAddr is the address at which the bridge is attached on the root bus,
 	// while deviceAddr is the address at which the network device is attached on the bridge.
-	PciAddr string
+	PciAddr string `json:"pciAddr,omitempty"`
 	// LinkType defines the type of interface described by this structure.
 	// The expected values are the one that are defined by the netlink
 	// library, regarding each type of link. Here is a non exhaustive
 	// list: "veth", "macvtap", "vlan", "macvlan", "tap", ...
-	LinkType string
+	LinkType string `json:"linkType,omitempty"`
 }
 
 // Route describes a network route.
diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go
index 7bae3278..bb83b1bb 100644
--- a/virtcontainers/qemu.go
+++ b/virtcontainers/qemu.go
@@ -1361,7 +1361,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) (err erro
 	return nil
 }
 
-func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File) error {
+func (q *qemu) hotAddNetDevice(deviceName, name, hardAddr string, VMFds, VhostFds []*os.File) error {
 	var (
 		VMFdNames    []string
 		VhostFdNames []string
@@ -1381,7 +1381,12 @@ func (q *qemu) hotAddNetDevice(name, hardAddr string, VMFds, VhostFds []*os.File
 		VhostFd.Close()
 		VhostFdNames = append(VhostFdNames, fdName)
 	}
-	return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
+
+	if len(VMFdNames) != 0 || len(VhostFdNames) != 0 {
+		return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", name, VMFdNames, VhostFdNames)
+	}
+
+	return q.qmpMonitorCh.qmp.ExecuteNetdevAdd(q.qmpMonitorCh.ctx, "tap", name, deviceName, "no", "no", 0)
 }
 
 func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
@@ -1404,7 +1409,7 @@ func (q *qemu) hotplugNetDevice(endpoint Endpoint, op operation) (err error) {
 
 	devID := "virtio-" + tap.ID
 	if op == addDevice {
-		if err = q.hotAddNetDevice(tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
+		if err = q.hotAddNetDevice(tap.TAPIface.Name, tap.Name, endpoint.HardwareAddr(), tap.VMFds, tap.VhostFds); err != nil {
 			return err
 		}
 
diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go
index ba704249..c8981a41 100644
--- a/virtcontainers/sandbox.go
+++ b/virtcontainers/sandbox.go
@@ -937,6 +937,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
 	}
 
 	return NetworkInfo{
+		Device: inf.Device,
 		Iface: NetlinkIface{
 			LinkAttrs: netlink.LinkAttrs{
 				Name:         inf.Name,
@@ -950,7 +951,7 @@ func (s *Sandbox) generateNetInfo(inf *vcTypes.Interface) (NetworkInfo, error) {
 }
 
 // AddInterface adds new nic to the sandbox.
-func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
+func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (grpcIf *vcTypes.Interface, err error) {
 	netInfo, err := s.generateNetInfo(inf)
 	if err != nil {
 		return nil, err
@@ -962,28 +963,41 @@ func (s *Sandbox) AddInterface(inf *vcTypes.Interface) (*vcTypes.Interface, erro
 	}
 
 	endpoint.SetProperties(netInfo)
-	if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
+	if err = doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error {
 		s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint")
 		return endpoint.HotAttach(s.hypervisor)
 	}); err != nil {
 		return nil, err
 	}
 
+	defer func() {
+		if err != nil {
+			if errDetach := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); errDetach != nil {
+				s.Logger().WithField("endpoint-type", endpoint.Type()).Errorf("rollback hot attach endpoint failed")
+			}
+		}
+	}()
+
 	// Update the sandbox storage
 	s.networkNS.Endpoints = append(s.networkNS.Endpoints, endpoint)
-	if err := s.Save(); err != nil {
+	if err = s.Save(); err != nil {
 		return nil, err
 	}
 
 	// Add network for vm
 	inf.PciAddr = endpoint.PciAddr()
-	return s.agent.updateInterface(inf)
+	grpcIf, err = s.agent.updateInterface(inf)
+	if err != nil {
+		return nil, err
+	}
+
+	return
 }
 
 // RemoveInterface removes a nic of the sandbox.
 func (s *Sandbox) RemoveInterface(inf *vcTypes.Interface) (*vcTypes.Interface, error) {
 	for i, endpoint := range s.networkNS.Endpoints {
-		if endpoint.HardwareAddr() == inf.HwAddr {
+		if endpoint.HardwareAddr() == inf.HwAddr || endpoint.Name() == inf.Name {
 			s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint")
 			if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil {
 				return inf, err
diff --git a/virtcontainers/tap_endpoint.go b/virtcontainers/tap_endpoint.go
index cb441b87..7d33d5a2 100644
--- a/virtcontainers/tap_endpoint.go
+++ b/virtcontainers/tap_endpoint.go
@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
@@ -111,7 +112,7 @@ func (endpoint *TapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPat
 	return nil
 }
 
-func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
+func createTapNetworkEndpoint(idx int, ifName string, tapIfName string) (*TapEndpoint, error) {
 	if idx < 0 {
 		return &TapEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
 	}
@@ -131,6 +132,10 @@ func createTapNetworkEndpoint(idx int, ifName string) (*TapEndpoint, error) {
 		endpoint.TapInterface.Name = ifName
 	}
 
+	if tapIfName != "" {
+		endpoint.TapInterface.TAPIface.Name = tapIfName
+	}
+
 	return endpoint, nil
 }
 
@@ -145,9 +150,19 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
 	if err != nil {
 		return fmt.Errorf("Could not create TAP interface: %s", err)
 	}
+
+	defer func() {
+		if err != nil {
+			if errDel := netHandle.LinkDel(tapLink); errDel != nil {
+				networkLogger().WithError(errDel).Error("tapNetwork fail to rollback del link")
+			}
+		}
+	}()
+
 	endpoint.TapInterface.VMFds = fds
 	if !disableVhostNet {
-		vhostFds, err := createVhostFds(int(numCPUs))
+		var vhostFds []*os.File
+		vhostFds, err = createVhostFds(int(numCPUs))
 		if err != nil {
 			return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.TapInterface.Name, err)
 		}
@@ -161,10 +176,10 @@ func tapNetwork(endpoint *TapEndpoint, numCPUs uint32, disableVhostNet bool) err
 	// bridge created by the network plugin on the host actually expects
 	// to see traffic from this MAC address and not another one.
 	endpoint.TapInterface.TAPIface.HardAddr = linkAttrs.HardwareAddr.String()
-	if err := netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
+	if err = netHandle.LinkSetMTU(tapLink, linkAttrs.MTU); err != nil {
 		return fmt.Errorf("Could not set TAP MTU %d: %s", linkAttrs.MTU, err)
 	}
-	if err := netHandle.LinkSetUp(tapLink); err != nil {
+	if err = netHandle.LinkSetUp(tapLink); err != nil {
 		return fmt.Errorf("Could not enable TAP %s: %s", endpoint.TapInterface.Name, err)
 	}
 	return nil
-- 
2.14.3 (Apple Git-98)

