From cf2b34f477cba88641de3719bf5c8f933b919bcc Mon Sep 17 00:00:00 2001
From: holyfei <yangfeiyu20092010@163.com>
Date: Mon, 17 Aug 2020 21:44:57 +0800
Subject: [PATCH 35/50] device: mount blockdevices in the guest VM

reason: support mount blockdevices in the guest VM like
"-v /dev/blockdevice:/home/test"

Signed-off-by: yangfeiyu <yangfeiyu2@huawei.com>
---
 cli/config/configuration-qemu.toml.in |  5 ++++
 pkg/katautils/config.go               |  6 +++++
 virtcontainers/kata_agent.go          | 27 ++++++++++++++++---
 virtcontainers/kata_agent_test.go     |  2 +-
 virtcontainers/utils/utils.go         | 50 +++++++++++++++++++++++++++++++++++
 virtcontainers/vm_test.go             |  2 +-
 6 files changed, 86 insertions(+), 6 deletions(-)

diff --git a/cli/config/configuration-qemu.toml.in b/cli/config/configuration-qemu.toml.in
index 46f8b632..aa11b38f 100644
--- a/cli/config/configuration-qemu.toml.in
+++ b/cli/config/configuration-qemu.toml.in
@@ -391,6 +391,11 @@ path = "@SHIMPATH@"
 #
 kernel_modules=[]
 
+# If enabled, when we pass a block device to the guest VM
+# through -v, such as `docker run -v /dev/loop100:/foo/bar`,
+# agent will create a directory in guest VM and mount the
+# file system on `/dev/loop100` inside the container
+enable_blk_mount = true
 
 [netmon]
 # If enabled, the network monitoring process gets started when the
diff --git a/pkg/katautils/config.go b/pkg/katautils/config.go
index 94c916a0..51120311 100644
--- a/pkg/katautils/config.go
+++ b/pkg/katautils/config.go
@@ -160,6 +160,7 @@ type agent struct {
 	TraceMode     string   `toml:"trace_mode"`
 	TraceType     string   `toml:"trace_type"`
 	KernelModules []string `toml:"kernel_modules"`
+	MountBlkInVM  bool     `toml:"enable_blk_mount"`
 }
 
 type netmon struct {
@@ -463,6 +464,10 @@ func (h hypervisor) getInitrdAndImage() (initrd string, image string, err error)
 	return
 }
 
+func (a agent) mountBlkDevInVM() bool {
+	return a.MountBlkInVM
+}
+
 func (p proxy) path() (string, error) {
 	path := p.Path
 	if path == "" {
@@ -978,6 +983,7 @@ func updateRuntimeConfigAgent(configPath string, tomlConf tomlConfig, config *oc
 				TraceMode:     agent.traceMode(),
 				TraceType:     agent.traceType(),
 				KernelModules: agent.kernelModules(),
+				MountBlkInVM: agent.mountBlkDevInVM(),
 			}
 		default:
 			return fmt.Errorf("%s agent type is not supported", k)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 16662949..b0f88c15 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -30,6 +30,7 @@ import (
 	ns "github.com/kata-containers/runtime/virtcontainers/pkg/nsenter"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/rootless"
 	vcTypes "github.com/kata-containers/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/runtime/virtcontainers/store"
 	"github.com/kata-containers/runtime/virtcontainers/types"
@@ -64,6 +65,7 @@ var (
 	errorMissingOCISpec         = errors.New("Missing OCI specification")
 	defaultKataHostSharedDir    = "/run/kata-containers/shared/sandboxes/"
 	defaultKataGuestSharedDir   = "/run/kata-containers/shared/containers/"
+	kataGuestStorageDir         = "/run/kata-containers/storage/containers/"
 	mountGuestTag               = "kataShared"
 	defaultKataGuestSandboxDir  = "/run/kata-containers/sandbox/"
 	type9pFs                    = "9p"
@@ -199,6 +201,7 @@ type KataAgentConfig struct {
 	TraceMode         string
 	TraceType         string
 	KernelModules     []string
+	MountBlkInVM      bool
 }
 
 // KataAgentState is the structure describing the data stored from this
@@ -1061,7 +1064,12 @@ func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages
 			// Create a temporary location to mount the Storage. Mounting to the correct location
 			// will be handled by the OCI mount structure.
 			filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
-			path := filepath.Join(kataGuestSandboxStorageDir(), filename)
+			var path string
+			if v.Driver == kataBlkDevType || v.Driver == kataSCSIDevType {
+				path = filepath.Join(kataGuestStorageDir, filename)
+			} else {
+				path = filepath.Join(kataGuestSandboxStorageDir(), filename)
+			}
 
 			k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
 			ociMounts[index].Source = path
@@ -1436,7 +1444,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	// Note this call modifies the list of container devices to make sure
 	// all hotplugged devices are unplugged, so this needs be done
 	// after devices passed with --device are handled.
-	volumeStorages, err := k.handleBlockVolumes(c)
+	volumeStorages, err := k.handleBlockVolumes(sandbox, c)
 	if err != nil {
 		return nil, err
 	}
@@ -1609,7 +1617,7 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
 
 // handleBlockVolumes handles volumes that are block devices files
 // by passing the block devices as Storage to the agent.
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
+func (k *kataAgent) handleBlockVolumes(sandbox *Sandbox, c *Container) ([]*grpc.Storage, error) {
 
 	var volumeStorages []*grpc.Storage
 
@@ -1648,9 +1656,20 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 		}
 
 		vol.MountPoint = m.Destination
-		if vol.Fstype == "" {
+
+		ac, _ := sandbox.config.AgentConfig.(KataAgentConfig)
+		if ac.MountBlkInVM {
+			// Ensure the block device is formatted, for the devices here are specified as volumes
+			fsType, gerr := utils.GetDevFormat(m.Source)
+			if gerr != nil || fsType == "" {
+				k.Logger().WithField("device", id).WithError(gerr).Error("get device format failed")
+				return nil, gerr
+			}
+			vol.Fstype = fmt.Sprintf("bind-%s", fsType)
+		} else {
 			vol.Fstype = "bind"
 		}
+
 		if len(vol.Options) == 0 {
 			vol.Options = []string{"bind"}
 		}
diff --git a/virtcontainers/kata_agent_test.go b/virtcontainers/kata_agent_test.go
index 18a5a0a6..68caaab2 100644
--- a/virtcontainers/kata_agent_test.go
+++ b/virtcontainers/kata_agent_test.go
@@ -446,7 +446,7 @@ func TestHandleBlockVolume(t *testing.T) {
 	containers[c.id].sandbox = &sandbox
 	containers[c.id].mounts = mounts
 
-	volumeStorages, err := k.handleBlockVolumes(c)
+	volumeStorages, err := k.handleBlockVolumes(&sandbox, c)
 	assert.Nil(t, err, "Error while handling block volumes")
 
 	vStorage := &pb.Storage{
diff --git a/virtcontainers/utils/utils.go b/virtcontainers/utils/utils.go
index 9490faa1..36ac67a7 100644
--- a/virtcontainers/utils/utils.go
+++ b/virtcontainers/utils/utils.go
@@ -379,3 +379,53 @@ func RoundVCPUNumber(value string) (int, error) {
 	cpus := int(math.Ceil(cpuNum))
 	return cpus, nil
 }
+
+// GetDevFormat get the formated filesystem of input disk use `blkid`, Such as `ext4`,`xfs`,etc.
+func GetDevFormat(disk string) (string, error) {
+	// refer to https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/util/mount/mount_linux.go#L512
+	args := []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", disk}
+	dataOut, err := exec.Command("blkid", args...).Output()
+	output := string(dataOut)
+
+	if err != nil {
+		if strings.Contains(err.Error(), "exit status 2") {
+			// Disk device is unformatted.
+			// For `blkid`, if the specified token (TYPE/PTTYPE, etc) was
+			// not found, or no (specified) devices could be identified, an
+			// exit code of 2 is returned.
+			return "", nil
+		}
+		return "", err
+	}
+
+	var fstype string
+
+	lines := strings.Split(output, "\n")
+	for _, l := range lines {
+		if len(l) <= 0 {
+			// Ignore empty line.
+			continue
+		}
+		// if we use busybox as rootfs,the output of command
+		// `busybox blkid` is different with original`blkid`,
+		// so we should make a compatible parse
+		subLine := strings.Split(l, " ")
+		for _, sl := range subLine {
+			if len(subLine) <= 0 {
+				continue
+			}
+			cs := strings.Split(sl, "=")
+			if len(cs) != 2 {
+				continue
+			}
+			if cs[0] == "TYPE" {
+				fstype = cs[1]
+				if strings.Contains(fstype, `"`) {
+					fstype = strings.Replace(fstype, `"`, "", -1)
+				}
+			}
+		}
+	}
+
+	return fstype, nil
+}
diff --git a/virtcontainers/vm_test.go b/virtcontainers/vm_test.go
index 36fd5c2f..d2414232 100644
--- a/virtcontainers/vm_test.go
+++ b/virtcontainers/vm_test.go
@@ -125,7 +125,7 @@ func TestVMConfigGrpc(t *testing.T) {
 		HypervisorType:   QemuHypervisor,
 		HypervisorConfig: newQemuConfig(),
 		AgentType:        KataContainersAgent,
-		AgentConfig:      KataAgentConfig{false, true, false, false, 0, "", "", []string{}},
+		AgentConfig:      KataAgentConfig{false, true, false, false, 0, "", "", []string{}, false },
 		ProxyType:        NoopProxyType,
 	}
 
-- 
2.14.3 (Apple Git-98)

