Ken/tar2cim

clean.ps1

@@ -1 +1,2 @@
+go clean -i -r -cache
 del dmverity-vhd.exe

cmd/dmverity-vhd/cim.go

@@ -51,7 +51,7 @@ func tarToCim(tarReader io.Reader, parentLayers ParentLayers, out string, layerN
 		cimimport.WithLayerIntegrity(),
 	}
 
-	log.Tracef("before cimimport.ImportBlockCIMLayerWithOpts for layer %s", layerName)
+	log.Tracef("before cimimport.ImportBlockCIMLayerWithOpts for layer %s cim name %s", layerName, cimName)
 	size, importErr := cimimport.ImportBlockCIMLayerWithOpts(context.Background(), tarReader, blockCIM, importOpts...)
 	log.Tracef("after cimimport.ImportBlockCIMLayerWithOpts for layer %s, size %d", layerName, size)
 	if importErr != nil {

cmd/dmverity-vhd/containerregistry.go

@@ -81,6 +81,8 @@ func fetchContainerRegistryImage(
 		return nil, fmt.Errorf("unable to fetch image %q, make sure it exists: %w", imageName, err)
 	}
 
+	log.Tracef("done - fetchContainerRegistryImage %s", imageName)
+
 	return
 }
 

cmd/dmverity-vhd/hashlayer.go

@@ -12,49 +12,111 @@ import (
 )
 
 func parseHashLayerArgs(ctx *cli.Context) (tarPath string, platform string, err error) {
-	log.Trace("parseHashLayerArgs called")
-
 	tarPath = ctx.String(inputFlag)
 	platform = ctx.String(platformFlag)
 	return
 }
 
-func hashLayer(tarPath string, platform string) error {
+func hashLayer(tarPath string, platform string) (string, error) {
 	log.Trace("hashLayer called")
 
 	tarReader, err := os.Open(tarPath)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer tarReader.Close()
 
 	entryReader, closer, err := decompressIfNeeded(tarReader)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if closer != nil {
 		defer closer.Close()
 	}
 
 	entryReader, isTar := isTar(entryReader)
 	if !isTar {
-		return fmt.Errorf("input file is not a tar archive")
+		return "", fmt.Errorf("input file is not a tar archive")
 	}
 
 	var hash string
 	if strings.HasPrefix(platform, "linux") {
+		log.Trace("Using tar2ext4 ConvertAndComputeRootDigest")
 		hash, err = tar2ext4.ConvertAndComputeRootDigest(entryReader)
 	} else if strings.HasPrefix(platform, "windows") {
 		cimOut, err := os.MkdirTemp("", filepath.Base(tarPath))
 		if err != nil {
-			return err
+			return "", err
 		}
 		parentLayers := make(ParentLayers, 0)
+		log.Trace("tar2cim")
 		hash, _, err = tarToCim(entryReader, parentLayers, cimOut, filepath.Base(tarPath))
 	}
 	if err != nil {
-		return err
+		return "", err
+	}
+	log.Tracef("done hashLayer: %s", hash)
+	return hash, nil
+}
+
+func parseTar2HashedArgs(ctx *cli.Context) (tarPath string, platform string, err error) {
+	log.Trace("parseHashLayerArgs called")
+
+	tarPath = ctx.String(inputFlag)
+	platform = ctx.String(platformFlag)
+	return
+}
+
+func tar2hashed(tarPath string, destPath string, cimOrext4 string) (string, error) {
+	log.Trace("tar2hashed called")
+
+	tarReader, err := os.Open(tarPath)
+	if err != nil {
+		return "", err
+	}
+	defer tarReader.Close()
+
+	entryReader, closer, err := decompressIfNeeded(tarReader)
+	if err != nil {
+		return "", err
+	}
+	if closer != nil {
+		defer closer.Close()
+	}
+
+	entryReader, isTar := isTar(entryReader)
+	if !isTar {
+		return "", fmt.Errorf("input file is not a tar archive")
+	}
+
+	var hash string
+	if cimOrext4 == "ext4" {
+		opts := []tar2ext4.Option{
+			tar2ext4.ConvertWhiteout,
+		}
+
+		opts = append(opts, tar2ext4.AppendDMVerity)
+		out, err := os.Create(destPath)
+		if err != nil {
+			return "", fmt.Errorf("failed to create layer file %s: %w", destPath, err)
+		}
+		defer out.Close()
+
+		log.Trace("Using tar2ext4 Convert")
+		err = tar2ext4.Convert(entryReader, out, opts...)
+	} else if cimOrext4 == "cim" {
+		if err != nil {
+			return "", err
+		}
+		parentLayers := make(ParentLayers, 0)
+		cimOutPath := filepath.Dir(destPath)
+		layerName := filepath.Base(destPath)
+		log.Trace("tar2cim")
+		hash, _, err = tarToCim(entryReader, parentLayers, cimOutPath, layerName)
+	}
+	if err != nil {
+		return "", err
 	}
-	fmt.Printf("%s\n", hash)
-	return nil
+	log.Tracef("done tar2hashed: %s", hash)
+	return hash, nil
 }

cmd/dmverity-vhd/instrumentation.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"runtime"
+	"runtime/pprof"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -18,6 +20,27 @@ func setLoggingLevel(ctx *cli.Context) {
 	}
 }
 
+var profilerEnabled bool = false
+
+func setupProfiler(ctx *cli.Context) {
+	profilerPath := ctx.GlobalString(profilerFlag)
+
+	if len(profilerPath) > 0 {
+		f, err := os.Create(profilerPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		profilerEnabled = true
+	}
+}
+
+func stopProfiler(ctx *cli.Context) {
+	if profilerEnabled {
+		pprof.StopCPUProfile()
+	}
+}
+
 func TraceMemUsage() {
 	if log.IsLevelEnabled(log.TraceLevel) {
 		var m runtime.MemStats

cmd/dmverity-vhd/main.go

@@ -1,12 +1,14 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 
+
 	"github.com/Microsoft/hcsshim/ext4/dmverity"
 )
 
@@ -15,8 +17,11 @@ const (
 	passwordFlag       = "password"
 	platformFlag       = "platform"
 	inputFlag          = "input"
+	outputFlag         = "output"
+	typeFlag           = "type"
 	verboseFlag        = "verbose"
 	traceFlag          = "trace"
+	profilerFlag       = "profiler" // enable profiling
 	outputDirFlag      = "out-dir"
 	dockerFlag         = "docker"
 	bufferedReaderFlag = "buffered-reader"
@@ -50,8 +55,9 @@ func main() {
 		createVHDCommand,
 		rootHashVHDCommand,
 		hashLayerCommand,
+		tar2hashedCommand,
 	}
-	app.Usage = "dmverity-vhd is a command line tool for creating LCOW layer VHDs with dm-verity hashes."
+	app.Usage = "dmverity-vhd is a command line tool for creating LCOW layer VHDs with dm-verity hashes and WCOW layer integrity checked CIMs."
 	app.Flags = []cli.Flag{
 		cli.BoolFlag{
 			Name:  verboseFlag + ",v",
@@ -73,6 +79,10 @@ func main() {
 			Name:  bufferedReaderFlag + ",b",
 			Usage: "Optional: use buffered opener for image",
 		},
+		cli.StringFlag{
+			Name:  profilerFlag,
+			Usage: "Optional: profile and put the results in this file",
+		},
 	}
 
 	if err := app.Run(os.Args); err != nil {
@@ -113,14 +123,17 @@ var createVHDCommand = cli.Command{
 		},
 	},
 	Action: func(ctx *cli.Context) error {
+		setupProfiler(ctx)
 		setLoggingLevel(ctx)
 		log.Trace("createVHDCommand called")
 
 		imageName, outDir, verityHashDev, verityData, imageFetcher, imageParser, manifestParser, err := parseCreateVhdArgs(ctx)
 		if err != nil {
 			return err
 		}
-		return createVhd(imageFetcher, imageParser, manifestParser, imageName, outDir, verityHashDev, verityData)
+		err = createVhd(imageFetcher, imageParser, manifestParser, imageName, outDir, verityHashDev, verityData)
+		stopProfiler(ctx)
+		return err
 	},
 }
 
@@ -148,19 +161,22 @@ var rootHashVHDCommand = cli.Command{
 		},
 	},
 	Action: func(ctx *cli.Context) error {
+		setupProfiler(ctx)
 		setLoggingLevel(ctx)
 		log.Trace("rootHashVHDCommand called")
 
 		imageFetcher, imageParser, manifestParser, layerParser, err := parseRoothashArgs(ctx)
 		if err != nil {
 			return err
 		}
-		return roothash(imageFetcher, imageParser, manifestParser, layerParser)
+		err = roothash(imageFetcher, imageParser, manifestParser, layerParser)
+		stopProfiler(ctx)
+		return err
 	},
 }
 
 var hashLayerCommand = cli.Command{
-	Name:  "hashLayer",
+	Name:  "hashlayer",
 	Usage: "compute root hashes for each LCOW layer VHD",
 	Flags: []cli.Flag{
 		cli.StringFlag{
@@ -175,13 +191,64 @@ var hashLayerCommand = cli.Command{
 		},
 	},
 	Action: func(ctx *cli.Context) error {
+		setupProfiler(ctx)
 		setLoggingLevel(ctx)
 		log.Trace("hashLayerCommand called")
 
 		tarPath, platform, err := parseHashLayerArgs(ctx)
 		if err != nil {
 			return err
 		}
-		return hashLayer(tarPath, platform)
+
+		hash, err := hashLayer(tarPath, platform)
+		fmt.Printf("%s\n", hash)
+		log.Trace("hashLayer done")
+		stopProfiler(ctx)
+		return err
+	},
+}
+
+var tar2hashedCommand = cli.Command{
+	Name:  "tar2hashed",
+	Usage: "convert from tar to integrity protected ext4fs or CIMfs",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:     inputFlag + ",i",
+			Usage:    "Required: path to layer tar",
+			Required: true,
+		},
+		cli.StringFlag{
+			Name:     outputFlag + ",o",
+			Usage:    "Required: path to resulting file",
+			Required: true,
+		},
+		cli.StringFlag{
+			Name:     typeFlag + ",t",
+			Usage:    "Required: output image type, cim or ext4",
+			Required: true,
+		},
+	},
+	Action: func(ctx *cli.Context) error {
+		setupProfiler(ctx)
+		setLoggingLevel(ctx)
+		log.Trace("tar2hashedCommand called")
+
+		srcTarPath := ctx.String(inputFlag)
+		destPath := ctx.String(outputFlag)
+		cimOrext4 := ctx.String(typeFlag)
+
+		if cimOrext4 != "cim" && cimOrext4 != "ext4" {
+			return errors.New("type must be either cim or ext4")
+		}
+
+		hash, err := tar2hashed(srcTarPath, destPath, cimOrext4)
+		if err != nil {
+			log.Infof("tar2hash failed: %s", err.Error())
+		} else {
+			log.Infof("%s", hash)
+		}
+		log.Trace("tar2hashedCommand done")
+		stopProfiler(ctx)
+		return nil
 	},
 }