diff --git a/clean.ps1 b/clean.ps1 index c9701dd..2de2b91 100644 --- a/clean.ps1 +++ b/clean.ps1 @@ -1 +1,2 @@ +go clean -i -r -cache del dmverity-vhd.exe diff --git a/cmd/dmverity-vhd/cim.go b/cmd/dmverity-vhd/cim.go index 8db0659..c13e042 100644 --- a/cmd/dmverity-vhd/cim.go +++ b/cmd/dmverity-vhd/cim.go @@ -51,7 +51,7 @@ func tarToCim(tarReader io.Reader, parentLayers ParentLayers, out string, layerN cimimport.WithLayerIntegrity(), } - log.Tracef("before cimimport.ImportBlockCIMLayerWithOpts for layer %s", layerName) + log.Tracef("before cimimport.ImportBlockCIMLayerWithOpts for layer %s cim name %s", layerName, cimName) size, importErr := cimimport.ImportBlockCIMLayerWithOpts(context.Background(), tarReader, blockCIM, importOpts...) log.Tracef("after cimimport.ImportBlockCIMLayerWithOpts for layer %s, size %d", layerName, size) if importErr != nil { diff --git a/cmd/dmverity-vhd/containerregistry.go b/cmd/dmverity-vhd/containerregistry.go index 825332c..d9c1c0b 100644 --- a/cmd/dmverity-vhd/containerregistry.go +++ b/cmd/dmverity-vhd/containerregistry.go @@ -81,6 +81,8 @@ func fetchContainerRegistryImage( return nil, fmt.Errorf("unable to fetch image %q, make sure it exists: %w", imageName, err) } + log.Tracef("done - fetchContainerRegistryImage %s", imageName) + return } diff --git a/cmd/dmverity-vhd/hashlayer.go b/cmd/dmverity-vhd/hashlayer.go index a71c7b4..41ef9a7 100644 --- a/cmd/dmverity-vhd/hashlayer.go +++ b/cmd/dmverity-vhd/hashlayer.go @@ -12,25 +12,23 @@ import ( ) func parseHashLayerArgs(ctx *cli.Context) (tarPath string, platform string, err error) { - log.Trace("parseHashLayerArgs called") - tarPath = ctx.String(inputFlag) platform = ctx.String(platformFlag) return } -func hashLayer(tarPath string, platform string) error { +func hashLayer(tarPath string, platform string) (string, error) { log.Trace("hashLayer called") tarReader, err := os.Open(tarPath) if err != nil { - return err + return "", err } defer tarReader.Close() entryReader, closer, err := decompressIfNeeded(tarReader) if err != nil { - return err + return "", err } if closer != nil { defer closer.Close() @@ -38,23 +36,87 @@ func hashLayer(tarPath string, platform string) error { entryReader, isTar := isTar(entryReader) if !isTar { - return fmt.Errorf("input file is not a tar archive") + return "", fmt.Errorf("input file is not a tar archive") } var hash string if strings.HasPrefix(platform, "linux") { + log.Trace("Using tar2ext4 ConvertAndComputeRootDigest") hash, err = tar2ext4.ConvertAndComputeRootDigest(entryReader) } else if strings.HasPrefix(platform, "windows") { cimOut, err := os.MkdirTemp("", filepath.Base(tarPath)) if err != nil { - return err + return "", err } parentLayers := make(ParentLayers, 0) + log.Trace("tar2cim") hash, _, err = tarToCim(entryReader, parentLayers, cimOut, filepath.Base(tarPath)) } if err != nil { - return err + return "", err + } + log.Tracef("done hashLayer: %s", hash) + return hash, nil +} + +func parseTar2HashedArgs(ctx *cli.Context) (tarPath string, platform string, err error) { + log.Trace("parseHashLayerArgs called") + + tarPath = ctx.String(inputFlag) + platform = ctx.String(platformFlag) + return +} + +func tar2hashed(tarPath string, destPath string, cimOrext4 string) (string, error) { + log.Trace("tar2hashed called") + + tarReader, err := os.Open(tarPath) + if err != nil { + return "", err + } + defer tarReader.Close() + + entryReader, closer, err := decompressIfNeeded(tarReader) + if err != nil { + return "", err + } + if closer != nil { + defer closer.Close() + } + + entryReader, isTar := isTar(entryReader) + if !isTar { + return "", fmt.Errorf("input file is not a tar archive") + } + + var hash string + if cimOrext4 == "ext4" { + opts := []tar2ext4.Option{ + tar2ext4.ConvertWhiteout, + } + + opts = append(opts, tar2ext4.AppendDMVerity) + out, err := os.Create(destPath) + if err != nil { + return "", fmt.Errorf("failed to create layer file %s: %w", destPath, err) + } + defer out.Close() + + log.Trace("Using tar2ext4 Convert") + err = tar2ext4.Convert(entryReader, out, opts...) + } else if cimOrext4 == "cim" { + if err != nil { + return "", err + } + parentLayers := make(ParentLayers, 0) + cimOutPath := filepath.Dir(destPath) + layerName := filepath.Base(destPath) + log.Trace("tar2cim") + hash, _, err = tarToCim(entryReader, parentLayers, cimOutPath, layerName) + } + if err != nil { + return "", err } - fmt.Printf("%s\n", hash) - return nil + log.Tracef("done tar2hashed: %s", hash) + return hash, nil } diff --git a/cmd/dmverity-vhd/instrumentation.go b/cmd/dmverity-vhd/instrumentation.go index 073ea0e..ec6c19f 100644 --- a/cmd/dmverity-vhd/instrumentation.go +++ b/cmd/dmverity-vhd/instrumentation.go @@ -1,7 +1,9 @@ package main import ( + "os" "runtime" + "runtime/pprof" log "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -18,6 +20,27 @@ func setLoggingLevel(ctx *cli.Context) { } } +var profilerEnabled bool = false + +func setupProfiler(ctx *cli.Context) { + profilerPath := ctx.GlobalString(profilerFlag) + + if len(profilerPath) > 0 { + f, err := os.Create(profilerPath) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + profilerEnabled = true + } +} + +func stopProfiler(ctx *cli.Context) { + if profilerEnabled { + pprof.StopCPUProfile() + } +} + func TraceMemUsage() { if log.IsLevelEnabled(log.TraceLevel) { var m runtime.MemStats diff --git a/cmd/dmverity-vhd/main.go b/cmd/dmverity-vhd/main.go index 5afba79..d191355 100644 --- a/cmd/dmverity-vhd/main.go +++ b/cmd/dmverity-vhd/main.go @@ -1,12 +1,14 @@ package main import ( + "errors" "fmt" "os" log "github.com/sirupsen/logrus" "github.com/urfave/cli" + "github.com/Microsoft/hcsshim/ext4/dmverity" ) @@ -15,8 +17,11 @@ const ( passwordFlag = "password" platformFlag = "platform" inputFlag = "input" + outputFlag = "output" + typeFlag = "type" verboseFlag = "verbose" traceFlag = "trace" + profilerFlag = "profiler" // enable profiling outputDirFlag = "out-dir" dockerFlag = "docker" bufferedReaderFlag = "buffered-reader" @@ -50,8 +55,9 @@ func main() { createVHDCommand, rootHashVHDCommand, hashLayerCommand, + tar2hashedCommand, } - app.Usage = "dmverity-vhd is a command line tool for creating LCOW layer VHDs with dm-verity hashes." + app.Usage = "dmverity-vhd is a command line tool for creating LCOW layer VHDs with dm-verity hashes and WCOW layer integrity checked CIMs." app.Flags = []cli.Flag{ cli.BoolFlag{ Name: verboseFlag + ",v", @@ -73,6 +79,10 @@ func main() { Name: bufferedReaderFlag + ",b", Usage: "Optional: use buffered opener for image", }, + cli.StringFlag{ + Name: profilerFlag, + Usage: "Optional: profile and put the results in this file", + }, } if err := app.Run(os.Args); err != nil { @@ -113,6 +123,7 @@ var createVHDCommand = cli.Command{ }, }, Action: func(ctx *cli.Context) error { + setupProfiler(ctx) setLoggingLevel(ctx) log.Trace("createVHDCommand called") @@ -120,7 +131,9 @@ var createVHDCommand = cli.Command{ if err != nil { return err } - return createVhd(imageFetcher, imageParser, manifestParser, imageName, outDir, verityHashDev, verityData) + err = createVhd(imageFetcher, imageParser, manifestParser, imageName, outDir, verityHashDev, verityData) + stopProfiler(ctx) + return err }, } @@ -148,6 +161,7 @@ var rootHashVHDCommand = cli.Command{ }, }, Action: func(ctx *cli.Context) error { + setupProfiler(ctx) setLoggingLevel(ctx) log.Trace("rootHashVHDCommand called") @@ -155,12 +169,14 @@ var rootHashVHDCommand = cli.Command{ if err != nil { return err } - return roothash(imageFetcher, imageParser, manifestParser, layerParser) + err = roothash(imageFetcher, imageParser, manifestParser, layerParser) + stopProfiler(ctx) + return err }, } var hashLayerCommand = cli.Command{ - Name: "hashLayer", + Name: "hashlayer", Usage: "compute root hashes for each LCOW layer VHD", Flags: []cli.Flag{ cli.StringFlag{ @@ -175,6 +191,7 @@ var hashLayerCommand = cli.Command{ }, }, Action: func(ctx *cli.Context) error { + setupProfiler(ctx) setLoggingLevel(ctx) log.Trace("hashLayerCommand called") @@ -182,6 +199,56 @@ var hashLayerCommand = cli.Command{ if err != nil { return err } - return hashLayer(tarPath, platform) + + hash, err := hashLayer(tarPath, platform) + fmt.Printf("%s\n", hash) + log.Trace("hashLayer done") + stopProfiler(ctx) + return err + }, +} + +var tar2hashedCommand = cli.Command{ + Name: "tar2hashed", + Usage: "convert from tar to integrity protected ext4fs or CIMfs", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: inputFlag + ",i", + Usage: "Required: path to layer tar", + Required: true, + }, + cli.StringFlag{ + Name: outputFlag + ",o", + Usage: "Required: path to resulting file", + Required: true, + }, + cli.StringFlag{ + Name: typeFlag + ",t", + Usage: "Required: output image type, cim or ext4", + Required: true, + }, + }, + Action: func(ctx *cli.Context) error { + setupProfiler(ctx) + setLoggingLevel(ctx) + log.Trace("tar2hashedCommand called") + + srcTarPath := ctx.String(inputFlag) + destPath := ctx.String(outputFlag) + cimOrext4 := ctx.String(typeFlag) + + if cimOrext4 != "cim" && cimOrext4 != "ext4" { + return errors.New("type must be either cim or ext4") + } + + hash, err := tar2hashed(srcTarPath, destPath, cimOrext4) + if err != nil { + log.Infof("tar2hash failed: %s", err.Error()) + } else { + log.Infof("%s", hash) + } + log.Trace("tar2hashedCommand done") + stopProfiler(ctx) + return nil }, }