diff --git a/cmd/gcs-sidecar/main.go b/cmd/gcs-sidecar/main.go new file mode 100644 index 0000000000..4cd0d70b34 --- /dev/null +++ b/cmd/gcs-sidecar/main.go @@ -0,0 +1,244 @@ +//go:build windows +// +build windows + +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "time" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + shimlog "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/debug" + + sidecar "github.com/Microsoft/hcsshim/internal/gcs-sidecar" +) + +var ( + defaultLogFile = "C:\\gcs-sidecar-logs.log" + defaultLogLevel = "trace" +) + +type handler struct { + fromsvc chan error +} + +// Accepts new connection and closes listener. +func acceptAndClose(ctx context.Context, l net.Listener) (net.Conn, error) { + var conn net.Conn + ch := make(chan error) + go func() { + var err error + conn, err = l.Accept() + ch <- err + }() + select { + case err := <-ch: + l.Close() + return conn, err + case <-ctx.Done(): + } + l.Close() + err := <-ch + if err == nil { + return conn, err + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, err +} + +func (h *handler) Execute(args []string, r <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { + const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE) + + status <- svc.Status{State: svc.StartPending, Accepts: 0} + // unblock runService() + h.fromsvc <- nil + + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + +loop: + for c := range r { + switch c.Cmd { + case svc.Interrogate: + status <- c.CurrentStatus + case svc.Stop, svc.Shutdown: + logrus.Println("Shutting service...!") + break loop + case svc.Pause: + status <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted} + case svc.Continue: + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + default: + logrus.Printf("Unexpected service control request #%d", c) + } + } + + status <- svc.Status{State: svc.StopPending} + return false, 1 +} + +func runService(name string, isDebug bool) error { + h := &handler{ + fromsvc: make(chan error), + } + + var err error + go func() { + if isDebug { + err = debug.Run(name, h) + if err != nil { + logrus.Errorf("Error running service in debug mode.Err: %v", err) + } + } else { + err = svc.Run(name, h) + if err != nil { + logrus.Errorf("Error running service in Service Control mode.Err %v", err) + } + } + h.fromsvc <- err + }() + + // Wait for the first signal from the service handler. + logrus.Tracef("waiting for first signal from service handler\n") + return <-h.fromsvc +} + +func main() { + logLevel := flag.String("loglevel", + defaultLogLevel, + "Logging Level: trace, debug, info, warning, error, fatal, panic.") + logFile := flag.String("logfile", + defaultLogFile, + "Logging Target. Default is at C:\\gcs-sidecar-logs.log inside UVM") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "\nUsage of %s:\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "Examples:\n") + fmt.Fprintf(os.Stderr, " %s -loglevel=trace -logfile=C:\\sidecarLogs.log \n", os.Args[0]) + } + + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + logFileHandle, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0666) + if err != nil { + fmt.Printf("error opening file: %v", err) + } + defer logFileHandle.Close() + + logrus.AddHook(shimlog.NewHook()) + + level, err := logrus.ParseLevel(*logLevel) + if err != nil { + logrus.Fatal(err) + } + logrus.SetLevel(level) + logrus.SetOutput(logFileHandle) + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.RegisterExporter(&oc.LogrusExporter{}) + + if err := windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(logFileHandle.Fd())); err != nil { + logrus.WithError(err).Error("error redirecting handle") + return + } + os.Stderr = logFileHandle + + chsrv := make(chan error) + go func() { + defer close(chsrv) + + if err := runService("gcs-sidecar", false); err != nil { + logrus.Errorf("error starting gcs-sidecar service: %v", err) + } + + chsrv <- err + }() + + select { + case <-ctx.Done(): + logrus.Error("context deadline exceeded") + return + case r := <-chsrv: + if r != nil { + logrus.Error(r) + return + } + } + + // 1. Start external server to connect with inbox GCS + listener, err := winio.ListenHvsock(&winio.HvsockAddr{ + VMID: prot.HvGUIDLoopback, + ServiceID: prot.WindowsGcsHvsockServiceID, + }) + if err != nil { + logrus.WithError(err).Error("error starting listener for sidecar <-> inbox gcs communication") + return + } + + var gcsListener net.Listener = listener + gcsCon, err := acceptAndClose(ctx, gcsListener) + if err != nil { + logrus.WithError(err).Error("error accepting inbox GCS connection") + return + } + + // 2. Setup connection with external gcs connection started from hcsshim + hvsockAddr := &winio.HvsockAddr{ + VMID: prot.HvGUIDParent, + ServiceID: prot.WindowsSidecarGcsHvsockServiceID, + } + + logrus.WithFields(logrus.Fields{ + "hvsockAddr": hvsockAddr, + }).Tracef("Dialing to hcsshim external bridge at address %v", hvsockAddr) + shimCon, err := winio.Dial(ctx, hvsockAddr) + if err != nil { + logrus.WithError(err).Error("error dialing hcsshim external bridge") + return + } + + // gcs-sidecar can be used for non-confidentail hyperv wcow + // as well. So we do not always want to check for initialPolicyStance + var initialEnforcer securitypolicy.SecurityPolicyEnforcer + // TODO (kiashok/Mahati): The initialPolicyStance is set to allow + // only for dev. This will eventually be set to allow/deny depending on + // on whether SNP is supported or not. + initialPolicyStance := "allow" + switch initialPolicyStance { + case "allow": + initialEnforcer = &securitypolicy.OpenDoorSecurityPolicyEnforcer{} + logrus.Tracef("initial-policy-stance: allow") + case "deny": + initialEnforcer = &securitypolicy.ClosedDoorSecurityPolicyEnforcer{} + logrus.Tracef("initial-policy-stance: deny") + default: + logrus.Error("unknown initial-policy-stance") + } + + // 3. Create bridge and initializa + brdg := sidecar.NewBridge(shimCon, gcsCon, initialEnforcer) + brdg.AssignHandlers() + + // 3. Listen and serve for hcsshim requests. + err = brdg.ListenAndServeShimRequests() + if err != nil { + logrus.WithError(err).Error("failed to serve request") + } +} diff --git a/internal/bridgeutils/commonutils/utilities.go b/internal/bridgeutils/commonutils/utilities.go new file mode 100644 index 0000000000..4409825ad1 --- /dev/null +++ b/internal/bridgeutils/commonutils/utilities.go @@ -0,0 +1,83 @@ +package commonutils + +import ( + "encoding/json" + "fmt" + "io" + "math" + "strconv" + + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" + "github.com/sirupsen/logrus" +) + +type ErrorRecord struct { + Result int32 // HResult + Message string + StackTrace string `json:",omitempty"` + ModuleName string + FileName string + Line uint32 + FunctionName string `json:",omitempty"` +} + +// UnmarshalJSONWithHresult unmarshals the given data into the given interface, and +// wraps any error returned in an HRESULT error. +func UnmarshalJSONWithHresult(data []byte, v interface{}) error { + if err := json.Unmarshal(data, v); err != nil { + return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) + } + return nil +} + +// DecodeJSONWithHresult decodes the JSON from the given reader into the given +// interface, and wraps any error returned in an HRESULT error. +func DecodeJSONWithHresult(r io.Reader, v interface{}) error { + if err := json.NewDecoder(r).Decode(v); err != nil { + return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) + } + return nil +} + +func SetErrorForResponseBaseUtil(errForResponse error, moduleName string) (hresult gcserr.Hresult, errorMessage string, newRecord ErrorRecord) { + errorMessage = errForResponse.Error() + stackString := "" + fileName := "" + // We use -1 as a sentinel if no line number found (or it cannot be parsed), + // but that will ultimately end up as [math.MaxUint32], so set it to that explicitly. + // (Still keep using -1 for backwards compatibility ...) + lineNumber := uint32(math.MaxUint32) + functionName := "" + if stack := gcserr.BaseStackTrace(errForResponse); stack != nil { + bottomFrame := stack[0] + stackString = fmt.Sprintf("%+v", stack) + fileName = fmt.Sprintf("%s", bottomFrame) + lineNumberStr := fmt.Sprintf("%d", bottomFrame) + if n, err := strconv.ParseUint(lineNumberStr, 10, 32); err == nil { + lineNumber = uint32(n) + } else { + logrus.WithFields(logrus.Fields{ + "line-number": lineNumberStr, + logrus.ErrorKey: err, + }).Error("opengcs::bridge::setErrorForResponseBase - failed to parse line number, using -1 instead") + } + functionName = fmt.Sprintf("%n", bottomFrame) + } + hresult, err := gcserr.GetHresult(errForResponse) + if err != nil { + // Default to using the generic failure HRESULT. + hresult = gcserr.HrFail + } + + newRecord = ErrorRecord{ + Result: int32(hresult), + Message: errorMessage, + StackTrace: stackString, + ModuleName: moduleName, + FileName: fileName, + Line: lineNumber, + FunctionName: functionName, + } + + return hresult, errorMessage, newRecord +} diff --git a/internal/guest/gcserr/errors.go b/internal/bridgeutils/gcserr/errors.go similarity index 100% rename from internal/guest/gcserr/errors.go rename to internal/bridgeutils/gcserr/errors.go diff --git a/internal/fsformatter/formatter_driver.go b/internal/fsformatter/formatter_driver.go new file mode 100644 index 0000000000..14f743cf0b --- /dev/null +++ b/internal/fsformatter/formatter_driver.go @@ -0,0 +1,283 @@ +//go:build windows +// +build windows + +package fsformatter + +import ( + "context" + "encoding/binary" + "syscall" + "unicode/utf16" + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// This file contains all the supporting structures needed to make +// an ioctl call to RefsFormatter. +const ( + ioctlKernelFormatVolumeFormat = 0x40001000 + // This is used to construct the disk path that refsFormatter + // understands. `harddisk%d` here refers to the disk number + // associated with the corresponding lun of the attached + // scsi device. + VirtualDevObjectPathFormat = "\\device\\harddisk%d\\partition0" + checksumTypeSha256 = uint16(4) + refsChecksumType = checksumTypeSha256 + maxSizeOfKernelFormatVolumeFormatRefsParameters = 16 * 8 // 128 bytes + sizeOfWchar = int(unsafe.Sizeof(uint16(0))) + kernelFormatVolumeMaxVolumeLabelLength = uint32(33 * sizeOfWchar) + kernelFormatVolumeWin32DriverPath = "\\\\?\\KernelFSFormatter" + // Allocate large enough buffer for output from fsFormatter + maxSizeOfOutputBuffer = uint32(512) + + // KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS member offsets + clusterSizeOffset = 0 + checksumTypeOffset = 4 + useDataIntegrityOffset = 6 + majorVersionOffset = 8 + minorVersionOffset = 10 +) + +type kernelFormatVolumeFilesystemTypes uint32 + +const ( + kernelFormatVolumeFilesystemTypeInvalid = kernelFormatVolumeFilesystemTypes(iota) + kernelFormatVolumeFilesystemTypeRefs = kernelFormatVolumeFilesystemTypes(1) + kernelFormatVolumeFilesystemTypeMax = kernelFormatVolumeFilesystemTypes(2) +) + +// We only want to allow refs formatting +func (filesystemType kernelFormatVolumeFilesystemTypes) String() string { + switch filesystemType { + case kernelFormatVolumeFilesystemTypeRefs: + return "KERNEL_FORMAT_VOLUME_FILESYSTEM_TYPE_REFS" + default: + return "Unknown" + } +} + +type kernelFormatVolumeFormatInputBufferFlags uint32 + +const kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) + +func (flag kernelFormatVolumeFormatInputBufferFlags) String() string { + switch flag { + case kernelFormatVolumeFormatInputBufferFlagNone: + return "kernelFormatVolumeFormatInputBufferFlagNone" + default: + return "Unknown" + } +} + +type KernelFormatVolumeFormatRefsParameters struct { + ClusterSize uint32 + MetadataChecksumType uint16 + UseDataIntegrity bool + MajorVersion uint16 + MinorVersion uint16 +} + +type KernelFormatVolumeFormatFsParameters struct { + FileSystemType kernelFormatVolumeFilesystemTypes + // Represents a WCHAR character array + VolumeLabel [kernelFormatVolumeMaxVolumeLabelLength / uint32(sizeOfWchar)]uint16 + // Length of volume label in bytes + VolumeLabelLength uint16 + // RefsFormatterParams represents the following union + /* + union { + + KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS RefsParameters; + + // + // This structure can't grow in size nor change in alignment. 16 ULONGLONGs + // should be more than enough for supporting other filesystems down the + // line. This also serves to enforce 8 byte alignment. + // + Reserved [16]uint64 + }; + */ + RefsFormatterParams [128]byte +} + +type KernelFormatVolumeFormatInputBuffer struct { + Size uint64 + FsParameters KernelFormatVolumeFormatFsParameters + Flags kernelFormatVolumeFormatInputBufferFlags + Reserved [4]uint32 + // Size of DiskPathBuffer in bytes + DiskPathLength uint16 + // DiskPathBuffer holds the disk path. It represents a + // variable size WCHAR character array + DiskPathBuffer []uint16 +} + +type kernelFormatVolumeFormatOutputBufferFlags uint32 + +const kernelFormatVolumeFormatOutputBufferFlagsNone = kernelFormatVolumeFormatOutputBufferFlags(0x00000000) + +func (flag kernelFormatVolumeFormatOutputBufferFlags) String() string { + switch flag { + case kernelFormatVolumeFormatOutputBufferFlagsNone: + return "kernelFormatVolumeFormatOutputBufferFlagsNone" + default: + return "Unknown" + } +} + +type KernelFormarVolumeFormatOutputBuffer struct { + Size uint32 + Flags kernelFormatVolumeFormatOutputBufferFlags + Reserved [4]uint32 + // VolumePathLength holds size of VolumePathBuffer + // in bytes + VolumePathLength uint16 + // VolumePathBuffer holds the mounted volume path + // as returned from refsFormatter. It represents + // a variable size WCHAR character array + VolumePathBuffer []uint16 +} + +// GetVolumePathBufferOffset gets offset to KernelFormarVolumeFormatOutputBuffer{}.VolumePathBuffer +func GetVolumePathBufferOffset() uint32 { + volPathBufferOffset := uint32(unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Size) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Flags) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Reserved) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.VolumePathLength)) + + return volPathBufferOffset +} + +// getInputBufferSize gets the total size needed for input buffer +func getInputBufferSize(wcharDiskPathLength uint16) uint32 { + bufferSize := uint32(unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Size)+ + /* This is specifically for the union in KernelFormatVolumeFormatFsParameters */ + unsafe.Offsetof(KernelFormatVolumeFormatFsParameters{}.RefsFormatterParams)+ + maxSizeOfKernelFormatVolumeFormatRefsParameters+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Flags)+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Reserved)+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.DiskPathLength)) + + uint32(wcharDiskPathLength) + + return bufferSize +} + +// getInputBufferDiskPathBufferOffset gets offset to KernelFormatVolumeFormatInputBuffer{}.DiskPathBuffer +func getInputBufferDiskPathBufferOffset() uint32 { + diskPathBufferOffset := uint32(unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Size) + + unsafe.Offsetof(KernelFormatVolumeFormatFsParameters{}.RefsFormatterParams) + + maxSizeOfKernelFormatVolumeFormatRefsParameters + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Flags) + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Reserved) + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.DiskPathLength)) + + return diskPathBufferOffset +} + +// KmFmtCreateFormatOutputBuffer formats an output buffer as expected +// by the fsFormatter driver +func KmFmtCreateFormatOutputBuffer() *KernelFormarVolumeFormatOutputBuffer { + buf := make([]uint16, maxSizeOfOutputBuffer) + outputBuffer := (*KernelFormarVolumeFormatOutputBuffer)(unsafe.Pointer(&buf[0])) + outputBuffer.Size = uint32(maxSizeOfOutputBuffer) + + return outputBuffer +} + +func toUTF16(s string) []uint16 { + return utf16.Encode([]rune(s)) +} + +// KmFmtCreateFormatInputBuffer formats an input buffer as expected +// by the refsFormatter driver. +// diskPath represents disk path in VirtualDevObjectPathFormat. +func KmFmtCreateFormatInputBuffer(diskPath string) *KernelFormatVolumeFormatInputBuffer { + refsParametersBuf := make([]byte, unsafe.Sizeof(KernelFormatVolumeFormatRefsParameters{})) + refsParameters := (*KernelFormatVolumeFormatRefsParameters)(unsafe.Pointer(&refsParametersBuf[0])) + + utf16DiskPath := toUTF16(diskPath) + wcharDiskPathLength := uint16(len(utf16DiskPath) * sizeOfWchar) + + refsParameters.ClusterSize = 0x1000 + refsParameters.MetadataChecksumType = refsChecksumType + refsParameters.UseDataIntegrity = true + refsParameters.MajorVersion = uint16(3) + refsParameters.MinorVersion = uint16(14) + + bufferSize := getInputBufferSize(wcharDiskPathLength) + buf := make([]byte, bufferSize) + inputBuffer := (*KernelFormatVolumeFormatInputBuffer)(unsafe.Pointer(&buf[0])) + + inputBuffer.Size = uint64(bufferSize) + inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagNone + + inputBuffer.FsParameters.FileSystemType = kernelFormatVolumeFilesystemTypeRefs + inputBuffer.FsParameters.VolumeLabelLength = 0 + inputBuffer.FsParameters.VolumeLabel = [33]uint16{} + + // Write KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS + binary.LittleEndian.PutUint32(inputBuffer.FsParameters.RefsFormatterParams[clusterSizeOffset:], refsParameters.ClusterSize) + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[checksumTypeOffset:], refsParameters.MetadataChecksumType) + if refsParameters.UseDataIntegrity { + inputBuffer.FsParameters.RefsFormatterParams[useDataIntegrityOffset] = 1 + } else { + inputBuffer.FsParameters.RefsFormatterParams[useDataIntegrityOffset] = 0 + } + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[majorVersionOffset:], refsParameters.MajorVersion) + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[minorVersionOffset:], refsParameters.MinorVersion) + + // Finally write the diskPathLength and diskPathBuffer with the input disk path + inputBuffer.DiskPathLength = wcharDiskPathLength + // DiskBuffer writing + ptr := unsafe.Add(unsafe.Pointer(inputBuffer), getInputBufferDiskPathBufferOffset()) + // Convert the string to UTF-16 slice + utf16Array := toUTF16(diskPath) + diskPathBuf := unsafe.Slice((*uint16)(ptr), len(utf16Array)) + copy(diskPathBuf, utf16Array) + + return inputBuffer +} + +// InvokeFsFormatter makes an ioctl call to the fsFormatter driver and returns +// a path to the mountedVolume +func InvokeFsFormatter(ctx context.Context, diskPath string) (string, error) { + // Prepare input and output buffers as expected by refsFormatter + inputBuffer := KmFmtCreateFormatInputBuffer(diskPath) + outputBuffer := KmFmtCreateFormatOutputBuffer() + + utf16DriverPath, _ := windows.UTF16PtrFromString(kernelFormatVolumeWin32DriverPath) + deviceHandle, err := windows.CreateFile(utf16DriverPath, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + 0, + nil, + windows.OPEN_EXISTING, + 0, + 0) + if err != nil { + return "", errors.Wrap(err, "failed to get handle to refsFormatter driver") + } + defer windows.Close(deviceHandle) + + // Ioctl to fsFormatter driver + var bytesReturned uint32 + if err := windows.DeviceIoControl( + deviceHandle, + ioctlKernelFormatVolumeFormat, + (*byte)(unsafe.Pointer(inputBuffer)), + uint32(inputBuffer.Size), + (*byte)(unsafe.Pointer(outputBuffer)), + outputBuffer.Size, + &bytesReturned, + nil, + ); err != nil { + return "", errors.Wrap(err, "ioctl to refsFormatter driver failed") + } + + // Read the returned volume path from the corresponding offset in outputBuffer + ptr := unsafe.Pointer(uintptr(unsafe.Pointer(outputBuffer)) + uintptr(GetVolumePathBufferOffset())) + utf16Data := unsafe.Slice((*uint16)(ptr), outputBuffer.VolumePathLength/2) + mountedVolumePath := syscall.UTF16ToString(utf16Data) + return mountedVolumePath, err +} diff --git a/internal/gcs-sidecar/bridge.go b/internal/gcs-sidecar/bridge.go new file mode 100644 index 0000000000..545d2b7eb3 --- /dev/null +++ b/internal/gcs-sidecar/bridge.go @@ -0,0 +1,541 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "go.opencensus.io/trace/tracestate" + "golang.org/x/sys/windows" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" +) + +type Bridge struct { + mu sync.Mutex + hostState *Host + // List of handlers for handling different rpc message requests. + rpcHandlerList map[prot.RPCProc]HandlerFunc + + // hcsshim and inbox GCS connections respectively. + shimConn io.ReadWriteCloser + inboxGCSConn io.ReadWriteCloser + + // Response channels to forward incoming requests to inbox GCS + // and send responses back to hcsshim respectively. + sendToGCSCh chan request + sendToShimCh chan bridgeResponse +} + +// SequenceID is used to correlate requests and responses. +type sequenceID uint64 + +// messageHeader is the common header present in all communications messages. +type messageHeader struct { + Type prot.MsgType + Size uint32 + ID sequenceID +} + +type bridgeResponse struct { + ctx context.Context + header messageHeader + response []byte +} + +type request struct { + // Context created once received from the bridge. + ctx context.Context + // header is the wire format message header that preceded the message for + // this request. + header messageHeader + // activityID is the id of the specific activity for this request. + activityID guid.GUID + // message is the portion of the request that follows the `Header`. + message []byte +} + +func NewBridge(shimConn io.ReadWriteCloser, inboxGCSConn io.ReadWriteCloser, initialEnforcer securitypolicy.SecurityPolicyEnforcer) *Bridge { + hostState := NewHost(initialEnforcer) + return &Bridge{ + rpcHandlerList: make(map[prot.RPCProc]HandlerFunc), + hostState: hostState, + shimConn: shimConn, + inboxGCSConn: inboxGCSConn, + sendToGCSCh: make(chan request), + sendToShimCh: make(chan bridgeResponse), + } +} + +func NewPolicyEnforcer(initialEnforcer securitypolicy.SecurityPolicyEnforcer) *SecurityPolicyEnforcer { + return &SecurityPolicyEnforcer{ + securityPolicyEnforcerSet: false, + securityPolicyEnforcer: initialEnforcer, + } +} + +// UnknownMessage represents the default handler logic for an unmatched request +// type sent from the bridge. +func UnknownMessage(r *request) error { + log.G(r.ctx).Debugf("bridge: function not supported, header type %v", prot.MsgType(r.header.Type).String()) + return gcserr.WrapHresult(errors.Errorf("bridge: function not supported, header type: %v", r.header.Type), gcserr.HrNotImpl) +} + +// HandlerFunc is an adapter to use functions as handlers. +type HandlerFunc func(*request) error + +func (b *Bridge) getRequestHandler(r *request) (HandlerFunc, error) { + b.mu.Lock() + defer b.mu.Unlock() + + var handler HandlerFunc + var ok bool + messageType := r.header.Type + rpcProcID := prot.RPCProc(messageType &^ prot.MsgTypeMask) + if handler, ok = b.rpcHandlerList[rpcProcID]; !ok { + return nil, UnknownMessage(r) + } + return handler, nil +} + +// ServeMsg serves request by calling appropriate handler functions. +func (b *Bridge) ServeMsg(r *request) error { + if r == nil { + panic("bridge: nil request to handler") + } + + var handler HandlerFunc + var err error + if handler, err = b.getRequestHandler(r); err != nil { + return UnknownMessage(r) + } + return handler(r) +} + +// Handle registers the handler for the given message id and protocol version. +func (b *Bridge) Handle(rpcProcID prot.RPCProc, handlerFunc HandlerFunc) { + b.mu.Lock() + defer b.mu.Unlock() + + if handlerFunc == nil { + panic("empty function handler") + } + + if _, ok := b.rpcHandlerList[rpcProcID]; ok { + logrus.WithFields(logrus.Fields{ + "message-type": rpcProcID.String(), + }).Warn("overwriting bridge handler") + } + + b.rpcHandlerList[rpcProcID] = handlerFunc +} + +func (b *Bridge) HandleFunc(rpcProcID prot.RPCProc, handler func(*request) error) { + if handler == nil { + panic("bridge: nil handler func") + } + + b.Handle(rpcProcID, HandlerFunc(handler)) +} + +// AssignHandlers creates and assigns appropriate event handlers +// for the different bridge message types. +func (b *Bridge) AssignHandlers() { + b.HandleFunc(prot.RPCCreate, b.createContainer) + b.HandleFunc(prot.RPCStart, b.startContainer) + b.HandleFunc(prot.RPCShutdownGraceful, b.shutdownGraceful) + b.HandleFunc(prot.RPCShutdownForced, b.shutdownForced) + b.HandleFunc(prot.RPCExecuteProcess, b.executeProcess) + b.HandleFunc(prot.RPCWaitForProcess, b.waitForProcess) + b.HandleFunc(prot.RPCSignalProcess, b.signalProcess) + b.HandleFunc(prot.RPCResizeConsole, b.resizeConsole) + b.HandleFunc(prot.RPCGetProperties, b.getProperties) + b.HandleFunc(prot.RPCModifySettings, b.modifySettings) + b.HandleFunc(prot.RPCNegotiateProtocol, b.negotiateProtocol) + b.HandleFunc(prot.RPCDumpStacks, b.dumpStacks) + b.HandleFunc(prot.RPCDeleteContainerState, b.deleteContainerState) + b.HandleFunc(prot.RPCUpdateContainer, b.updateContainer) + b.HandleFunc(prot.RPCLifecycleNotification, b.lifecycleNotification) +} + +// readMessage reads the message from io.Reader +func readMessage(r io.Reader) (messageHeader, []byte, error) { + var h [prot.HdrSize]byte + _, err := io.ReadFull(r, h[:]) + if err != nil { + return messageHeader{}, nil, err + } + var header messageHeader + buf := bytes.NewReader(h[:]) + err = binary.Read(buf, binary.LittleEndian, &header) + if err != nil { + logrus.WithError(err).Errorf("error reading message header") + return messageHeader{}, nil, err + } + + n := header.Size + if n < prot.HdrSize || n > prot.MaxMsgSize { + logrus.Errorf("invalid message size %d", n) + return messageHeader{}, nil, fmt.Errorf("invalid message size %d: %w", n, err) + } + + n -= prot.HdrSize + msg := make([]byte, n) + _, err = io.ReadFull(r, msg) + if err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return messageHeader{}, nil, err + } + + return header, msg, nil +} + +func isLocalDisconnectError(err error) bool { + return errors.Is(err, windows.WSAECONNABORTED) +} + +// Sends request to the inbox GCS channel +func (b *Bridge) forwardRequestToGcs(req *request) { + b.sendToGCSCh <- *req +} + +func getContextAndSpan(baseSpanCtx *prot.Ocspancontext) (context.Context, *trace.Span) { + var ctx context.Context + var span *trace.Span + if baseSpanCtx != nil { + sc := trace.SpanContext{} + if bytes, err := hex.DecodeString(baseSpanCtx.TraceID); err == nil { + copy(sc.TraceID[:], bytes) + } + if bytes, err := hex.DecodeString(baseSpanCtx.SpanID); err == nil { + copy(sc.SpanID[:], bytes) + } + sc.TraceOptions = trace.TraceOptions(baseSpanCtx.TraceOptions) + if baseSpanCtx.Tracestate != "" { + if bytes, err := base64.StdEncoding.DecodeString(baseSpanCtx.Tracestate); err == nil { + var entries []tracestate.Entry + if err := json.Unmarshal(bytes, &entries); err == nil { + if ts, err := tracestate.New(nil, entries...); err == nil { + sc.Tracestate = ts + } + } + } + } + ctx, span = oc.StartSpanWithRemoteParent( + context.Background(), + "sidecar::request", + sc, + oc.WithServerSpanKind, + ) + } else { + ctx, span = oc.StartSpan( + context.Background(), + "sidecar::request", + oc.WithServerSpanKind, + ) + } + + return ctx, span +} + +func sendWithContextCancel[T any](ctx context.Context, sendCh chan<- T, msg T) error { + select { + case sendCh <- msg: + // Sent successfully + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Sends response to the hcsshim channel +func (b *Bridge) sendResponseToShim(ctx context.Context, rpcProcType prot.RPCProc, id sequenceID, response interface{}) error { + respType := prot.MsgTypeResponse | prot.MsgType(rpcProcType) + msgb, err := json.Marshal(response) + if err != nil { + return err + } + msgHeader := messageHeader{ + Type: respType, + Size: uint32(len(msgb) + prot.HdrSize), + ID: id, + } + + resp := bridgeResponse{ + ctx: ctx, + header: msgHeader, + response: msgb, + } + + return sendWithContextCancel(ctx, b.sendToShimCh, resp) +} + +// ListenAndServeShimRequests listens to messages on the hcsshim +// and inbox GCS connections and schedules them for processing. +// After processing, messages are forwarded to inbox GCS on success +// and responses from inbox GCS or error messages are sent back +// to hcsshim via bridge connection. +func (b *Bridge) ListenAndServeShimRequests() error { + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + shimRequestChan := make(chan request) + // goroutines share the error channel to send error responses + // back to hcsshim. Therefore use buffered error channel. + sidecarErrChan := make(chan error, 5) + + // Goroutine 1: Listen to requests from hcsshim + wg.Add(1) + go func() { + defer wg.Done() + defer close(shimRequestChan) + + var recverr error + br := bufio.NewReader(b.shimConn) + for { + select { + case <-ctx.Done(): + return + default: + header, msg, err := readMessage(br) + if err != nil { + if errors.Is(err, io.EOF) || isLocalDisconnectError(err) { + // hcsshim handles these errors explicitly. Therefore, + // returning the exact same error. + recverr = err + } else { + recverr = fmt.Errorf("bridge read from shim connection failed: %w", err) + } + logrus.Error(recverr) + _ = sendWithContextCancel(ctx, sidecarErrChan, recverr) + return + } + + var msgBase prot.RequestBase + _ = json.Unmarshal(msg, &msgBase) + reqCtx, span := getContextAndSpan(msgBase.OpenCensusSpanContext) + span.AddAttributes( + trace.Int64Attribute("message-id", int64(header.ID)), + trace.StringAttribute("message-type", header.Type.String()), + trace.StringAttribute("activityID", msgBase.ActivityID.String()), + trace.StringAttribute("containerID", msgBase.ContainerID)) + + req := request{ + ctx: reqCtx, + activityID: msgBase.ActivityID, + header: header, + message: msg, + } + if err := sendWithContextCancel(ctx, shimRequestChan, req); err != nil { + log.G(ctx).WithError(err).Error("failed to send request to shimRequestChan") + return + } + } + } + }() + // Goroutine 2: Process each bridge request received from shim asynchronously. + wg.Add(1) + go func() { + defer wg.Done() + + for { + // Requests are served sequentially to avoid + // racing/reordering of incoming message order. + // This becomes important for confidential cases + // where the shim could be compromised and replay + // messages out of order. + select { + case <-ctx.Done(): + return + case req, ok := <-shimRequestChan: + if !ok { + return + } + if err := b.ServeMsg(&req); err != nil { + log.G(req.ctx).WithError(err).Errorf("failed to serve request: %v", req.header.Type.String()) + // In case of error, create appropriate response message to + // be sent to hcsshim. + resp := &prot.ResponseBase{ + Result: int32(windows.ERROR_GEN_FAILURE), + ErrorMessage: err.Error(), + ActivityID: req.activityID, + } + setErrorForResponseBase(resp, err, "gcs-sidecar" /* moduleName */) + // Prepares and sends message to hcsshim via b.sendToShimCh + responseErr := b.sendResponseToShim(req.ctx, prot.RPCProc(prot.MsgTypeResponse), req.header.ID, resp) + if responseErr != nil { + log.G(req.ctx).WithError(err).Errorf("failed to send response to shim") + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + } + }() + // Goroutine 3: Forrward requests to inbox GCS + wg.Add(1) + go func() { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + case req, ok := <-b.sendToGCSCh: + if !ok { + return + } + + // Forward message to gcs + log.G(req.ctx).Tracef("bridge send to gcs, req %v, %v", req.header.Type.String(), string(req.message)) + buffer, err := b.prepareResponseMessage(req.header, req.message) + if err != nil { + responseErr := fmt.Errorf("error preparing response: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + + _, err = buffer.WriteTo(b.inboxGCSConn) + if err != nil { + responseErr := fmt.Errorf("err forwarding shim req to inbox GCS: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + }() + // Goroutine 4: Receive response from gcs and forward to hcsshim + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + default: + header, message, err := readMessage(b.inboxGCSConn) + if err != nil { + var recverr error + if errors.Is(err, io.EOF) || isLocalDisconnectError(err) { + // hcsshim handles these errors explicitly. Therefore, + // returning the exact same error. + recverr = err + } else { + recverr = fmt.Errorf("bridge read from gcs failed: %w", err) + } + logrus.Error(recverr) + _ = sendWithContextCancel(ctx, sidecarErrChan, recverr) + return + } + + // Forward to shim + resp := bridgeResponse{ + ctx: context.Background(), + header: header, + response: message, + } + if err := sendWithContextCancel(ctx, b.sendToShimCh, resp); err != nil { + log.G(ctx).WithError(err).Error("failed to send request to b.sendToShimCh") + return + } + } + } + }() + // Goroutine 5: Send response to hcsshim + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-b.sendToShimCh: + if !ok { + return + } + + // Send response to shim + logrus.Tracef("Send response to shim. Header:{ID: %v, Type: %v, Size: %v} msg: %v", resp.header.ID, + resp.header.Type, resp.header.Size, string(resp.response)) + buffer, err := b.prepareResponseMessage(resp.header, resp.response) + if err != nil { + responseErr := fmt.Errorf("error preparing response: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + _, sendErr := buffer.WriteTo(b.shimConn) + if sendErr != nil { + responseErr := fmt.Errorf("err sending response to shim: %w", sendErr) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + }() + + var finalErr error + select { + case finalErr = <-sidecarErrChan: + cancel() + case <-ctx.Done(): + finalErr = ctx.Err() + cancel() + } + + // Close read b.ShimConn and b.inboxGCSConn + _ = b.shimConn.Close() + _ = b.inboxGCSConn.Close() + + // Close sidecarErrChan after all the goroutines have finished + // sending to it. + wg.Wait() + close(sidecarErrChan) + + return finalErr +} + +// Prepare response message +func (b *Bridge) prepareResponseMessage(header messageHeader, message []byte) (bytes.Buffer, error) { + // Create a buffer to hold the serialized header data + var headerBuf bytes.Buffer + err := binary.Write(&headerBuf, binary.LittleEndian, header) + if err != nil { + return headerBuf, err + } + + // Write message header followed by actual payload. + var buf bytes.Buffer + buf.Write(headerBuf.Bytes()) + buf.Write(message[:]) + return buf, nil +} + +// setErrorForResponseBase modifies the passed-in ResponseBase to +// contain information pertaining to the given error. +func setErrorForResponseBase(response *prot.ResponseBase, errForResponse error, moduleName string) { + hresult, errorMessage, newRecord := commonutils.SetErrorForResponseBaseUtil(errForResponse, moduleName) + response.Result = int32(hresult) + response.ErrorMessage = errorMessage + response.ErrorRecords = append(response.ErrorRecords, newRecord) +} diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go new file mode 100644 index 0000000000..e321d4e092 --- /dev/null +++ b/internal/gcs-sidecar/handlers.go @@ -0,0 +1,491 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/fsformatter" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/internal/windevice" + "github.com/Microsoft/hcsshim/pkg/cimfs" +) + +const ( + sandboxStateDirName = "WcSandboxState" + hivesDirName = "Hives" + devPathFormat = "\\\\.\\PHYSICALDRIVE%d" +) + +// - Handler functions handle the incoming message requests. It +// also enforces security policy for confidential cwcow containers. +// - These handler functions may do some additional processing before +// forwarding requests to inbox GCS or send responses back to hcsshim. +// - In case of any error encountered during processing, appropriate error +// messages are returned and responses are sent back to hcsshim from ListenAndServer(). +// TODO (kiashok): Verbose logging is for WIP and will be removed eventually. +func (b *Bridge) createContainer(req *request) (err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::createContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerCreate + var containerConfig json.RawMessage + r.ContainerConfig.Value = &containerConfig + if err = commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal createContainer: %w", err) + } + + // containerConfig can be of type uvnConfig or hcsschema.HostedSystem + var ( + uvmConfig prot.UvmConfig + hostedSystemConfig hcsschema.HostedSystem + ) + if err = commonutils.UnmarshalJSONWithHresult(containerConfig, &uvmConfig); err == nil && + uvmConfig.SystemType != "" { + systemType := uvmConfig.SystemType + timeZoneInformation := uvmConfig.TimeZoneInformation + log.G(ctx).Tracef("createContainer: uvmConfig: {systemType: %v, timeZoneInformation: %v}}", systemType, timeZoneInformation) + } else if err = commonutils.UnmarshalJSONWithHresult(containerConfig, &hostedSystemConfig); err == nil && + hostedSystemConfig.SchemaVersion != nil && hostedSystemConfig.Container != nil { + schemaVersion := hostedSystemConfig.SchemaVersion + container := hostedSystemConfig.Container + log.G(ctx).Tracef("createContainer: HostedSystemConfig: {schemaVersion: %v, container: %v}}", schemaVersion, container) + } else { + return fmt.Errorf("invalid request to createContainer") + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) startContainer(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::startContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal startContainer: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) shutdownGraceful(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::shutdownGraceful") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal shutdownGraceful: %w", err) + } + + // TODO (kiashok/Mahati): Since gcs-sidecar can be used for all types of windows + // containers, it is important to check if we want to + // enforce policy or not. + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) shutdownForced(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::shutdownForced") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal shutdownForced: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) executeProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::executeProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerExecuteProcess + var processParamSettings json.RawMessage + r.Settings.ProcessParameters.Value = &processParamSettings + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal executeProcess: %w", err) + } + + var processParams hcsschema.ProcessParameters + if err := commonutils.UnmarshalJSONWithHresult(processParamSettings, &processParams); err != nil { + return fmt.Errorf("executeProcess: invalid params type for request: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) waitForProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::waitForProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerWaitForProcess + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal waitForProcess: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) signalProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::signalProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerSignalProcess + var rawOpts json.RawMessage + r.Options = &rawOpts + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal signalProcess: %w", err) + } + + var wcowOptions guestresource.SignalProcessOptionsWCOW + if rawOpts != nil { + if err := commonutils.UnmarshalJSONWithHresult(rawOpts, &wcowOptions); err != nil { + return fmt.Errorf("signalProcess: invalid Options type for request: %w", err) + } + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) resizeConsole(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::resizeConsole") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerResizeConsole + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal resizeConsole: %v", req) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) getProperties(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::getProperties") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var getPropReqV2 prot.ContainerGetPropertiesV2 + if err := commonutils.UnmarshalJSONWithHresult(req.message, &getPropReqV2); err != nil { + return fmt.Errorf("failed to unmarshal getProperties: %v: %w", string(req.message), err) + } + log.G(req.ctx).Tracef("getProperties query: %v", getPropReqV2.Query.PropertyTypes) + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) negotiateProtocol(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::negotiateProtocol") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.NegotiateProtocolRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal negotiateProtocol") + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) dumpStacks(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::dumpStacks") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.DumpStacksRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal dumpStacks: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) deleteContainerState(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::deleteContainerState") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.DeleteContainerStateRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal deleteContainerState: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) updateContainer(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::updateContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + // No callers in the code for rpcUpdateContainer + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) lifecycleNotification(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::lifecycleNotification") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + // No callers in the code for rpcLifecycleNotification + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) modifySettings(req *request) (err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::modifySettings") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + log.G(ctx).Tracef("modifySettings: MsgType: %v, Payload: %v", req.header.Type, string(req.message)) + modifyRequest, err := unmarshalContainerModifySettings(req) + if err != nil { + return err + } + modifyGuestSettingsRequest := modifyRequest.Request.(*guestrequest.ModificationRequest) + guestResourceType := modifyGuestSettingsRequest.ResourceType + guestRequestType := modifyGuestSettingsRequest.RequestType + log.G(ctx).Tracef("modifySettings: resourceType: %v, requestType: %v", guestResourceType, guestRequestType) + + if guestRequestType == "" { + guestRequestType = guestrequest.RequestTypeAdd + } + + switch guestRequestType { + case guestrequest.RequestTypeAdd: + case guestrequest.RequestTypeRemove: + case guestrequest.RequestTypePreAdd: + case guestrequest.RequestTypeUpdate: + default: + return fmt.Errorf("invald guestRequestType %v", guestRequestType) + } + + if guestResourceType != "" { + switch guestResourceType { + case guestresource.ResourceTypeCombinedLayers: + settings := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWCombinedLayers) + log.G(ctx).Tracef("WCOWCombinedLayers: {%v}", settings) + + case guestresource.ResourceTypeNetworkNamespace: + settings := modifyGuestSettingsRequest.Settings.(*hcn.HostComputeNamespace) + log.G(ctx).Tracef("HostComputeNamespaces { %v}", settings) + + case guestresource.ResourceTypeNetwork: + settings := modifyGuestSettingsRequest.Settings.(*guestrequest.NetworkModifyRequest) + log.G(ctx).Tracef("NetworkModifyRequest { %v}", settings) + + case guestresource.ResourceTypeMappedVirtualDisk: + wcowMappedVirtualDisk := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWMappedVirtualDisk) + log.G(ctx).Tracef("wcowMappedVirtualDisk { %v}", wcowMappedVirtualDisk) + + case guestresource.ResourceTypeHvSocket: + hvSocketAddress := modifyGuestSettingsRequest.Settings.(*hcsschema.HvSocketAddress) + log.G(ctx).Tracef("hvSocketAddress { %v }", hvSocketAddress) + + case guestresource.ResourceTypeMappedDirectory: + settings := modifyGuestSettingsRequest.Settings.(*hcsschema.MappedDirectory) + log.G(ctx).Tracef("hcsschema.MappedDirectory { %v }", settings) + + case guestresource.ResourceTypeSecurityPolicy: + securityPolicyRequest := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWConfidentialOptions) + log.G(ctx).Tracef("WCOWConfidentialOptions: { %v}", securityPolicyRequest) + _ = b.hostState.SetWCOWConfidentialUVMOptions(securityPolicyRequest) + + // Send response back to shim + resp := &prot.ResponseBase{ + Result: 0, // 0 means success + ActivityID: req.activityID, + } + err := b.sendResponseToShim(req.ctx, prot.RPCModifySettings, req.header.ID, resp) + if err != nil { + return fmt.Errorf("error sending response to hcsshim: %w", err) + } + return nil + + case guestresource.ResourceTypeWCOWBlockCims: + // This is request to mount the merged cim at given volumeGUID + wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWBlockCIMMounts) + log.G(ctx).Tracef("WCOWBlockCIMMounts { %v}", wcowBlockCimMounts) + + // The block device takes some time to show up. Wait for a few seconds. + time.Sleep(2 * time.Second) + + var layerCIMs []*cimfs.BlockCIM + for _, blockCimDevice := range wcowBlockCimMounts.BlockCIMs { + // Get the scsi device path for the blockCim lun + devNumber, err := windevice.GetDeviceNumberFromControllerLUN( + ctx, + 0, /* controller is always 0 for wcow */ + uint8(blockCimDevice.Lun)) + if err != nil { + return fmt.Errorf("err getting scsiDevPath: %w", err) + } + layerCim := cimfs.BlockCIM{ + Type: cimfs.BlockCIMTypeDevice, + BlockPath: fmt.Sprintf(devPathFormat, devNumber), + CimName: blockCimDevice.CimName, + } + layerCIMs = append(layerCIMs, &layerCim) + } + if len(layerCIMs) > 1 { + // Get the topmost merge CIM and invoke the MountMergedBlockCIMs + _, err := cimfs.MountMergedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID) + if err != nil { + return fmt.Errorf("error mounting multilayer block cims: %w", err) + } + } else { + _, err := cimfs.Mount(filepath.Join(layerCIMs[0].BlockPath, layerCIMs[0].CimName), wcowBlockCimMounts.VolumeGUID, wcowBlockCimMounts.MountFlags) + if err != nil { + return fmt.Errorf("error mounting merged block cims: %w", err) + } + } + + // Send response back to shim + resp := &prot.ResponseBase{ + Result: 0, // 0 means success + ActivityID: req.activityID, + } + err = b.sendResponseToShim(req.ctx, prot.RPCModifySettings, req.header.ID, resp) + if err != nil { + return fmt.Errorf("error sending response to hcsshim: %w", err) + } + return nil + + case guestresource.ResourceTypeCWCOWCombinedLayers: + settings := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWCombinedLayers) + containerID := settings.ContainerID + log.G(ctx).Tracef("CWCOWCombinedLayers:: ContainerID: %v, ContainerRootPath: %v, Layers: %v, ScratchPath: %v", + containerID, settings.CombinedLayers.ContainerRootPath, settings.CombinedLayers.Layers, settings.CombinedLayers.ScratchPath) + + // TODO: Update modifyCombinedLayers with verified CimFS API + + // The following two folders are expected to be present in the scratch. + // But since we have just formatted the scratch we would need to + // create them manually. + sandboxStateDirectory := filepath.Join(settings.CombinedLayers.ContainerRootPath, sandboxStateDirName) + err = os.Mkdir(sandboxStateDirectory, 0777) + if err != nil { + return fmt.Errorf("failed to create sandboxStateDirectory: %w", err) + } + + hivesDirectory := filepath.Join(settings.CombinedLayers.ContainerRootPath, hivesDirName) + err = os.Mkdir(hivesDirectory, 0777) + if err != nil { + return fmt.Errorf("failed to create hivesDirectory: %w", err) + } + + // Reconstruct WCOWCombinedLayers{} req before forwarding to GCS + // as GCS does not understand ResourceTypeCWCOWCombinedLayers + modifyGuestSettingsRequest.ResourceType = guestresource.ResourceTypeCombinedLayers + modifyGuestSettingsRequest.Settings = settings.CombinedLayers + modifyRequest.Request = modifyGuestSettingsRequest + buf, err := json.Marshal(modifyRequest) + if err != nil { + return fmt.Errorf("failed to marshal rpcModifySettings: %w", err) + } + var newRequest request + newRequest.ctx = req.ctx + newRequest.header = req.header + newRequest.header.Size = uint32(len(buf)) + prot.HdrSize + newRequest.message = buf + req = &newRequest + + case guestresource.ResourceTypeMappedVirtualDiskForContainerScratch: + wcowMappedVirtualDisk := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWMappedVirtualDisk) + log.G(ctx).Tracef("ResourceTypeMappedVirtualDiskForContainerScratch: { %v }", wcowMappedVirtualDisk) + + // 1. TODO (Mahati): Need to enforce policy before calling into fsFormatter + // 2. Call fsFormatter to format the scratch disk. + // This will return the volume path of the mounted scratch. + // Scratch disk should be >= 30 GB for refs formatter to work. + + // fsFormatter understands only virtualDevObjectPathFormat. Therefore fetch the + // disk number for the corresponding lun + var devNumber uint32 + // It could take a few seconds for the attached scsi disk + // to show up inside the UVM. Therefore adding retry logic + // with delay here. + for try := 0; try < 5; try++ { + time.Sleep(1 * time.Second) + devNumber, err = windevice.GetDeviceNumberFromControllerLUN(req.ctx, + 0, /* Only one controller allowed in wcow hyperv */ + uint8(wcowMappedVirtualDisk.Lun)) + if err != nil { + if try == 4 { + // bail out + return fmt.Errorf("error getting diskNumber for LUN %d: %w", wcowMappedVirtualDisk.Lun, err) + } + continue + } else { + log.G(ctx).Tracef("DiskNumber of lun %d is: %d", wcowMappedVirtualDisk.Lun, devNumber) + break + } + } + diskPath := fmt.Sprintf(fsformatter.VirtualDevObjectPathFormat, devNumber) + log.G(ctx).Tracef("diskPath: %v, diskNumber: %v ", diskPath, devNumber) + mountedVolumePath, err := fsformatter.InvokeFsFormatter(req.ctx, diskPath) + if err != nil { + return fmt.Errorf("failed to invoke refsFormatter: %w", err) + } + log.G(ctx).Tracef("mountedVolumePath returned from InvokeFsFormatter: %v", mountedVolumePath) + + // Forward the req as is to inbox gcs and let it retreive the volume. + // While forwarding request to inbox gcs, make sure to replace the + // resourceType to ResourceTypeMappedVirtualDisk that inbox GCS + // understands. + modifyGuestSettingsRequest.ResourceType = guestresource.ResourceTypeMappedVirtualDisk + modifyRequest.Request = modifyGuestSettingsRequest + buf, err := json.Marshal(modifyRequest) + if err != nil { + return fmt.Errorf("failed to marshal WCOWMappedVirtualDisk: %w", err) + } + var newRequest request + newRequest.ctx = req.ctx + newRequest.header = req.header + newRequest.header.Size = uint32(len(buf)) + prot.HdrSize + newRequest.message = buf + req = &newRequest + + default: + // Invalid request + return fmt.Errorf("invald modifySettingsRequest: %v", guestResourceType) + } + } + + b.forwardRequestToGcs(req) + return nil +} diff --git a/internal/gcs-sidecar/host.go b/internal/gcs-sidecar/host.go new file mode 100644 index 0000000000..b17cd38448 --- /dev/null +++ b/internal/gcs-sidecar/host.go @@ -0,0 +1,64 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "errors" + "fmt" + "sync" + + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" +) + +type Host struct { + // state required for the security policy enforcement + policyMutex sync.Mutex + securityPolicyEnforcer securitypolicy.SecurityPolicyEnforcer + securityPolicyEnforcerSet bool +} + +type SecurityPolicyEnforcer struct { + // State required for the security policy enforcement + securityPolicyEnforcer securitypolicy.SecurityPolicyEnforcer + securityPolicyEnforcerSet bool +} + +func NewHost(initialEnforcer securitypolicy.SecurityPolicyEnforcer) *Host { + return &Host{ + securityPolicyEnforcer: initialEnforcer, + securityPolicyEnforcerSet: false, + } +} + +func (h *Host) SetWCOWConfidentialUVMOptions(securityPolicyRequest *guestresource.WCOWConfidentialOptions) error { + h.policyMutex.Lock() + defer h.policyMutex.Unlock() + + if h.securityPolicyEnforcerSet { + return errors.New("security policy has already been set") + } + + // This limit ensures messages are below the character truncation limit that + // can be imposed by an orchestrator + maxErrorMessageLength := 3 * 1024 + + // Initialize security policy enforcer for a given enforcer type and + // encoded security policy. + p, err := securitypolicy.CreateSecurityPolicyEnforcer( + "rego", + securityPolicyRequest.EncodedSecurityPolicy, + DefaultCRIMounts(), + DefaultCRIPrivilegedMounts(), + maxErrorMessageLength, + ) + if err != nil { + return fmt.Errorf("error creating security policy enforcer: %w", err) + } + + h.securityPolicyEnforcer = p + h.securityPolicyEnforcerSet = true + + return nil +} diff --git a/internal/gcs-sidecar/policy.go b/internal/gcs-sidecar/policy.go new file mode 100644 index 0000000000..13b96ce64d --- /dev/null +++ b/internal/gcs-sidecar/policy.go @@ -0,0 +1,19 @@ +//go:build windows +// +build windows + +package bridge + +import ( + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// DefaultCRIMounts returns default mounts added to windows spec by containerD. +func DefaultCRIMounts() []oci.Mount { + return []oci.Mount{} +} + +// DefaultCRIPrivilegedMounts returns a slice of mounts which are added to the +// windows container spec when a container runs in a privileged mode. +func DefaultCRIPrivilegedMounts() []oci.Mount { + return []oci.Mount{} +} diff --git a/internal/gcs-sidecar/uvm.go b/internal/gcs-sidecar/uvm.go new file mode 100644 index 0000000000..4fccdf3df6 --- /dev/null +++ b/internal/gcs-sidecar/uvm.go @@ -0,0 +1,123 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "encoding/json" + "fmt" + + "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" +) + +func unmarshalContainerModifySettings(req *request) (_ *prot.ContainerModifySettings, err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::unmarshalContainerModifySettings") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerModifySettings + var requestRawSettings json.RawMessage + r.Request = &requestRawSettings + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return nil, fmt.Errorf("failed to unmarshal rpcModifySettings: %w", err) + } + + var modifyGuestSettingsRequest guestrequest.ModificationRequest + var rawGuestRequest json.RawMessage + modifyGuestSettingsRequest.Settings = &rawGuestRequest + if err := commonutils.UnmarshalJSONWithHresult(requestRawSettings, &modifyGuestSettingsRequest); err != nil { + return nil, fmt.Errorf("invalid rpcModifySettings ModificationRequest: %w", err) + } + + if modifyGuestSettingsRequest.RequestType == "" { + modifyGuestSettingsRequest.RequestType = guestrequest.RequestTypeAdd + } + + if modifyGuestSettingsRequest.ResourceType != "" { + switch modifyGuestSettingsRequest.ResourceType { + case guestresource.ResourceTypeCWCOWCombinedLayers: + settings := &guestresource.CWCOWCombinedLayers{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeCWCOWCombinedLayers request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeCombinedLayers: + settings := &guestresource.WCOWCombinedLayers{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeCombinedLayers request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeNetworkNamespace: + settings := &hcn.HostComputeNamespace{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeNetworkNamespace request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeNetwork: + settings := &guestrequest.NetworkModifyRequest{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeNetwork request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeMappedVirtualDisk: + wcowMappedVirtualDisk := &guestresource.WCOWMappedVirtualDisk{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowMappedVirtualDisk); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedVirtualDisk request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk + + case guestresource.ResourceTypeHvSocket: + hvSocketAddress := &hcsschema.HvSocketAddress{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, hvSocketAddress); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeHvSocket request: %w", err) + } + modifyGuestSettingsRequest.Settings = hvSocketAddress + + case guestresource.ResourceTypeMappedDirectory: + settings := &hcsschema.MappedDirectory{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedDirectory request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeSecurityPolicy: + securityPolicyRequest := &guestresource.WCOWConfidentialOptions{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, securityPolicyRequest); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeSecurityPolicy request: %w", err) + } + modifyGuestSettingsRequest.Settings = securityPolicyRequest + + case guestresource.ResourceTypeMappedVirtualDiskForContainerScratch: + wcowMappedVirtualDisk := &guestresource.WCOWMappedVirtualDisk{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowMappedVirtualDisk); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedVirtualDiskForContainerScratch request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk + + case guestresource.ResourceTypeWCOWBlockCims: + wcowBlockCimMounts := &guestresource.WCOWBlockCIMMounts{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowBlockCimMounts); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeWCOWBlockCims request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowBlockCimMounts + + default: + // Invalid request + log.G(ctx).Errorf("Invald modifySettingsRequest: %v", modifyGuestSettingsRequest.ResourceType) + return nil, fmt.Errorf("invald modifySettingsRequest") + } + } + r.Request = &modifyGuestSettingsRequest + return &r, nil +} diff --git a/internal/gcs/bridge.go b/internal/gcs/bridge.go index 17e54f8242..c1e026778d 100644 --- a/internal/gcs/bridge.go +++ b/internal/gcs/bridge.go @@ -19,35 +19,22 @@ import ( "go.opencensus.io/trace" "golang.org/x/sys/windows" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" ) -const ( - hdrSize = 16 - hdrOffType = 0 - hdrOffSize = 4 - hdrOffID = 8 - - // maxMsgSize is the maximum size of an incoming message. This is not - // enforced by the guest today but some maximum must be set to avoid - // unbounded allocations. - // - // Matches HCS limitions on maximum (sent and received) message size. - maxMsgSize = 0x10000 -) - type requestMessage interface { - Base() *requestBase + Base() *prot.RequestBase } type responseMessage interface { - Base() *responseBase + Base() *prot.ResponseBase } // rpc represents an outstanding rpc request to the guest type rpc struct { - proc rpcProc + proc prot.RPCProc id int64 req requestMessage resp responseMessage @@ -80,7 +67,7 @@ const ( bridgeFailureTimeout = time.Minute * 5 ) -type notifyFunc func(*containerNotification) error +type notifyFunc func(*prot.ContainerNotification) error // newBridge returns a bridge on `conn`. It calls `notify` when a // notification message arrives from the guest. It logs transport errors and @@ -143,7 +130,7 @@ func (brdg *bridge) Wait() error { // AsyncRPC sends an RPC request to the guest but does not wait for a response. // If the message cannot be sent before the context is done, then an error is // returned. -func (brdg *bridge) AsyncRPC(ctx context.Context, proc rpcProc, req requestMessage, resp responseMessage) (*rpc, error) { +func (brdg *bridge) AsyncRPC(ctx context.Context, proc prot.RPCProc, req requestMessage, resp responseMessage) (*rpc, error) { call := &rpc{ ch: make(chan struct{}), proc: proc, @@ -224,7 +211,7 @@ func (call *rpc) Wait() { // If allowCancel is set and the context becomes done, returns an error without // waiting for a response. Avoid this on messages that are not idempotent or // otherwise safe to ignore the response of. -func (brdg *bridge) RPC(ctx context.Context, proc rpcProc, req requestMessage, resp responseMessage, allowCancel bool) error { +func (brdg *bridge) RPC(ctx context.Context, proc prot.RPCProc, req requestMessage, resp responseMessage, allowCancel bool) error { call, err := brdg.AsyncRPC(ctx, proc, req, resp) if err != nil { return err @@ -261,26 +248,26 @@ func (brdg *bridge) recvLoopRoutine() { } } -func readMessage(r io.Reader) (int64, msgType, []byte, error) { +func readMessage(r io.Reader) (int64, prot.MsgType, []byte, error) { _, span := oc.StartSpan(context.Background(), "bridge receive read message", oc.WithClientSpanKind) defer span.End() - var h [hdrSize]byte + var h [prot.HdrSize]byte _, err := io.ReadFull(r, h[:]) if err != nil { return 0, 0, nil, fmt.Errorf("header read: %w", err) } - typ := msgType(binary.LittleEndian.Uint32(h[hdrOffType:])) - n := binary.LittleEndian.Uint32(h[hdrOffSize:]) - id := int64(binary.LittleEndian.Uint64(h[hdrOffID:])) + typ := prot.MsgType(binary.LittleEndian.Uint32(h[prot.HdrOffType:])) + n := binary.LittleEndian.Uint32(h[prot.HdrOffSize:]) + id := int64(binary.LittleEndian.Uint64(h[prot.HdrOffID:])) span.AddAttributes( trace.StringAttribute("type", typ.String()), trace.Int64Attribute("message-id", id)) - if n < hdrSize || n > maxMsgSize { + if n < prot.HdrSize || n > prot.MaxMsgSize { return 0, 0, nil, fmt.Errorf("invalid message size %d", n) } - n -= hdrSize + n -= prot.HdrSize b := make([]byte, n) _, err = io.ReadFull(r, b) if err != nil { @@ -311,8 +298,8 @@ func (brdg *bridge) recvLoop() error { "type": typ.String(), "message-id": id}).Trace("bridge receive") - switch typ & msgTypeMask { - case msgTypeResponse: + switch typ & prot.MsgTypeMask { + case prot.MsgTypeResponse: // Find the request associated with this response. brdg.mu.Lock() call := brdg.rpcs[id] @@ -344,11 +331,11 @@ func (brdg *bridge) recvLoop() error { return err } - case msgTypeNotify: - if typ != notifyContainer|msgTypeNotify { + case prot.MsgTypeNotify: + if typ != prot.NotifyContainer|prot.MsgTypeNotify { return fmt.Errorf("bridge received unknown unknown notification message %s", typ) } - var ntf containerNotification + var ntf prot.ContainerNotification ntf.ResultInfo.Value = &json.RawMessage{} err := json.Unmarshal(b, &ntf) if err != nil { @@ -383,7 +370,7 @@ func (brdg *bridge) sendLoop() { } } -func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ msgType, id int64, req interface{}) error { +func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ prot.MsgType, id int64, req interface{}) error { var err error _, span := oc.StartSpan(context.Background(), "bridge send", oc.WithClientSpanKind) defer span.End() @@ -393,24 +380,24 @@ func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ msgTy trace.Int64Attribute("message-id", id)) // Prepare the buffer with the message. - var h [hdrSize]byte - binary.LittleEndian.PutUint32(h[hdrOffType:], uint32(typ)) - binary.LittleEndian.PutUint64(h[hdrOffID:], uint64(id)) + var h [prot.HdrSize]byte + binary.LittleEndian.PutUint32(h[prot.HdrOffType:], uint32(typ)) + binary.LittleEndian.PutUint64(h[prot.HdrOffID:], uint64(id)) buf.Write(h[:]) err = enc.Encode(req) if err != nil { return fmt.Errorf("bridge encode: %w", err) } // Update the message header with the size. - binary.LittleEndian.PutUint32(buf.Bytes()[hdrOffSize:], uint32(buf.Len())) + binary.LittleEndian.PutUint32(buf.Bytes()[prot.HdrOffSize:], uint32(buf.Len())) if brdg.log.Logger.GetLevel() > logrus.DebugLevel { - b := buf.Bytes()[hdrSize:] + b := buf.Bytes()[prot.HdrSize:] switch typ { // container environment vars are in rpCreate for linux; rpcExecuteProcess for windows - case msgType(rpcCreate) | msgTypeRequest: + case prot.MsgType(prot.RPCCreate) | prot.MsgTypeRequest: b, err = log.ScrubBridgeCreate(b) - case msgType(rpcExecuteProcess) | msgTypeRequest: + case prot.MsgType(prot.RPCExecuteProcess) | prot.MsgTypeRequest: b, err = log.ScrubBridgeExecProcess(b) } if err != nil { @@ -443,7 +430,7 @@ func (brdg *bridge) sendRPC(buf *bytes.Buffer, enc *json.Encoder, call *rpc) err brdg.rpcs[id] = call brdg.nextID++ brdg.mu.Unlock() - typ := msgType(call.proc) | msgTypeRequest + typ := prot.MsgType(call.proc) | prot.MsgTypeRequest err := brdg.writeMessage(buf, enc, typ, id, call.req) if err != nil { // Try to reclaim this request and fail it. diff --git a/internal/gcs/bridge_test.go b/internal/gcs/bridge_test.go index d6b3265c60..3da35e9d9d 100644 --- a/internal/gcs/bridge_test.go +++ b/internal/gcs/bridge_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/sirupsen/logrus" ) @@ -33,7 +34,7 @@ func pipeConn() (*stitched, *stitched) { return &stitched{r1, w2}, &stitched{r2, w1} } -func sendMessage(t *testing.T, w io.Writer, typ msgType, id int64, msg []byte) { +func sendMessage(t *testing.T, w io.Writer, typ prot.MsgType, id int64, msg []byte) { t.Helper() var h [16]byte binary.LittleEndian.PutUint32(h[:], uint32(typ)) @@ -63,18 +64,18 @@ func reflector(t *testing.T, rw io.ReadWriteCloser, delay time.Duration) { return } time.Sleep(delay) // delay is used to test timeouts (when non-zero) - typ ^= msgTypeResponse ^ msgTypeRequest + typ ^= prot.MsgTypeResponse ^ prot.MsgTypeRequest sendMessage(t, rw, typ, id, msg) } } type testReq struct { - requestBase + prot.RequestBase X, Y int } type testResp struct { - responseBase + prot.ResponseBase X, Y int } @@ -92,7 +93,7 @@ func TestBridgeRPC(t *testing.T) { defer b.Close() req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), rpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RPCCreate, &req, &resp, false) if err != nil { t.Fatal(err) } @@ -107,7 +108,7 @@ func TestBridgeRPCResponseTimeout(t *testing.T) { b.Timeout = time.Millisecond * 100 req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), rpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RPCCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -121,7 +122,7 @@ func TestBridgeRPCContextDone(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, rpcCreate, &req, &resp, true) + err := b.RPC(ctx, prot.RPCCreate, &req, &resp, true) if err != context.DeadlineExceeded { //nolint:errorlint t.Fatalf("expected deadline exceeded, got %s", err) } @@ -135,7 +136,7 @@ func TestBridgeRPCContextDoneNoCancel(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, rpcCreate, &req, &resp, false) + err := b.RPC(ctx, prot.RPCCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -145,13 +146,13 @@ func TestBridgeRPCBridgeClosed(t *testing.T) { b := startReflectedBridge(t, 0) eerr := errors.New("forcibly terminated") b.kill(eerr) - err := b.RPC(context.Background(), rpcCreate, nil, nil, false) + err := b.RPC(context.Background(), prot.RPCCreate, nil, nil, false) if err != eerr { //nolint:errorlint t.Fatal("unexpected: ", err) } } -func sendJSON(t *testing.T, w io.Writer, typ msgType, id int64, msg interface{}) error { +func sendJSON(t *testing.T, w io.Writer, typ prot.MsgType, id int64, msg interface{}) error { t.Helper() msgb, err := json.Marshal(msg) if err != nil { @@ -161,7 +162,7 @@ func sendJSON(t *testing.T, w io.Writer, typ msgType, id int64, msg interface{}) return nil } -func notifyThroughBridge(t *testing.T, typ msgType, msg interface{}, fn notifyFunc) error { +func notifyThroughBridge(t *testing.T, typ prot.MsgType, msg interface{}, fn notifyFunc) error { t.Helper() s, c := pipeConn() b := newBridge(s, fn, logrus.NewEntry(logrus.StandardLogger())) @@ -176,9 +177,9 @@ func notifyThroughBridge(t *testing.T, typ msgType, msg interface{}, fn notifyFu } func TestBridgeNotify(t *testing.T) { - ntf := &containerNotification{Operation: "testing"} + ntf := &prot.ContainerNotification{Operation: "testing"} recvd := false - err := notifyThroughBridge(t, msgTypeNotify|notifyContainer, ntf, func(nntf *containerNotification) error { + err := notifyThroughBridge(t, prot.MsgTypeNotify|prot.NotifyContainer, ntf, func(nntf *prot.ContainerNotification) error { if !reflect.DeepEqual(ntf, nntf) { t.Errorf("%+v != %+v", ntf, nntf) } @@ -194,9 +195,9 @@ func TestBridgeNotify(t *testing.T) { } func TestBridgeNotifyFailure(t *testing.T) { - ntf := &containerNotification{Operation: "testing"} + ntf := &prot.ContainerNotification{Operation: "testing"} errMsg := "notify should have failed" - err := notifyThroughBridge(t, msgTypeNotify|notifyContainer, ntf, func(nntf *containerNotification) error { + err := notifyThroughBridge(t, prot.MsgTypeNotify|prot.NotifyContainer, ntf, func(nntf *prot.ContainerNotification) error { return errors.New(errMsg) }) if err == nil || !strings.Contains(err.Error(), errMsg) { diff --git a/internal/gcs/container.go b/internal/gcs/container.go index a64408b834..549abd35a2 100644 --- a/internal/gcs/container.go +++ b/internal/gcs/container.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -53,12 +54,12 @@ func (gc *GuestConnection) CreateContainer(ctx context.Context, cid string, conf if err != nil { return nil, err } - req := containerCreate{ - requestBase: makeRequest(ctx, cid), - ContainerConfig: anyInString{config}, + req := prot.ContainerCreate{ + RequestBase: makeRequest(ctx, cid), + ContainerConfig: prot.AnyInString{Value: config}, } - var resp containerCreateResponse - err = gc.brdg.RPC(ctx, rpcCreate, &req, &resp, false) + var resp prot.ContainerCreateResponse + err = gc.brdg.RPC(ctx, prot.RPCCreate, &req, &resp, false) if err != nil { return nil, err } @@ -129,27 +130,27 @@ func (c *Container) Modify(ctx context.Context, config interface{}) (err error) defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerModifySettings{ - requestBase: makeRequest(ctx, c.id), + req := prot.ContainerModifySettings{ + RequestBase: makeRequest(ctx, c.id), Request: config, } - var resp responseBase - return c.gc.brdg.RPC(ctx, rpcModifySettings, &req, &resp, false) + var resp prot.ResponseBase + return c.gc.brdg.RPC(ctx, prot.RPCModifySettings, &req, &resp, false) } -// Properties returns the requested container properties targeting a V1 schema container. +// Properties returns the requested container properties targeting a V1 schema prot.Container. func (c *Container) Properties(ctx context.Context, types ...schema1.PropertyType) (_ *schema1.ContainerProperties, err error) { ctx, span := oc.StartSpan(ctx, "gcs::Container::Properties", oc.WithClientSpanKind) defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerGetProperties{ - requestBase: makeRequest(ctx, c.id), - Query: containerPropertiesQuery{PropertyTypes: types}, + req := prot.ContainerGetProperties{ + RequestBase: makeRequest(ctx, c.id), + Query: prot.ContainerPropertiesQuery{PropertyTypes: types}, } - var resp containerGetPropertiesResponse - err = c.gc.brdg.RPC(ctx, rpcGetProperties, &req, &resp, true) + var resp prot.ContainerGetPropertiesResponse + err = c.gc.brdg.RPC(ctx, prot.RPCGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -163,12 +164,12 @@ func (c *Container) PropertiesV2(ctx context.Context, types ...hcsschema.Propert defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerGetPropertiesV2{ - requestBase: makeRequest(ctx, c.id), - Query: containerPropertiesQueryV2{PropertyTypes: types}, + req := prot.ContainerGetPropertiesV2{ + RequestBase: makeRequest(ctx, c.id), + Query: prot.ContainerPropertiesQueryV2{PropertyTypes: types}, } - var resp containerGetPropertiesResponseV2 - err = c.gc.brdg.RPC(ctx, rpcGetProperties, &req, &resp, true) + var resp prot.ContainerGetPropertiesResponseV2 + err = c.gc.brdg.RPC(ctx, prot.RPCGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -183,13 +184,13 @@ func (c *Container) Start(ctx context.Context) (err error) { span.AddAttributes(trace.StringAttribute("cid", c.id)) req := makeRequest(ctx, c.id) - var resp responseBase - return c.gc.brdg.RPC(ctx, rpcStart, &req, &resp, false) + var resp prot.ResponseBase + return c.gc.brdg.RPC(ctx, prot.RPCStart, &req, &resp, false) } -func (c *Container) shutdown(ctx context.Context, proc rpcProc) error { +func (c *Container) shutdown(ctx context.Context, proc prot.RPCProc) error { req := makeRequest(ctx, c.id) - var resp responseBase + var resp prot.ResponseBase err := c.gc.brdg.RPC(ctx, proc, &req, &resp, true) if err != nil { if uint32(resp.Result) != hrComputeSystemDoesNotExist { @@ -215,7 +216,7 @@ func (c *Container) Shutdown(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, rpcShutdownGraceful) + return c.shutdown(ctx, prot.RPCShutdownGraceful) } // Terminate sends a forceful terminate request to the container. The container @@ -229,7 +230,7 @@ func (c *Container) Terminate(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, rpcShutdownForced) + return c.shutdown(ctx, prot.RPCShutdownForced) } func (c *Container) WaitChannel() <-chan struct{} { diff --git a/internal/gcs/guestconnection.go b/internal/gcs/guestconnection.go index fe974b5c17..9107dd4d3b 100644 --- a/internal/gcs/guestconnection.go +++ b/internal/gcs/guestconnection.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -28,7 +29,7 @@ import ( const ( protocolVersion = 4 - firstIoChannelVsockPort = LinuxGcsVsockPort + 1 + firstIoChannelVsockPort = prot.LinuxGcsVsockPort + 1 nullContainerID = "00000000-0000-0000-0000-000000000000" ) @@ -117,12 +118,12 @@ func (gc *GuestConnection) Protocol() uint32 { // isColdStart should be true when the UVM is being connected to for the first time post-boot. // It should be false for subsequent connections (e.g. if reconnecting to an existing UVM). func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGuestState *InitialGuestState) (err error) { - req := negotiateProtocolRequest{ + req := prot.NegotiateProtocolRequest{ MinimumVersion: protocolVersion, MaximumVersion: protocolVersion, } - var resp negotiateProtocolResponse - err = gc.brdg.RPC(ctx, rpcNegotiateProtocol, &req, &resp, true) + var resp prot.NegotiateProtocolResponse + err = gc.brdg.RPC(ctx, prot.RPCNegotiateProtocol, &req, &resp, true) if err != nil { return err } @@ -141,25 +142,25 @@ func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGu } if isColdStart && resp.Capabilities.SendHostCreateMessage { - conf := &uvmConfig{ + conf := &prot.UvmConfig{ SystemType: "Container", } if initGuestState != nil && initGuestState.Timezone != nil { conf.TimeZoneInformation = initGuestState.Timezone } - createReq := containerCreate{ - requestBase: makeRequest(ctx, nullContainerID), - ContainerConfig: anyInString{conf}, + createReq := prot.ContainerCreate{ + RequestBase: makeRequest(ctx, nullContainerID), + ContainerConfig: prot.AnyInString{Value: conf}, } - var createResp responseBase - err = gc.brdg.RPC(ctx, rpcCreate, &createReq, &createResp, true) + var createResp prot.ResponseBase + err = gc.brdg.RPC(ctx, prot.RPCCreate, &createReq, &createResp, true) if err != nil { return err } if resp.Capabilities.SendHostStartMessage { startReq := makeRequest(ctx, nullContainerID) - var startResp responseBase - err = gc.brdg.RPC(ctx, rpcStart, &startReq, &startResp, true) + var startResp prot.ResponseBase + err = gc.brdg.RPC(ctx, prot.RPCStart, &startReq, &startResp, true) if err != nil { return err } @@ -175,12 +176,12 @@ func (gc *GuestConnection) Modify(ctx context.Context, settings interface{}) (er defer span.End() defer func() { oc.SetSpanStatus(span, err) }() - req := containerModifySettings{ - requestBase: makeRequest(ctx, nullContainerID), + req := prot.ContainerModifySettings{ + RequestBase: makeRequest(ctx, nullContainerID), Request: settings, } - var resp responseBase - return gc.brdg.RPC(ctx, rpcModifySettings, &req, &resp, false) + var resp prot.ResponseBase + return gc.brdg.RPC(ctx, prot.RPCModifySettings, &req, &resp, false) } func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err error) { @@ -188,11 +189,11 @@ func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err defer span.End() defer func() { oc.SetSpanStatus(span, err) }() - req := dumpStacksRequest{ - requestBase: makeRequest(ctx, nullContainerID), + req := prot.DumpStacksRequest{ + RequestBase: makeRequest(ctx, nullContainerID), } - var resp dumpStacksResponse - err = gc.brdg.RPC(ctx, rpcDumpStacks, &req, &resp, false) + var resp prot.DumpStacksResponse + err = gc.brdg.RPC(ctx, prot.RPCDumpStacks, &req, &resp, false) return resp.GuestStacks, err } @@ -202,11 +203,11 @@ func (gc *GuestConnection) DeleteContainerState(ctx context.Context, cid string) defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", cid)) - req := deleteContainerStateRequest{ - requestBase: makeRequest(ctx, cid), + req := prot.DeleteContainerStateRequest{ + RequestBase: makeRequest(ctx, cid), } - var resp responseBase - return gc.brdg.RPC(ctx, rpcDeleteContainerState, &req, &resp, false) + var resp prot.ResponseBase + return gc.brdg.RPC(ctx, prot.RPCDeleteContainerState, &req, &resp, false) } // Close terminates the guest connection. It is undefined to call any other @@ -263,7 +264,7 @@ func (gc *GuestConnection) requestNotify(cid string, ch chan struct{}) error { return nil } -func (gc *GuestConnection) notify(ntf *containerNotification) error { +func (gc *GuestConnection) notify(ntf *prot.ContainerNotification) error { cid := ntf.ContainerID gc.mu.Lock() ch := gc.notifyChs[cid] @@ -287,14 +288,14 @@ func (gc *GuestConnection) clearNotifies() { } } -func makeRequest(ctx context.Context, cid string) requestBase { - r := requestBase{ +func makeRequest(ctx context.Context, cid string) prot.RequestBase { + r := prot.RequestBase{ ContainerID: cid, } span := trace.FromContext(ctx) if span != nil { sc := span.SpanContext() - r.OpenCensusSpanContext = &ocspancontext{ + r.OpenCensusSpanContext = &prot.Ocspancontext{ TraceID: hex.EncodeToString(sc.TraceID[:]), SpanID: hex.EncodeToString(sc.SpanID[:]), TraceOptions: uint32(sc.TraceOptions), diff --git a/internal/gcs/guestconnection_test.go b/internal/gcs/guestconnection_test.go index facb0dd34b..6a72cb8a3f 100644 --- a/internal/gcs/guestconnection_test.go +++ b/internal/gcs/guestconnection_test.go @@ -21,6 +21,7 @@ import ( "go.opencensus.io/trace" "go.opencensus.io/trace/tracestate" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/oc" ) @@ -55,24 +56,24 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { } return err } - switch proc := rpcProc(typ &^ msgTypeRequest); proc { - case rpcNegotiateProtocol: - err := sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &negotiateProtocolResponse{ + switch proc := prot.RPCProc(typ &^ prot.MsgTypeRequest); proc { + case prot.RPCNegotiateProtocol: + err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.NegotiateProtocolResponse{ Version: protocolVersion, - Capabilities: gcsCapabilities{ + Capabilities: prot.GcsCapabilities{ RuntimeOsType: "linux", }, }) if err != nil { return err } - case rpcCreate: - err := sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &containerCreateResponse{}) + case prot.RPCCreate: + err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ContainerCreateResponse{}) if err != nil { return err } - case rpcExecuteProcess: - var req containerExecuteProcess + case prot.RPCExecuteProcess: + var req prot.ContainerExecuteProcess var params baseProcessParams req.Settings.ProcessParameters.Value = ¶ms err := json.Unmarshal(b, &req) @@ -111,27 +112,27 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { stdout.Close() }() } - err = sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &containerExecuteProcessResponse{ + err = sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ContainerExecuteProcessResponse{ ProcessID: 42, }) if err != nil { return err } - case rpcWaitForProcess: + case prot.RPCWaitForProcess: // nothing - case rpcShutdownForced: - var req requestBase + case prot.RPCShutdownForced: + var req prot.RequestBase err = json.Unmarshal(b, &req) if err != nil { return err } - err = sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &responseBase{}) + err = sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ResponseBase{}) if err != nil { return err } time.Sleep(50 * time.Millisecond) - err = sendJSON(t, rw, msgType(msgTypeNotify|notifyContainer), 0, &containerNotification{ - requestBase: requestBase{ + err = sendJSON(t, rw, prot.MsgType(prot.MsgTypeNotify|prot.NotifyContainer), 0, &prot.ContainerNotification{ + RequestBase: prot.RequestBase{ ContainerID: req.ContainerID, }, }) diff --git a/internal/gcs/process.go b/internal/gcs/process.go index 87c5c29ae4..91d3a87faa 100644 --- a/internal/gcs/process.go +++ b/internal/gcs/process.go @@ -12,6 +12,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/oc" @@ -29,7 +30,7 @@ type Process struct { cid string id uint32 waitCall *rpc - waitResp containerWaitForProcessResponse + waitResp prot.ContainerWaitForProcessResponse stdin, stdout, stderr *ioChannel stdinCloseWriteOnce sync.Once stdinCloseWriteErr error @@ -52,10 +53,10 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac return nil, err } - req := containerExecuteProcess{ - requestBase: makeRequest(ctx, cid), - Settings: executeProcessSettings{ - ProcessParameters: anyInString{params}, + req := prot.ContainerExecuteProcess{ + RequestBase: makeRequest(ctx, cid), + Settings: prot.ExecuteProcessSettings{ + ProcessParameters: prot.AnyInString{Value: params}, }, } @@ -68,8 +69,8 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac // Construct the stdio channels. Windows guests expect hvsock service IDs // instead of vsock ports. - var hvsockSettings executeProcessStdioRelaySettings - var vsockSettings executeProcessVsockStdioRelaySettings + var hvsockSettings prot.ExecuteProcessStdioRelaySettings + var vsockSettings prot.ExecuteProcessVsockStdioRelaySettings if gc.os == "windows" { req.Settings.StdioRelaySettings = &hvsockSettings } else { @@ -100,20 +101,20 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac hvsockSettings.StdErr = &g } - var resp containerExecuteProcessResponse - err = gc.brdg.RPC(ctx, rpcExecuteProcess, &req, &resp, false) + var resp prot.ContainerExecuteProcessResponse + err = gc.brdg.RPC(ctx, prot.RPCExecuteProcess, &req, &resp, false) if err != nil { return nil, err } p.id = resp.ProcessID log.G(ctx).WithField("pid", p.id).Debug("created process pid") // Start a wait message. - waitReq := containerWaitForProcess{ - requestBase: makeRequest(ctx, cid), + waitReq := prot.ContainerWaitForProcess{ + RequestBase: makeRequest(ctx, cid), ProcessID: p.id, TimeoutInMs: 0xffffffff, } - p.waitCall, err = gc.brdg.AsyncRPC(ctx, rpcWaitForProcess, &waitReq, &p.waitResp) + p.waitCall, err = gc.brdg.AsyncRPC(ctx, prot.RPCWaitForProcess, &waitReq, &p.waitResp) if err != nil { return nil, fmt.Errorf("failed to wait on process, leaking process: %w", err) } @@ -220,14 +221,14 @@ func (p *Process) ResizeConsole(ctx context.Context, width, height uint16) (err trace.StringAttribute("cid", p.cid), trace.Int64Attribute("pid", int64(p.id))) - req := containerResizeConsole{ - requestBase: makeRequest(ctx, p.cid), + req := prot.ContainerResizeConsole{ + RequestBase: makeRequest(ctx, p.cid), ProcessID: p.id, Height: height, Width: width, } - var resp responseBase - return p.gc.brdg.RPC(ctx, rpcResizeConsole, &req, &resp, true) + var resp prot.ResponseBase + return p.gc.brdg.RPC(ctx, prot.RPCResizeConsole, &req, &resp, true) } // Signal sends a signal to the process, returning whether it was delivered. @@ -239,15 +240,15 @@ func (p *Process) Signal(ctx context.Context, options interface{}) (_ bool, err trace.StringAttribute("cid", p.cid), trace.Int64Attribute("pid", int64(p.id))) - req := containerSignalProcess{ - requestBase: makeRequest(ctx, p.cid), + req := prot.ContainerSignalProcess{ + RequestBase: makeRequest(ctx, p.cid), ProcessID: p.id, Options: options, } - var resp responseBase + var resp prot.ResponseBase // FUTURE: SIGKILL is idempotent and can safely be cancelled, but this interface // does currently make it easy to determine what signal is being sent. - err = p.gc.brdg.RPC(ctx, rpcSignalProcess, &req, &resp, false) + err = p.gc.brdg.RPC(ctx, prot.RPCSignalProcess, &req, &resp, false) if err != nil { if uint32(resp.Result) != hrNotFound { return false, err diff --git a/internal/gcs/prot/protocol.go b/internal/gcs/prot/protocol.go new file mode 100644 index 0000000000..6be8f95482 --- /dev/null +++ b/internal/gcs/prot/protocol.go @@ -0,0 +1,399 @@ +//go:build windows + +package prot + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/hcs/schema1" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" +) + +const ( + HdrSize = 16 + HdrOffType = 0 + HdrOffSize = 4 + HdrOffID = 8 + + // maxMsgSize is the maximum size of an incoming message. This is not + // enforced by the guest today but some maximum must be set to avoid + // unbounded allocations. + MaxMsgSize = 0x10000 + + // LinuxGcsVsockPort is the vsock port number that the Linux GCS will + // connect to. + LinuxGcsVsockPort = 0x40000000 +) + +// e0e16197-dd56-4a10-9195-5ee7a155a838 +var HvGUIDLoopback = guid.GUID{ + Data1: 0xe0e16197, + Data2: 0xdd56, + Data3: 0x4a10, + Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38}, +} + +// a42e7cda-d03f-480c-9cc2-a4de20abb878 +var HvGUIDParent = guid.GUID{ + Data1: 0xa42e7cda, + Data2: 0xd03f, + Data3: 0x480c, + Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78}, +} + +// WindowsGcsHvsockServiceID is the hvsock service ID that the Windows GCS +// will connect to. +var WindowsGcsHvsockServiceID = guid.GUID{ + Data1: 0xacef5661, + Data2: 0x84a1, + Data3: 0x4e44, + Data4: [8]uint8{0x85, 0x6b, 0x62, 0x45, 0xe6, 0x9f, 0x46, 0x20}, +} + +// WindowsSidecarGcsHvsockServiceID is the hvsock service ID that the Windows GCS +// sidecar will connect to. This is only used in the confidential mode. +var WindowsSidecarGcsHvsockServiceID = guid.GUID{ + Data1: 0xae8da506, + Data2: 0xa019, + Data3: 0x4553, + Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, +} + +// WindowsGcsHvHostID is the hvsock address for the parent of the VM running the GCS +var WindowsGcsHvHostID = guid.GUID{ + Data1: 0x894cc2d6, + Data2: 0x9d79, + Data3: 0x424f, + Data4: [8]uint8{0x93, 0xfe, 0x42, 0x96, 0x9a, 0xe6, 0xd8, 0xd1}, +} + +type AnyInString struct { + Value interface{} +} + +func (a *AnyInString) MarshalText() ([]byte, error) { + return json.Marshal(a.Value) +} + +func (a *AnyInString) UnmarshalText(b []byte) error { + return json.Unmarshal(b, &a.Value) +} + +type RPCProc uint32 + +const ( + RPCCreate RPCProc = (iota+1)<<8 | 1 + RPCStart + RPCShutdownGraceful + RPCShutdownForced + RPCExecuteProcess + RPCWaitForProcess + RPCSignalProcess + RPCResizeConsole + RPCGetProperties + RPCModifySettings + RPCNegotiateProtocol + RPCDumpStacks + RPCDeleteContainerState + RPCUpdateContainer + RPCLifecycleNotification +) + +func (rpc RPCProc) String() string { + switch rpc { + case RPCCreate: + return "Create" + case RPCStart: + return "Start" + case RPCShutdownGraceful: + return "ShutdownGraceful" + case RPCShutdownForced: + return "ShutdownForced" + case RPCExecuteProcess: + return "ExecuteProcess" + case RPCWaitForProcess: + return "WaitForProcess" + case RPCSignalProcess: + return "SignalProcess" + case RPCResizeConsole: + return "ResizeConsole" + case RPCGetProperties: + return "GetProperties" + case RPCModifySettings: + return "ModifySettings" + case RPCNegotiateProtocol: + return "NegotiateProtocol" + case RPCDumpStacks: + return "DumpStacks" + case RPCDeleteContainerState: + return "DeleteContainerState" + case RPCUpdateContainer: + return "UpdateContainer" + case RPCLifecycleNotification: + return "LifecycleNotification" + default: + return "0x" + strconv.FormatUint(uint64(rpc), 16) + } +} + +type MsgType uint32 + +const ( + MsgTypeRequest MsgType = 0x10100000 + MsgTypeResponse MsgType = 0x20100000 + MsgTypeNotify MsgType = 0x30100000 + MsgTypeMask MsgType = 0xfff00000 + + NotifyContainer = 1<<8 | 1 +) + +func (typ MsgType) String() string { + var s string + switch typ & MsgTypeMask { + case MsgTypeRequest: + s = "Request(" + case MsgTypeResponse: + s = "Response(" + case MsgTypeNotify: + s = "Notify(" + switch typ - MsgTypeNotify { + case NotifyContainer: + s += "Container" + default: + s += fmt.Sprintf("%#x", uint32(typ)) + } + return s + ")" + default: + return fmt.Sprintf("%#x", uint32(typ)) + } + s += RPCProc(typ &^ MsgTypeMask).String() + return s + ")" +} + +// Ocspancontext is the internal JSON representation of the OpenCensus +// `trace.SpanContext` for fowarding to a GCS that supports it. +type Ocspancontext struct { + // TraceID is the `hex` encoded string of the OpenCensus + // `SpanContext.TraceID` to propagate to the guest. + TraceID string `json:",omitempty"` + // SpanID is the `hex` encoded string of the OpenCensus `SpanContext.SpanID` + // to propagate to the guest. + SpanID string `json:",omitempty"` + + // TraceOptions is the OpenCensus `SpanContext.TraceOptions` passed through + // to propagate to the guest. + TraceOptions uint32 `json:",omitempty"` + + // Tracestate is the `base64` encoded string of marshaling the OpenCensus + // `SpanContext.TraceState.Entries()` to JSON. + // + // If `SpanContext.Tracestate == nil || + // len(SpanContext.Tracestate.Entries()) == 0` this will be `""`. + Tracestate string `json:",omitempty"` +} + +type RequestBase struct { + ContainerID string `json:"ContainerId"` + ActivityID guid.GUID `json:"ActivityId"` + + // OpenCensusSpanContext is the encoded OpenCensus `trace.SpanContext` if + // set when making the request. + // + // NOTE: This is not a part of the protocol but because its a JSON protocol + // adding fields is a non-breaking change. If the guest supports it this is + // just additive context. + OpenCensusSpanContext *Ocspancontext `json:"ocsc,omitempty"` +} + +func (req *RequestBase) Base() *RequestBase { + return req +} + +type ResponseBase struct { + Result int32 // HResult + ErrorMessage string `json:",omitempty"` + ActivityID guid.GUID `json:"ActivityId,omitempty"` + ErrorRecords []commonutils.ErrorRecord `json:",omitempty"` +} + +func (resp *ResponseBase) Base() *ResponseBase { + return resp +} + +type NegotiateProtocolRequest struct { + RequestBase + MinimumVersion uint32 + MaximumVersion uint32 +} + +type NegotiateProtocolResponse struct { + ResponseBase + Version uint32 `json:",omitempty"` + Capabilities GcsCapabilities `json:",omitempty"` +} + +type DumpStacksRequest struct { + RequestBase +} + +type DumpStacksResponse struct { + ResponseBase + GuestStacks string +} + +type DeleteContainerStateRequest struct { + RequestBase +} + +type ContainerCreate struct { + RequestBase + ContainerConfig AnyInString +} + +type UvmConfig struct { + SystemType string // must be "Container" + TimeZoneInformation *hcsschema.TimeZoneInformation +} + +type ContainerNotification struct { + RequestBase + Type string // Compute.System.NotificationType + Operation string // Compute.System.ActiveOperation + Result int32 // HResult + ResultInfo AnyInString `json:",omitempty"` +} + +type ContainerExecuteProcess struct { + RequestBase + Settings ExecuteProcessSettings +} + +type ExecuteProcessSettings struct { + ProcessParameters AnyInString + StdioRelaySettings *ExecuteProcessStdioRelaySettings `json:",omitempty"` + VsockStdioRelaySettings *ExecuteProcessVsockStdioRelaySettings `json:",omitempty"` +} + +type ExecuteProcessStdioRelaySettings struct { + StdIn *guid.GUID `json:",omitempty"` + StdOut *guid.GUID `json:",omitempty"` + StdErr *guid.GUID `json:",omitempty"` +} + +type ExecuteProcessVsockStdioRelaySettings struct { + StdIn uint32 `json:",omitempty"` + StdOut uint32 `json:",omitempty"` + StdErr uint32 `json:",omitempty"` +} + +type ContainerResizeConsole struct { + RequestBase + ProcessID uint32 `json:"ProcessId"` + Height uint16 + Width uint16 +} + +type ContainerWaitForProcess struct { + RequestBase + ProcessID uint32 `json:"ProcessId"` + TimeoutInMs uint32 +} + +type ContainerSignalProcess struct { + RequestBase + ProcessID uint32 `json:"ProcessId"` + Options interface{} `json:",omitempty"` +} + +type ContainerPropertiesQuery schema1.PropertyQuery + +func (q *ContainerPropertiesQuery) MarshalText() ([]byte, error) { + return json.Marshal((*schema1.PropertyQuery)(q)) +} + +func (q *ContainerPropertiesQuery) UnmarshalText(b []byte) error { + return json.Unmarshal(b, (*schema1.PropertyQuery)(q)) +} + +type ContainerPropertiesQueryV2 hcsschema.PropertyQuery + +func (q *ContainerPropertiesQueryV2) MarshalText() ([]byte, error) { + return json.Marshal((*hcsschema.PropertyQuery)(q)) +} + +func (q *ContainerPropertiesQueryV2) UnmarshalText(b []byte) error { + return json.Unmarshal(b, (*hcsschema.PropertyQuery)(q)) +} + +type ContainerGetProperties struct { + RequestBase + Query ContainerPropertiesQuery +} + +type ContainerGetPropertiesV2 struct { + RequestBase + Query ContainerPropertiesQueryV2 +} + +type ContainerModifySettings struct { + RequestBase + Request interface{} +} + +type GcsCapabilities struct { + SendHostCreateMessage bool + SendHostStartMessage bool + HvSocketConfigOnStartup bool + SendLifecycleNotifications bool + SupportedSchemaVersions []hcsschema.Version + RuntimeOsType string + GuestDefinedCapabilities json.RawMessage +} + +type ContainerCreateResponse struct { + ResponseBase +} + +type ContainerExecuteProcessResponse struct { + ResponseBase + ProcessID uint32 `json:"ProcessId"` +} + +type ContainerWaitForProcessResponse struct { + ResponseBase + ExitCode uint32 +} + +type ContainerProperties schema1.ContainerProperties + +func (p *ContainerProperties) MarshalText() ([]byte, error) { + return json.Marshal((*schema1.ContainerProperties)(p)) +} + +func (p *ContainerProperties) UnmarshalText(b []byte) error { + return json.Unmarshal(b, (*schema1.ContainerProperties)(p)) +} + +type ContainerPropertiesV2 hcsschema.Properties + +func (p *ContainerPropertiesV2) MarshalText() ([]byte, error) { + return json.Marshal((*hcsschema.Properties)(p)) +} + +func (p *ContainerPropertiesV2) UnmarshalText(b []byte) error { + return json.Unmarshal(b, (*hcsschema.Properties)(p)) +} + +type ContainerGetPropertiesResponse struct { + ResponseBase + Properties ContainerProperties +} + +type ContainerGetPropertiesResponseV2 struct { + ResponseBase + Properties ContainerPropertiesV2 +} diff --git a/internal/gcs/protocol.go b/internal/gcs/protocol.go deleted file mode 100644 index 7aeeb4991f..0000000000 --- a/internal/gcs/protocol.go +++ /dev/null @@ -1,371 +0,0 @@ -//go:build windows - -package gcs - -import ( - "encoding/json" - "fmt" - "strconv" - - "github.com/Microsoft/go-winio/pkg/guid" - "github.com/Microsoft/hcsshim/internal/hcs/schema1" - hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" -) - -// LinuxGcsVsockPort is the vsock port number that the Linux GCS will -// connect to. -const LinuxGcsVsockPort = 0x40000000 - -// WindowsGcsHvsockServiceID is the hvsock service ID that the Windows GCS -// will connect to. -var WindowsGcsHvsockServiceID = guid.GUID{ - Data1: 0xacef5661, - Data2: 0x84a1, - Data3: 0x4e44, - Data4: [8]uint8{0x85, 0x6b, 0x62, 0x45, 0xe6, 0x9f, 0x46, 0x20}, -} - -// WindowsGcsHvHostID is the hvsock address for the parent of the VM running the GCS -var WindowsGcsHvHostID = guid.GUID{ - Data1: 0x894cc2d6, - Data2: 0x9d79, - Data3: 0x424f, - Data4: [8]uint8{0x93, 0xfe, 0x42, 0x96, 0x9a, 0xe6, 0xd8, 0xd1}, -} - -type anyInString struct { - Value interface{} -} - -func (a *anyInString) MarshalText() ([]byte, error) { - return json.Marshal(a.Value) -} - -func (a *anyInString) UnmarshalText(b []byte) error { - return json.Unmarshal(b, &a.Value) -} - -type rpcProc uint32 - -const ( - rpcCreate rpcProc = (iota+1)<<8 | 1 - rpcStart - rpcShutdownGraceful - rpcShutdownForced - rpcExecuteProcess - rpcWaitForProcess - rpcSignalProcess - rpcResizeConsole - rpcGetProperties - rpcModifySettings - rpcNegotiateProtocol - rpcDumpStacks - rpcDeleteContainerState - rpcUpdateContainer - rpcLifecycleNotification -) - -func (rpc rpcProc) String() string { - switch rpc { - case rpcCreate: - return "Create" - case rpcStart: - return "Start" - case rpcShutdownGraceful: - return "ShutdownGraceful" - case rpcShutdownForced: - return "ShutdownForced" - case rpcExecuteProcess: - return "ExecuteProcess" - case rpcWaitForProcess: - return "WaitForProcess" - case rpcSignalProcess: - return "SignalProcess" - case rpcResizeConsole: - return "ResizeConsole" - case rpcGetProperties: - return "GetProperties" - case rpcModifySettings: - return "ModifySettings" - case rpcNegotiateProtocol: - return "NegotiateProtocol" - case rpcDumpStacks: - return "DumpStacks" - case rpcDeleteContainerState: - return "DeleteContainerState" - case rpcUpdateContainer: - return "UpdateContainer" - case rpcLifecycleNotification: - return "LifecycleNotification" - default: - return "0x" + strconv.FormatUint(uint64(rpc), 16) - } -} - -type msgType uint32 - -const ( - msgTypeRequest msgType = 0x10100000 - msgTypeResponse msgType = 0x20100000 - msgTypeNotify msgType = 0x30100000 - msgTypeMask msgType = 0xfff00000 - - notifyContainer = 1<<8 | 1 -) - -func (typ msgType) String() string { - var s string - switch typ & msgTypeMask { - case msgTypeRequest: - s = "Request(" - case msgTypeResponse: - s = "Response(" - case msgTypeNotify: - s = "Notify(" - switch typ - msgTypeNotify { - case notifyContainer: - s += "Container" - default: - s += fmt.Sprintf("%#x", uint32(typ)) - } - return s + ")" - default: - return fmt.Sprintf("%#x", uint32(typ)) - } - s += rpcProc(typ &^ msgTypeMask).String() - return s + ")" -} - -// ocspancontext is the internal JSON representation of the OpenCensus -// `trace.SpanContext` for fowarding to a GCS that supports it. -type ocspancontext struct { - // TraceID is the `hex` encoded string of the OpenCensus - // `SpanContext.TraceID` to propagate to the guest. - TraceID string `json:",omitempty"` - // SpanID is the `hex` encoded string of the OpenCensus `SpanContext.SpanID` - // to propagate to the guest. - SpanID string `json:",omitempty"` - - // TraceOptions is the OpenCensus `SpanContext.TraceOptions` passed through - // to propagate to the guest. - TraceOptions uint32 `json:",omitempty"` - - // Tracestate is the `base64` encoded string of marshaling the OpenCensus - // `SpanContext.TraceState.Entries()` to JSON. - // - // If `SpanContext.Tracestate == nil || - // len(SpanContext.Tracestate.Entries()) == 0` this will be `""`. - Tracestate string `json:",omitempty"` -} - -type requestBase struct { - ContainerID string `json:"ContainerId"` - ActivityID guid.GUID `json:"ActivityId"` - - // OpenCensusSpanContext is the encoded OpenCensus `trace.SpanContext` if - // set when making the request. - // - // NOTE: This is not a part of the protocol but because its a JSON protocol - // adding fields is a non-breaking change. If the guest supports it this is - // just additive context. - OpenCensusSpanContext *ocspancontext `json:"ocsc,omitempty"` -} - -func (req *requestBase) Base() *requestBase { - return req -} - -type responseBase struct { - Result int32 // HResult - ErrorMessage string `json:",omitempty"` - ActivityID guid.GUID `json:"ActivityId,omitempty"` - ErrorRecords []errorRecord `json:",omitempty"` -} - -type errorRecord struct { - Result int32 // HResult - Message string - StackTrace string `json:",omitempty"` - ModuleName string - FileName string - Line uint32 - FunctionName string `json:",omitempty"` -} - -func (resp *responseBase) Base() *responseBase { - return resp -} - -type negotiateProtocolRequest struct { - requestBase - MinimumVersion uint32 - MaximumVersion uint32 -} - -type negotiateProtocolResponse struct { - responseBase - Version uint32 `json:",omitempty"` - Capabilities gcsCapabilities `json:",omitempty"` -} - -type dumpStacksRequest struct { - requestBase -} - -type dumpStacksResponse struct { - responseBase - GuestStacks string -} - -type deleteContainerStateRequest struct { - requestBase -} - -type containerCreate struct { - requestBase - ContainerConfig anyInString -} - -type uvmConfig struct { - SystemType string // must be "Container" - TimeZoneInformation *hcsschema.TimeZoneInformation -} - -type containerNotification struct { - requestBase - Type string // Compute.System.NotificationType - Operation string // Compute.System.ActiveOperation - Result int32 // HResult - ResultInfo anyInString `json:",omitempty"` -} - -type containerExecuteProcess struct { - requestBase - Settings executeProcessSettings -} - -type executeProcessSettings struct { - ProcessParameters anyInString - StdioRelaySettings *executeProcessStdioRelaySettings `json:",omitempty"` - VsockStdioRelaySettings *executeProcessVsockStdioRelaySettings `json:",omitempty"` -} - -type executeProcessStdioRelaySettings struct { - StdIn *guid.GUID `json:",omitempty"` - StdOut *guid.GUID `json:",omitempty"` - StdErr *guid.GUID `json:",omitempty"` -} - -type executeProcessVsockStdioRelaySettings struct { - StdIn uint32 `json:",omitempty"` - StdOut uint32 `json:",omitempty"` - StdErr uint32 `json:",omitempty"` -} - -type containerResizeConsole struct { - requestBase - ProcessID uint32 `json:"ProcessId"` - Height uint16 - Width uint16 -} - -type containerWaitForProcess struct { - requestBase - ProcessID uint32 `json:"ProcessId"` - TimeoutInMs uint32 -} - -type containerSignalProcess struct { - requestBase - ProcessID uint32 `json:"ProcessId"` - Options interface{} `json:",omitempty"` -} - -type containerPropertiesQuery schema1.PropertyQuery - -func (q *containerPropertiesQuery) MarshalText() ([]byte, error) { - return json.Marshal((*schema1.PropertyQuery)(q)) -} - -func (q *containerPropertiesQuery) UnmarshalText(b []byte) error { - return json.Unmarshal(b, (*schema1.PropertyQuery)(q)) -} - -type containerPropertiesQueryV2 hcsschema.PropertyQuery - -func (q *containerPropertiesQueryV2) MarshalText() ([]byte, error) { - return json.Marshal((*hcsschema.PropertyQuery)(q)) -} - -func (q *containerPropertiesQueryV2) UnmarshalText(b []byte) error { - return json.Unmarshal(b, (*hcsschema.PropertyQuery)(q)) -} - -type containerGetProperties struct { - requestBase - Query containerPropertiesQuery -} - -type containerGetPropertiesV2 struct { - requestBase - Query containerPropertiesQueryV2 -} - -type containerModifySettings struct { - requestBase - Request interface{} -} - -type gcsCapabilities struct { - SendHostCreateMessage bool - SendHostStartMessage bool - HvSocketConfigOnStartup bool - SendLifecycleNotifications bool - SupportedSchemaVersions []hcsschema.Version - RuntimeOsType string - GuestDefinedCapabilities json.RawMessage -} - -type containerCreateResponse struct { - responseBase -} - -type containerExecuteProcessResponse struct { - responseBase - ProcessID uint32 `json:"ProcessId"` -} - -type containerWaitForProcessResponse struct { - responseBase - ExitCode uint32 -} - -type containerProperties schema1.ContainerProperties - -func (p *containerProperties) MarshalText() ([]byte, error) { - return json.Marshal((*schema1.ContainerProperties)(p)) -} - -func (p *containerProperties) UnmarshalText(b []byte) error { - return json.Unmarshal(b, (*schema1.ContainerProperties)(p)) -} - -type containerPropertiesV2 hcsschema.Properties - -func (p *containerPropertiesV2) MarshalText() ([]byte, error) { - return json.Marshal((*hcsschema.Properties)(p)) -} - -func (p *containerPropertiesV2) UnmarshalText(b []byte) error { - return json.Unmarshal(b, (*hcsschema.Properties)(p)) -} - -type containerGetPropertiesResponse struct { - responseBase - Properties containerProperties -} - -type containerGetPropertiesResponseV2 struct { - responseBase - Properties containerPropertiesV2 -} diff --git a/internal/guest/bridge/bridge.go b/internal/guest/bridge/bridge.go index f14663344f..024c7108b9 100644 --- a/internal/guest/bridge/bridge.go +++ b/internal/guest/bridge/bridge.go @@ -11,9 +11,7 @@ import ( "encoding/json" "fmt" "io" - "math" "os" - "strconv" "sync" "sync/atomic" "time" @@ -23,7 +21,8 @@ import ( "go.opencensus.io/trace" "go.opencensus.io/trace/tracestate" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/log" @@ -360,7 +359,7 @@ func (b *Bridge) ListenAndServe(bridgeIn io.ReadCloser, bridgeOut io.WriteCloser if span != nil { oc.SetSpanStatus(span, err) } - setErrorForResponseBase(resp.Base(), err) + setErrorForResponseBase(resp.Base(), err, "gcs" /* moduleName */) } br.response = resp b.responseChan <- br @@ -446,45 +445,9 @@ func (b *Bridge) PublishNotification(n *prot.ContainerNotification) { // setErrorForResponseBase modifies the passed-in MessageResponseBase to // contain information pertaining to the given error. -func setErrorForResponseBase(response *prot.MessageResponseBase, errForResponse error) { - errorMessage := errForResponse.Error() - stackString := "" - fileName := "" - // We use -1 as a sentinel if no line number found (or it cannot be parsed), - // but that will ultimately end up as [math.MaxUint32], so set it to that explicitly. - // (Still keep using -1 for backwards compatibility ...) - lineNumber := uint32(math.MaxUint32) - functionName := "" - if stack := gcserr.BaseStackTrace(errForResponse); stack != nil { - bottomFrame := stack[0] - stackString = fmt.Sprintf("%+v", stack) - fileName = fmt.Sprintf("%s", bottomFrame) - lineNumberStr := fmt.Sprintf("%d", bottomFrame) - if n, err := strconv.ParseUint(lineNumberStr, 10, 32); err == nil { - lineNumber = uint32(n) - } else { - logrus.WithFields(logrus.Fields{ - "line-number": lineNumberStr, - logrus.ErrorKey: err, - }).Error("opengcs::bridge::setErrorForResponseBase - failed to parse line number, using -1 instead") - } - functionName = fmt.Sprintf("%n", bottomFrame) - } - hresult, err := gcserr.GetHresult(errForResponse) - if err != nil { - // Default to using the generic failure HRESULT. - hresult = gcserr.HrFail - } +func setErrorForResponseBase(response *prot.MessageResponseBase, errForResponse error, moduleName string) { + hresult, errorMessage, newRecord := commonutils.SetErrorForResponseBaseUtil(errForResponse, moduleName) response.Result = int32(hresult) response.ErrorMessage = errorMessage - newRecord := prot.ErrorRecord{ - Result: int32(hresult), - Message: errorMessage, - StackTrace: stackString, - ModuleName: "gcs", - FileName: fileName, - Line: lineNumber, - FunctionName: functionName, - } response.ErrorRecords = append(response.ErrorRecords, newRecord) } diff --git a/internal/guest/bridge/bridge_unit_test.go b/internal/guest/bridge/bridge_unit_test.go index 67f583da05..613eaf326b 100644 --- a/internal/guest/bridge/bridge_unit_test.go +++ b/internal/guest/bridge/bridge_unit_test.go @@ -12,7 +12,7 @@ import ( "sync" "testing" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/transport" "github.com/pkg/errors" diff --git a/internal/guest/bridge/bridge_v2.go b/internal/guest/bridge/bridge_v2.go index f9712abc9d..800094e549 100644 --- a/internal/guest/bridge/bridge_v2.go +++ b/internal/guest/bridge/bridge_v2.go @@ -13,8 +13,8 @@ import ( "go.opencensus.io/trace" "golang.org/x/sys/unix" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/guest/stdio" diff --git a/internal/guest/commonutils/utilities.go b/internal/guest/commonutils/utilities.go deleted file mode 100644 index adcf70e6c2..0000000000 --- a/internal/guest/commonutils/utilities.go +++ /dev/null @@ -1,26 +0,0 @@ -package commonutils - -import ( - "encoding/json" - "io" - - "github.com/Microsoft/hcsshim/internal/guest/gcserr" -) - -// UnmarshalJSONWithHresult unmarshals the given data into the given interface, and -// wraps any error returned in an HRESULT error. -func UnmarshalJSONWithHresult(data []byte, v interface{}) error { - if err := json.Unmarshal(data, v); err != nil { - return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) - } - return nil -} - -// DecodeJSONWithHresult decodes the JSON from the given reader into the given -// interface, and wraps any error returned in an HRESULT error. -func DecodeJSONWithHresult(r io.Reader, v interface{}) error { - if err := json.NewDecoder(r).Decode(v); err != nil { - return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) - } - return nil -} diff --git a/internal/guest/prot/protocol.go b/internal/guest/prot/protocol.go index 891891d510..576ac5e5f1 100644 --- a/internal/guest/prot/protocol.go +++ b/internal/guest/prot/protocol.go @@ -11,7 +11,7 @@ import ( oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" @@ -501,9 +501,9 @@ type ResourceModificationRequestResponse struct { Settings interface{} `json:",omitempty"` } -// ContainerModifySettings is the message from the HCS specifying how a certain +// containerModifySettings is the message from the HCS specifying how a certain // container resource should be modified. -type ContainerModifySettings struct { +type containerModifySettings struct { MessageBase Request interface{} } @@ -512,9 +512,9 @@ type ContainerModifySettings struct { // ContainerModifySettings message. This function is required because properties // such as `Settings` can be of many types identified by the `ResourceType` and // require dynamic unmarshalling. -func UnmarshalContainerModifySettings(b []byte) (*ContainerModifySettings, error) { +func UnmarshalContainerModifySettings(b []byte) (*containerModifySettings, error) { // Unmarshal the message. - var request ContainerModifySettings + var request containerModifySettings var requestRawSettings json.RawMessage request.Request = &requestRawSettings if err := commonutils.UnmarshalJSONWithHresult(b, &request); err != nil { @@ -601,26 +601,13 @@ func UnmarshalContainerModifySettings(b []byte) (*ContainerModifySettings, error return &request, nil } -// ErrorRecord represents a single error to be reported back to the HCS. It -// allows for specifying information about the source of the error, as well as -// an error message and stack trace. -type ErrorRecord struct { - Result int32 - Message string - StackTrace string `json:",omitempty"` - ModuleName string - FileName string - Line uint32 - FunctionName string `json:",omitempty"` -} - // MessageResponseBase is the base type embedded in all messages sent from the // GCS to the HCS except for ContainerNotification. type MessageResponseBase struct { Result int32 - ActivityID string `json:"ActivityId,omitempty"` - ErrorMessage string `json:",omitempty"` // Only used by hcsshim external bridge - ErrorRecords []ErrorRecord `json:",omitempty"` + ActivityID string `json:"ActivityId,omitempty"` + ErrorMessage string `json:",omitempty"` // Only used by hcsshim external bridge + ErrorRecords []commonutils.ErrorRecord `json:",omitempty"` } // Base returns the response base by reference. diff --git a/internal/guest/runtime/hcsv2/container.go b/internal/guest/runtime/hcsv2/container.go index f763f2615a..9627daf645 100644 --- a/internal/guest/runtime/hcsv2/container.go +++ b/internal/guest/runtime/hcsv2/container.go @@ -18,7 +18,7 @@ import ( "github.com/sirupsen/logrus" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime" specGuest "github.com/Microsoft/hcsshim/internal/guest/spec" diff --git a/internal/guest/runtime/hcsv2/network.go b/internal/guest/runtime/hcsv2/network.go index 136f544153..f6052e84f9 100644 --- a/internal/guest/runtime/hcsv2/network.go +++ b/internal/guest/runtime/hcsv2/network.go @@ -14,7 +14,7 @@ import ( "github.com/vishvananda/netns" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/network" "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" diff --git a/internal/guest/runtime/hcsv2/process.go b/internal/guest/runtime/hcsv2/process.go index e29e6e62f7..e94c9792f6 100644 --- a/internal/guest/runtime/hcsv2/process.go +++ b/internal/guest/runtime/hcsv2/process.go @@ -10,7 +10,7 @@ import ( "sync" "syscall" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/runtime" "github.com/Microsoft/hcsshim/internal/guest/stdio" "github.com/Microsoft/hcsshim/internal/log" diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 9e98cefc91..6dddc6f41d 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -22,17 +22,8 @@ import ( "github.com/Microsoft/cosesign1go/pkg/cosesign1" didx509resolver "github.com/Microsoft/didx509go/pkg/did-x509-resolver" - "github.com/Microsoft/hcsshim/pkg/annotations" - "github.com/Microsoft/hcsshim/pkg/securitypolicy" - cgroup1stats "github.com/containerd/cgroups/v3/cgroup1/stats" - "github.com/mattn/go-shellwords" - "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" - + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/debug" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" "github.com/Microsoft/hcsshim/internal/guest/policy" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime" @@ -50,6 +41,14 @@ import ( "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/verity" + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + cgroup1stats "github.com/containerd/cgroups/v3/cgroup1/stats" + "github.com/mattn/go-shellwords" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // UVMContainerID is the ContainerID that will be sent on any prot.MessageBase diff --git a/internal/guest/runtime/runc/runc.go b/internal/guest/runtime/runc/runc.go index 555fd17a7e..cd11cefdda 100644 --- a/internal/guest/runtime/runc/runc.go +++ b/internal/guest/runtime/runc/runc.go @@ -16,7 +16,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" "github.com/Microsoft/hcsshim/internal/guest/runtime" "github.com/Microsoft/hcsshim/internal/guest/stdio" ) diff --git a/internal/guest/runtime/runtime.go b/internal/guest/runtime/runtime.go index a8c5231cfc..db24459c27 100644 --- a/internal/guest/runtime/runtime.go +++ b/internal/guest/runtime/runtime.go @@ -8,7 +8,7 @@ import ( "io" "syscall" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/stdio" oci "github.com/opencontainers/runtime-spec/specs-go" ) diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index 89c2003d7e..b848017c13 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -4,6 +4,7 @@ import ( "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/go-winio/pkg/guid" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) @@ -33,6 +34,10 @@ const ( // ResourceTypeCombinedLayers is the modify resource type for combined // layers ResourceTypeCombinedLayers guestrequest.ResourceType = "CombinedLayers" + // ResourceTypeCWCOWCombinedLayers is the modify resource type for combined + // layers call for cwcow cases. This resource type wraps containerID around + // ResourceTypeCombinedLayers. + ResourceTypeCWCOWCombinedLayers guestrequest.ResourceType = "CWCOWCombinedLayers" // ResourceTypeVPMemDevice is the modify resource type for VPMem devices ResourceTypeVPMemDevice guestrequest.ResourceType = "VPMemDevice" // ResourceTypeVPCIDevice is the modify resource type for vpci devices @@ -46,6 +51,12 @@ const ( ResourceTypeSecurityPolicy guestrequest.ResourceType = "SecurityPolicy" // ResourceTypePolicyFragment is the modify resource type for injecting policy fragments. ResourceTypePolicyFragment guestrequest.ResourceType = "SecurityPolicyFragment" + // ResourceTypeWCOWBlockCims is the modify resource type for mounting block cims for hyperv + // wcow containers. + ResourceTypeWCOWBlockCims guestrequest.ResourceType = "WCOWBlockCims" + // ResourceTypeMappedVirtualDiskForContainerScratch is the modify resource type + // specifically for refs formatting and mounting scratch vhds for c-wcow cases only. + ResourceTypeMappedVirtualDiskForContainerScratch guestrequest.ResourceType = "MappedVirtualDiskForContainerScratch" ) // This class is used by a modify request to add or remove a combined layers @@ -67,6 +78,11 @@ type WCOWCombinedLayers struct { ScratchPath string `json:"ScratchPath,omitempty"` } +type CWCOWCombinedLayers struct { + ContainerID string `json:"ContainerID,omitempty"` + CombinedLayers WCOWCombinedLayers `json:"CombinedLayers,omitempty"` +} + // Defines the schema for hosted settings passed to GCS and/or OpenGCS // SCSIDevice represents a SCSI device that is attached to the system. @@ -92,6 +108,18 @@ type LCOWMappedVirtualDisk struct { Filesystem string `json:"Filesystem,omitempty"` } +type BlockCIMDevice struct { + CimName string + Lun int32 +} + +type WCOWBlockCIMMounts struct { + // BlockCIMs should be ordered from merged CIM followed by Layer n .. layer 1 + BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` + VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` + MountFlags uint32 `json:"MountFlags,omitempty"` +} + type WCOWMappedVirtualDisk struct { ContainerPath string `json:"ContainerPath,omitempty"` Lun int32 `json:"Lun,omitempty"` @@ -207,3 +235,14 @@ type LCOWConfidentialOptions struct { type LCOWSecurityPolicyFragment struct { Fragment string `json:"Fragment,omitempty"` } + +type WCOWConfidentialOptions struct { + EnforcerType string `json:"EnforcerType,omitempty"` + EncodedSecurityPolicy string `json:"EncodedSecurityPolicy,omitempty"` + // Optional security policy + WCOWSecurityPolicy string + // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length + WCOWSecurityPolicyEnabled bool + // Set which security policy enforcer to use (open door or rego). This allows for better fallback mechanic. + WCOWSecurityPolicyEnforcer string +} diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index 1ab71e4e73..1bb9461a14 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -20,7 +20,7 @@ import ( "go.opencensus.io/trace" "github.com/Microsoft/hcsshim/internal/copyfile" - "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -438,7 +438,7 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ // entropyVsockPort - 1 is the entropy port, // linuxLogVsockPort - 109 used by vsockexec to log stdout/stderr logging, // 0x40000000 + 1 (LinuxGcsVsockPort + 1) is the bridge (see guestconnectiuon.go) - hvSockets := []uint32{entropyVsockPort, linuxLogVsockPort, gcs.LinuxGcsVsockPort, gcs.LinuxGcsVsockPort + 1} + hvSockets := []uint32{entropyVsockPort, linuxLogVsockPort, prot.LinuxGcsVsockPort, prot.LinuxGcsVsockPort + 1} hvSockets = append(hvSockets, opts.ExtraVSockPorts...) for _, whichSocket := range hvSockets { key := winio.VsockServiceID(whichSocket).String() @@ -988,7 +988,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error if opts.UseGuestConnection { log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge") - l, err := uvm.listenVsock(gcs.LinuxGcsVsockPort) + l, err := uvm.listenVsock(prot.LinuxGcsVsockPort) if err != nil { return nil, err } diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 0b91e42cf2..2b6f8c8b2d 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -15,7 +15,7 @@ import ( "github.com/sirupsen/logrus" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -57,15 +57,6 @@ type OptionsWCOW struct { AdditionalRegistryKeys []hcsschema.RegistryValue } -// WindowsSidecarGcsHvsockServiceID is the hvsock service ID that the Windows GCS -// sidecar will connect to. This is only used in the confidential mode. -var windowsSidecarGcsHvsockServiceID = guid.GUID{ - Data1: 0xae8da506, - Data2: 0xa019, - Data3: 0x4553, - Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, -} - // NewDefaultOptionsWCOW creates the default options for a bootable version of // WCOW. The caller `MUST` set the `BootFiles` on the returned value. // @@ -502,9 +493,9 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error return nil, fmt.Errorf("error while creating the compute system: %w", err) } - gcsServiceID := gcs.WindowsGcsHvsockServiceID + gcsServiceID := prot.WindowsGcsHvsockServiceID if opts.SecurityPolicyEnabled { - gcsServiceID = windowsSidecarGcsHvsockServiceID + gcsServiceID = prot.WindowsSidecarGcsHvsockServiceID } if err = uvm.startExternalGcsListener(ctx, gcsServiceID); err != nil { diff --git a/internal/uvm/start.go b/internal/uvm/start.go index ec8cdf1ee9..9ab8a9bdd5 100644 --- a/internal/uvm/start.go +++ b/internal/uvm/start.go @@ -18,6 +18,7 @@ import ( "golang.org/x/sys/windows" "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -134,7 +135,7 @@ func (uvm *UtilityVM) configureHvSocketForGCS(ctx context.Context) (err error) { hvsocketAddress := &hcsschema.HvSocketAddress{ LocalAddress: uvm.runtimeID.String(), - ParentAddress: gcs.WindowsGcsHvHostID.String(), + ParentAddress: prot.WindowsGcsHvHostID.String(), } conSetupReq := &hcsschema.ModifySettingRequest{ diff --git a/pkg/securitypolicy/securitypolicy_uvmpath_linux.go b/pkg/securitypolicy/securitypolicy_uvmpath_linux.go new file mode 100644 index 0000000000..4fdfaafdcc --- /dev/null +++ b/pkg/securitypolicy/securitypolicy_uvmpath_linux.go @@ -0,0 +1,34 @@ +//go:build linux +// +build linux + +package securitypolicy + +import ( + "strings" + + specInternal "github.com/Microsoft/hcsshim/internal/guest/spec" + "github.com/Microsoft/hcsshim/internal/guestpath" +) + +// This is being used by StandEnforcer. +// substituteUVMPath substitutes mount prefix to an appropriate path inside +// UVM. At policy generation time, it's impossible to tell what the sandboxID +// will be, so the prefix substitution needs to happen during runtime. +func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { + if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { + m.Source = specInternal.SandboxMountSource(sandboxID, m.Source) + } else if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { + m.Source = specInternal.HugePagesMountSource(sandboxID, m.Source) + } + return m +} + +// SandboxMountsDir returns sandbox mounts directory inside UVM/host. +func SandboxMountsDir(sandboxID string) string { + return specInternal.SandboxMountsDir((sandboxID)) +} + +// HugePagesMountsDir returns hugepages mounts directory inside UVM. +func HugePagesMountsDir(sandboxID string) string { + return specInternal.HugePagesMountsDir(sandboxID) +} diff --git a/pkg/securitypolicy/securitypolicy_uvmpath_windows.go b/pkg/securitypolicy/securitypolicy_uvmpath_windows.go new file mode 100644 index 0000000000..cc6949fa68 --- /dev/null +++ b/pkg/securitypolicy/securitypolicy_uvmpath_windows.go @@ -0,0 +1,24 @@ +//go:build windows +// +build windows + +package securitypolicy + +// This is being used by StandEnforcer and is a no-op for windows. +// substituteUVMPath substitutes mount prefix to an appropriate path inside +// UVM. At policy generation time, it's impossible to tell what the sandboxID +// will be, so the prefix substitution needs to happen during runtime. +func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { + //no-op for windows + _ = sandboxID + return m +} + +// SandboxMountsDir returns sandbox mounts directory inside UVM/host. +func SandboxMountsDir(sandboxID string) string { + return "" +} + +// HugePagesMountsDir returns hugepages mounts directory inside UVM. +func HugePagesMountsDir(sandboxID string) string { + return "" +} diff --git a/pkg/securitypolicy/securitypolicyenforcer.go b/pkg/securitypolicy/securitypolicyenforcer.go index 5806eb7544..8792966229 100644 --- a/pkg/securitypolicy/securitypolicyenforcer.go +++ b/pkg/securitypolicy/securitypolicyenforcer.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - package securitypolicy import ( @@ -10,15 +7,11 @@ import ( "fmt" "regexp" "strconv" - "strings" "sync" "syscall" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - - specGuest "github.com/Microsoft/hcsshim/internal/guest/spec" - "github.com/Microsoft/hcsshim/internal/guestpath" ) type createEnforcerFunc func(base64EncodedPolicy string, criMounts, criPrivilegedMounts []oci.Mount, maxErrorMessageLength int) (SecurityPolicyEnforcer, error) @@ -89,12 +82,15 @@ type SecurityPolicyEnforcer interface { GetUserInfo(containerID string, spec *oci.Process) (IDName, []IDName, string, error) } +//nolint type stringSet map[string]struct{} +//nolint func (s stringSet) add(item string) { s[item] = struct{}{} } +//nolint func (s stringSet) contains(item string) bool { _, contains := s[item] return contains @@ -856,18 +852,6 @@ func (c *securityPolicyContainer) matchMount(sandboxID string, m oci.Mount) (err return fmt.Errorf("mount is not allowed by policy: %+v", m) } -// substituteUVMPath substitutes mount prefix to an appropriate path inside -// UVM. At policy generation time, it's impossible to tell what the sandboxID -// will be, so the prefix substitution needs to happen during runtime. -func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { - if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { - m.Source = specGuest.SandboxMountSource(sandboxID, m.Source) - } else if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { - m.Source = specGuest.HugePagesMountSource(sandboxID, m.Source) - } - return m -} - func stringSlicesEqual(slice1, slice2 []string) bool { if len(slice1) != len(slice2) { return false diff --git a/test/gcs/container_test.go b/test/gcs/container_test.go index caf6ea2f77..4f3db015c0 100644 --- a/test/gcs/container_test.go +++ b/test/gcs/container_test.go @@ -12,7 +12,7 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "golang.org/x/sync/errgroup" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/stdio" testoci "github.com/Microsoft/hcsshim/test/internal/oci"