From bac985dad6006ab7b4dba86992097e117114f25e Mon Sep 17 00:00:00 2001 From: Kirtana Ashok Date: Mon, 21 Apr 2025 12:41:37 -0700 Subject: [PATCH 1/2] Refactor common bridge protocol code for reuse - Move common bridge protocol definitions to subpackage under internal/gcs - Move helper functions to internal/bridgeutils pkg so that they can be used by gcs-sidecar as well Signed-off-by: Kirtana Ashok --- internal/bridgeutils/commonutils/utilities.go | 83 ++++++ .../{guest => bridgeutils}/gcserr/errors.go | 0 internal/gcs/bridge.go | 35 +-- internal/gcs/bridge_test.go | 31 +- internal/gcs/container.go | 53 ++-- internal/gcs/guestconnection.go | 57 ++-- internal/gcs/guestconnection_test.go | 31 +- internal/gcs/process.go | 41 +-- internal/gcs/{ => prot}/protocol.go | 265 +++++++++--------- internal/guest/bridge/bridge.go | 47 +--- internal/guest/bridge/bridge_unit_test.go | 2 +- internal/guest/bridge/bridge_v2.go | 4 +- internal/guest/commonutils/utilities.go | 26 -- internal/guest/prot/protocol.go | 21 +- internal/guest/runtime/hcsv2/container.go | 2 +- internal/guest/runtime/hcsv2/network.go | 2 +- internal/guest/runtime/hcsv2/process.go | 2 +- internal/guest/runtime/hcsv2/uvm.go | 3 +- internal/guest/runtime/runc/runc.go | 2 +- internal/guest/runtime/runtime.go | 2 +- internal/uvm/create_lcow.go | 6 +- internal/uvm/create_wcow.go | 15 +- internal/uvm/start.go | 3 +- 23 files changed, 364 insertions(+), 369 deletions(-) create mode 100644 internal/bridgeutils/commonutils/utilities.go rename internal/{guest => bridgeutils}/gcserr/errors.go (100%) rename internal/gcs/{ => prot}/protocol.go (51%) delete mode 100644 internal/guest/commonutils/utilities.go diff --git a/internal/bridgeutils/commonutils/utilities.go b/internal/bridgeutils/commonutils/utilities.go new file mode 100644 index 0000000000..4409825ad1 --- /dev/null +++ b/internal/bridgeutils/commonutils/utilities.go @@ -0,0 +1,83 @@ +package commonutils + +import ( + "encoding/json" + "fmt" + "io" + "math" + "strconv" + + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" + "github.com/sirupsen/logrus" +) + +type ErrorRecord struct { + Result int32 // HResult + Message string + StackTrace string `json:",omitempty"` + ModuleName string + FileName string + Line uint32 + FunctionName string `json:",omitempty"` +} + +// UnmarshalJSONWithHresult unmarshals the given data into the given interface, and +// wraps any error returned in an HRESULT error. +func UnmarshalJSONWithHresult(data []byte, v interface{}) error { + if err := json.Unmarshal(data, v); err != nil { + return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) + } + return nil +} + +// DecodeJSONWithHresult decodes the JSON from the given reader into the given +// interface, and wraps any error returned in an HRESULT error. +func DecodeJSONWithHresult(r io.Reader, v interface{}) error { + if err := json.NewDecoder(r).Decode(v); err != nil { + return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) + } + return nil +} + +func SetErrorForResponseBaseUtil(errForResponse error, moduleName string) (hresult gcserr.Hresult, errorMessage string, newRecord ErrorRecord) { + errorMessage = errForResponse.Error() + stackString := "" + fileName := "" + // We use -1 as a sentinel if no line number found (or it cannot be parsed), + // but that will ultimately end up as [math.MaxUint32], so set it to that explicitly. + // (Still keep using -1 for backwards compatibility ...) + lineNumber := uint32(math.MaxUint32) + functionName := "" + if stack := gcserr.BaseStackTrace(errForResponse); stack != nil { + bottomFrame := stack[0] + stackString = fmt.Sprintf("%+v", stack) + fileName = fmt.Sprintf("%s", bottomFrame) + lineNumberStr := fmt.Sprintf("%d", bottomFrame) + if n, err := strconv.ParseUint(lineNumberStr, 10, 32); err == nil { + lineNumber = uint32(n) + } else { + logrus.WithFields(logrus.Fields{ + "line-number": lineNumberStr, + logrus.ErrorKey: err, + }).Error("opengcs::bridge::setErrorForResponseBase - failed to parse line number, using -1 instead") + } + functionName = fmt.Sprintf("%n", bottomFrame) + } + hresult, err := gcserr.GetHresult(errForResponse) + if err != nil { + // Default to using the generic failure HRESULT. + hresult = gcserr.HrFail + } + + newRecord = ErrorRecord{ + Result: int32(hresult), + Message: errorMessage, + StackTrace: stackString, + ModuleName: moduleName, + FileName: fileName, + Line: lineNumber, + FunctionName: functionName, + } + + return hresult, errorMessage, newRecord +} diff --git a/internal/guest/gcserr/errors.go b/internal/bridgeutils/gcserr/errors.go similarity index 100% rename from internal/guest/gcserr/errors.go rename to internal/bridgeutils/gcserr/errors.go diff --git a/internal/gcs/bridge.go b/internal/gcs/bridge.go index 17e54f8242..0ed38a4896 100644 --- a/internal/gcs/bridge.go +++ b/internal/gcs/bridge.go @@ -19,6 +19,7 @@ import ( "go.opencensus.io/trace" "golang.org/x/sys/windows" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" ) @@ -38,16 +39,16 @@ const ( ) type requestMessage interface { - Base() *requestBase + Base() *prot.RequestBase } type responseMessage interface { - Base() *responseBase + Base() *prot.ResponseBase } // rpc represents an outstanding rpc request to the guest type rpc struct { - proc rpcProc + proc prot.RpcProc id int64 req requestMessage resp responseMessage @@ -80,7 +81,7 @@ const ( bridgeFailureTimeout = time.Minute * 5 ) -type notifyFunc func(*containerNotification) error +type notifyFunc func(*prot.ContainerNotification) error // newBridge returns a bridge on `conn`. It calls `notify` when a // notification message arrives from the guest. It logs transport errors and @@ -143,7 +144,7 @@ func (brdg *bridge) Wait() error { // AsyncRPC sends an RPC request to the guest but does not wait for a response. // If the message cannot be sent before the context is done, then an error is // returned. -func (brdg *bridge) AsyncRPC(ctx context.Context, proc rpcProc, req requestMessage, resp responseMessage) (*rpc, error) { +func (brdg *bridge) AsyncRPC(ctx context.Context, proc prot.RpcProc, req requestMessage, resp responseMessage) (*rpc, error) { call := &rpc{ ch: make(chan struct{}), proc: proc, @@ -224,7 +225,7 @@ func (call *rpc) Wait() { // If allowCancel is set and the context becomes done, returns an error without // waiting for a response. Avoid this on messages that are not idempotent or // otherwise safe to ignore the response of. -func (brdg *bridge) RPC(ctx context.Context, proc rpcProc, req requestMessage, resp responseMessage, allowCancel bool) error { +func (brdg *bridge) RPC(ctx context.Context, proc prot.RpcProc, req requestMessage, resp responseMessage, allowCancel bool) error { call, err := brdg.AsyncRPC(ctx, proc, req, resp) if err != nil { return err @@ -261,7 +262,7 @@ func (brdg *bridge) recvLoopRoutine() { } } -func readMessage(r io.Reader) (int64, msgType, []byte, error) { +func readMessage(r io.Reader) (int64, prot.MsgType, []byte, error) { _, span := oc.StartSpan(context.Background(), "bridge receive read message", oc.WithClientSpanKind) defer span.End() @@ -270,7 +271,7 @@ func readMessage(r io.Reader) (int64, msgType, []byte, error) { if err != nil { return 0, 0, nil, fmt.Errorf("header read: %w", err) } - typ := msgType(binary.LittleEndian.Uint32(h[hdrOffType:])) + typ := prot.MsgType(binary.LittleEndian.Uint32(h[hdrOffType:])) n := binary.LittleEndian.Uint32(h[hdrOffSize:]) id := int64(binary.LittleEndian.Uint64(h[hdrOffID:])) span.AddAttributes( @@ -311,8 +312,8 @@ func (brdg *bridge) recvLoop() error { "type": typ.String(), "message-id": id}).Trace("bridge receive") - switch typ & msgTypeMask { - case msgTypeResponse: + switch typ & prot.MsgTypeMask { + case prot.MsgTypeResponse: // Find the request associated with this response. brdg.mu.Lock() call := brdg.rpcs[id] @@ -344,11 +345,11 @@ func (brdg *bridge) recvLoop() error { return err } - case msgTypeNotify: - if typ != notifyContainer|msgTypeNotify { + case prot.MsgTypeNotify: + if typ != prot.NotifyContainer|prot.MsgTypeNotify { return fmt.Errorf("bridge received unknown unknown notification message %s", typ) } - var ntf containerNotification + var ntf prot.ContainerNotification ntf.ResultInfo.Value = &json.RawMessage{} err := json.Unmarshal(b, &ntf) if err != nil { @@ -383,7 +384,7 @@ func (brdg *bridge) sendLoop() { } } -func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ msgType, id int64, req interface{}) error { +func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ prot.MsgType, id int64, req interface{}) error { var err error _, span := oc.StartSpan(context.Background(), "bridge send", oc.WithClientSpanKind) defer span.End() @@ -408,9 +409,9 @@ func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ msgTy b := buf.Bytes()[hdrSize:] switch typ { // container environment vars are in rpCreate for linux; rpcExecuteProcess for windows - case msgType(rpcCreate) | msgTypeRequest: + case prot.MsgType(prot.RpcCreate) | prot.MsgTypeRequest: b, err = log.ScrubBridgeCreate(b) - case msgType(rpcExecuteProcess) | msgTypeRequest: + case prot.MsgType(prot.RpcExecuteProcess) | prot.MsgTypeRequest: b, err = log.ScrubBridgeExecProcess(b) } if err != nil { @@ -443,7 +444,7 @@ func (brdg *bridge) sendRPC(buf *bytes.Buffer, enc *json.Encoder, call *rpc) err brdg.rpcs[id] = call brdg.nextID++ brdg.mu.Unlock() - typ := msgType(call.proc) | msgTypeRequest + typ := prot.MsgType(call.proc) | prot.MsgTypeRequest err := brdg.writeMessage(buf, enc, typ, id, call.req) if err != nil { // Try to reclaim this request and fail it. diff --git a/internal/gcs/bridge_test.go b/internal/gcs/bridge_test.go index d6b3265c60..5a6ab368f1 100644 --- a/internal/gcs/bridge_test.go +++ b/internal/gcs/bridge_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/sirupsen/logrus" ) @@ -33,7 +34,7 @@ func pipeConn() (*stitched, *stitched) { return &stitched{r1, w2}, &stitched{r2, w1} } -func sendMessage(t *testing.T, w io.Writer, typ msgType, id int64, msg []byte) { +func sendMessage(t *testing.T, w io.Writer, typ prot.MsgType, id int64, msg []byte) { t.Helper() var h [16]byte binary.LittleEndian.PutUint32(h[:], uint32(typ)) @@ -63,18 +64,18 @@ func reflector(t *testing.T, rw io.ReadWriteCloser, delay time.Duration) { return } time.Sleep(delay) // delay is used to test timeouts (when non-zero) - typ ^= msgTypeResponse ^ msgTypeRequest + typ ^= prot.MsgTypeResponse ^ prot.MsgTypeRequest sendMessage(t, rw, typ, id, msg) } } type testReq struct { - requestBase + prot.RequestBase X, Y int } type testResp struct { - responseBase + prot.ResponseBase X, Y int } @@ -92,7 +93,7 @@ func TestBridgeRPC(t *testing.T) { defer b.Close() req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), rpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RpcCreate, &req, &resp, false) if err != nil { t.Fatal(err) } @@ -107,7 +108,7 @@ func TestBridgeRPCResponseTimeout(t *testing.T) { b.Timeout = time.Millisecond * 100 req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), rpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RpcCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -121,7 +122,7 @@ func TestBridgeRPCContextDone(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, rpcCreate, &req, &resp, true) + err := b.RPC(ctx, prot.RpcCreate, &req, &resp, true) if err != context.DeadlineExceeded { //nolint:errorlint t.Fatalf("expected deadline exceeded, got %s", err) } @@ -135,7 +136,7 @@ func TestBridgeRPCContextDoneNoCancel(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, rpcCreate, &req, &resp, false) + err := b.RPC(ctx, prot.RpcCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -145,13 +146,13 @@ func TestBridgeRPCBridgeClosed(t *testing.T) { b := startReflectedBridge(t, 0) eerr := errors.New("forcibly terminated") b.kill(eerr) - err := b.RPC(context.Background(), rpcCreate, nil, nil, false) + err := b.RPC(context.Background(), prot.RpcCreate, nil, nil, false) if err != eerr { //nolint:errorlint t.Fatal("unexpected: ", err) } } -func sendJSON(t *testing.T, w io.Writer, typ msgType, id int64, msg interface{}) error { +func sendJSON(t *testing.T, w io.Writer, typ prot.MsgType, id int64, msg interface{}) error { t.Helper() msgb, err := json.Marshal(msg) if err != nil { @@ -161,7 +162,7 @@ func sendJSON(t *testing.T, w io.Writer, typ msgType, id int64, msg interface{}) return nil } -func notifyThroughBridge(t *testing.T, typ msgType, msg interface{}, fn notifyFunc) error { +func notifyThroughBridge(t *testing.T, typ prot.MsgType, msg interface{}, fn notifyFunc) error { t.Helper() s, c := pipeConn() b := newBridge(s, fn, logrus.NewEntry(logrus.StandardLogger())) @@ -176,9 +177,9 @@ func notifyThroughBridge(t *testing.T, typ msgType, msg interface{}, fn notifyFu } func TestBridgeNotify(t *testing.T) { - ntf := &containerNotification{Operation: "testing"} + ntf := &prot.ContainerNotification{Operation: "testing"} recvd := false - err := notifyThroughBridge(t, msgTypeNotify|notifyContainer, ntf, func(nntf *containerNotification) error { + err := notifyThroughBridge(t, prot.MsgTypeNotify|prot.NotifyContainer, ntf, func(nntf *prot.ContainerNotification) error { if !reflect.DeepEqual(ntf, nntf) { t.Errorf("%+v != %+v", ntf, nntf) } @@ -194,9 +195,9 @@ func TestBridgeNotify(t *testing.T) { } func TestBridgeNotifyFailure(t *testing.T) { - ntf := &containerNotification{Operation: "testing"} + ntf := &prot.ContainerNotification{Operation: "testing"} errMsg := "notify should have failed" - err := notifyThroughBridge(t, msgTypeNotify|notifyContainer, ntf, func(nntf *containerNotification) error { + err := notifyThroughBridge(t, prot.MsgTypeNotify|prot.NotifyContainer, ntf, func(nntf *prot.ContainerNotification) error { return errors.New(errMsg) }) if err == nil || !strings.Contains(err.Error(), errMsg) { diff --git a/internal/gcs/container.go b/internal/gcs/container.go index a64408b834..728f38a43a 100644 --- a/internal/gcs/container.go +++ b/internal/gcs/container.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -53,12 +54,12 @@ func (gc *GuestConnection) CreateContainer(ctx context.Context, cid string, conf if err != nil { return nil, err } - req := containerCreate{ - requestBase: makeRequest(ctx, cid), - ContainerConfig: anyInString{config}, + req := prot.ContainerCreate{ + RequestBase: makeRequest(ctx, cid), + ContainerConfig: prot.AnyInString{config}, } - var resp containerCreateResponse - err = gc.brdg.RPC(ctx, rpcCreate, &req, &resp, false) + var resp prot.ContainerCreateResponse + err = gc.brdg.RPC(ctx, prot.RpcCreate, &req, &resp, false) if err != nil { return nil, err } @@ -129,27 +130,27 @@ func (c *Container) Modify(ctx context.Context, config interface{}) (err error) defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerModifySettings{ - requestBase: makeRequest(ctx, c.id), + req := prot.ContainerModifySettings{ + RequestBase: makeRequest(ctx, c.id), Request: config, } - var resp responseBase - return c.gc.brdg.RPC(ctx, rpcModifySettings, &req, &resp, false) + var resp prot.ResponseBase + return c.gc.brdg.RPC(ctx, prot.RpcModifySettings, &req, &resp, false) } -// Properties returns the requested container properties targeting a V1 schema container. +// Properties returns the requested container properties targeting a V1 schema prot.Container. func (c *Container) Properties(ctx context.Context, types ...schema1.PropertyType) (_ *schema1.ContainerProperties, err error) { ctx, span := oc.StartSpan(ctx, "gcs::Container::Properties", oc.WithClientSpanKind) defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerGetProperties{ - requestBase: makeRequest(ctx, c.id), - Query: containerPropertiesQuery{PropertyTypes: types}, + req := prot.ContainerGetProperties{ + RequestBase: makeRequest(ctx, c.id), + Query: prot.ContainerPropertiesQuery{PropertyTypes: types}, } - var resp containerGetPropertiesResponse - err = c.gc.brdg.RPC(ctx, rpcGetProperties, &req, &resp, true) + var resp prot.ContainerGetPropertiesResponse + err = c.gc.brdg.RPC(ctx, prot.RpcGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -163,12 +164,12 @@ func (c *Container) PropertiesV2(ctx context.Context, types ...hcsschema.Propert defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", c.id)) - req := containerGetPropertiesV2{ - requestBase: makeRequest(ctx, c.id), - Query: containerPropertiesQueryV2{PropertyTypes: types}, + req := prot.ContainerGetPropertiesV2{ + RequestBase: makeRequest(ctx, c.id), + Query: prot.ContainerPropertiesQueryV2{PropertyTypes: types}, } - var resp containerGetPropertiesResponseV2 - err = c.gc.brdg.RPC(ctx, rpcGetProperties, &req, &resp, true) + var resp prot.ContainerGetPropertiesResponseV2 + err = c.gc.brdg.RPC(ctx, prot.RpcGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -183,13 +184,13 @@ func (c *Container) Start(ctx context.Context) (err error) { span.AddAttributes(trace.StringAttribute("cid", c.id)) req := makeRequest(ctx, c.id) - var resp responseBase - return c.gc.brdg.RPC(ctx, rpcStart, &req, &resp, false) + var resp prot.ResponseBase + return c.gc.brdg.RPC(ctx, prot.RpcStart, &req, &resp, false) } -func (c *Container) shutdown(ctx context.Context, proc rpcProc) error { +func (c *Container) shutdown(ctx context.Context, proc prot.RpcProc) error { req := makeRequest(ctx, c.id) - var resp responseBase + var resp prot.ResponseBase err := c.gc.brdg.RPC(ctx, proc, &req, &resp, true) if err != nil { if uint32(resp.Result) != hrComputeSystemDoesNotExist { @@ -215,7 +216,7 @@ func (c *Container) Shutdown(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, rpcShutdownGraceful) + return c.shutdown(ctx, prot.RpcShutdownGraceful) } // Terminate sends a forceful terminate request to the container. The container @@ -229,7 +230,7 @@ func (c *Container) Terminate(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, rpcShutdownForced) + return c.shutdown(ctx, prot.RpcShutdownForced) } func (c *Container) WaitChannel() <-chan struct{} { diff --git a/internal/gcs/guestconnection.go b/internal/gcs/guestconnection.go index fe974b5c17..9607c61261 100644 --- a/internal/gcs/guestconnection.go +++ b/internal/gcs/guestconnection.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/go-winio/pkg/guid" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -28,7 +29,7 @@ import ( const ( protocolVersion = 4 - firstIoChannelVsockPort = LinuxGcsVsockPort + 1 + firstIoChannelVsockPort = prot.LinuxGcsVsockPort + 1 nullContainerID = "00000000-0000-0000-0000-000000000000" ) @@ -117,12 +118,12 @@ func (gc *GuestConnection) Protocol() uint32 { // isColdStart should be true when the UVM is being connected to for the first time post-boot. // It should be false for subsequent connections (e.g. if reconnecting to an existing UVM). func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGuestState *InitialGuestState) (err error) { - req := negotiateProtocolRequest{ + req := prot.NegotiateProtocolRequest{ MinimumVersion: protocolVersion, MaximumVersion: protocolVersion, } - var resp negotiateProtocolResponse - err = gc.brdg.RPC(ctx, rpcNegotiateProtocol, &req, &resp, true) + var resp prot.NegotiateProtocolResponse + err = gc.brdg.RPC(ctx, prot.RpcNegotiateProtocol, &req, &resp, true) if err != nil { return err } @@ -141,25 +142,25 @@ func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGu } if isColdStart && resp.Capabilities.SendHostCreateMessage { - conf := &uvmConfig{ + conf := &prot.UvmConfig{ SystemType: "Container", } if initGuestState != nil && initGuestState.Timezone != nil { conf.TimeZoneInformation = initGuestState.Timezone } - createReq := containerCreate{ - requestBase: makeRequest(ctx, nullContainerID), - ContainerConfig: anyInString{conf}, + createReq := prot.ContainerCreate{ + RequestBase: makeRequest(ctx, nullContainerID), + ContainerConfig: prot.AnyInString{conf}, } - var createResp responseBase - err = gc.brdg.RPC(ctx, rpcCreate, &createReq, &createResp, true) + var createResp prot.ResponseBase + err = gc.brdg.RPC(ctx, prot.RpcCreate, &createReq, &createResp, true) if err != nil { return err } if resp.Capabilities.SendHostStartMessage { startReq := makeRequest(ctx, nullContainerID) - var startResp responseBase - err = gc.brdg.RPC(ctx, rpcStart, &startReq, &startResp, true) + var startResp prot.ResponseBase + err = gc.brdg.RPC(ctx, prot.RpcStart, &startReq, &startResp, true) if err != nil { return err } @@ -175,12 +176,12 @@ func (gc *GuestConnection) Modify(ctx context.Context, settings interface{}) (er defer span.End() defer func() { oc.SetSpanStatus(span, err) }() - req := containerModifySettings{ - requestBase: makeRequest(ctx, nullContainerID), + req := prot.ContainerModifySettings{ + RequestBase: makeRequest(ctx, nullContainerID), Request: settings, } - var resp responseBase - return gc.brdg.RPC(ctx, rpcModifySettings, &req, &resp, false) + var resp prot.ResponseBase + return gc.brdg.RPC(ctx, prot.RpcModifySettings, &req, &resp, false) } func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err error) { @@ -188,11 +189,11 @@ func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err defer span.End() defer func() { oc.SetSpanStatus(span, err) }() - req := dumpStacksRequest{ - requestBase: makeRequest(ctx, nullContainerID), + req := prot.DumpStacksRequest{ + RequestBase: makeRequest(ctx, nullContainerID), } - var resp dumpStacksResponse - err = gc.brdg.RPC(ctx, rpcDumpStacks, &req, &resp, false) + var resp prot.DumpStacksResponse + err = gc.brdg.RPC(ctx, prot.RpcDumpStacks, &req, &resp, false) return resp.GuestStacks, err } @@ -202,11 +203,11 @@ func (gc *GuestConnection) DeleteContainerState(ctx context.Context, cid string) defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("cid", cid)) - req := deleteContainerStateRequest{ - requestBase: makeRequest(ctx, cid), + req := prot.DeleteContainerStateRequest{ + RequestBase: makeRequest(ctx, cid), } - var resp responseBase - return gc.brdg.RPC(ctx, rpcDeleteContainerState, &req, &resp, false) + var resp prot.ResponseBase + return gc.brdg.RPC(ctx, prot.RpcDeleteContainerState, &req, &resp, false) } // Close terminates the guest connection. It is undefined to call any other @@ -263,7 +264,7 @@ func (gc *GuestConnection) requestNotify(cid string, ch chan struct{}) error { return nil } -func (gc *GuestConnection) notify(ntf *containerNotification) error { +func (gc *GuestConnection) notify(ntf *prot.ContainerNotification) error { cid := ntf.ContainerID gc.mu.Lock() ch := gc.notifyChs[cid] @@ -287,14 +288,14 @@ func (gc *GuestConnection) clearNotifies() { } } -func makeRequest(ctx context.Context, cid string) requestBase { - r := requestBase{ +func makeRequest(ctx context.Context, cid string) prot.RequestBase { + r := prot.RequestBase{ ContainerID: cid, } span := trace.FromContext(ctx) if span != nil { sc := span.SpanContext() - r.OpenCensusSpanContext = &ocspancontext{ + r.OpenCensusSpanContext = &prot.Ocspancontext{ TraceID: hex.EncodeToString(sc.TraceID[:]), SpanID: hex.EncodeToString(sc.SpanID[:]), TraceOptions: uint32(sc.TraceOptions), diff --git a/internal/gcs/guestconnection_test.go b/internal/gcs/guestconnection_test.go index facb0dd34b..8c505be54b 100644 --- a/internal/gcs/guestconnection_test.go +++ b/internal/gcs/guestconnection_test.go @@ -21,6 +21,7 @@ import ( "go.opencensus.io/trace" "go.opencensus.io/trace/tracestate" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/oc" ) @@ -55,24 +56,24 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { } return err } - switch proc := rpcProc(typ &^ msgTypeRequest); proc { - case rpcNegotiateProtocol: - err := sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &negotiateProtocolResponse{ + switch proc := prot.RpcProc(typ &^ prot.MsgTypeRequest); proc { + case prot.RpcNegotiateProtocol: + err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.NegotiateProtocolResponse{ Version: protocolVersion, - Capabilities: gcsCapabilities{ + Capabilities: prot.GcsCapabilities{ RuntimeOsType: "linux", }, }) if err != nil { return err } - case rpcCreate: - err := sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &containerCreateResponse{}) + case prot.RpcCreate: + err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ContainerCreateResponse{}) if err != nil { return err } - case rpcExecuteProcess: - var req containerExecuteProcess + case prot.RpcExecuteProcess: + var req prot.ContainerExecuteProcess var params baseProcessParams req.Settings.ProcessParameters.Value = ¶ms err := json.Unmarshal(b, &req) @@ -111,27 +112,27 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { stdout.Close() }() } - err = sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &containerExecuteProcessResponse{ + err = sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ContainerExecuteProcessResponse{ ProcessID: 42, }) if err != nil { return err } - case rpcWaitForProcess: + case prot.RpcWaitForProcess: // nothing - case rpcShutdownForced: - var req requestBase + case prot.RpcShutdownForced: + var req prot.RequestBase err = json.Unmarshal(b, &req) if err != nil { return err } - err = sendJSON(t, rw, msgTypeResponse|msgType(proc), id, &responseBase{}) + err = sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ResponseBase{}) if err != nil { return err } time.Sleep(50 * time.Millisecond) - err = sendJSON(t, rw, msgType(msgTypeNotify|notifyContainer), 0, &containerNotification{ - requestBase: requestBase{ + err = sendJSON(t, rw, prot.MsgType(prot.MsgTypeNotify|prot.NotifyContainer), 0, &prot.ContainerNotification{ + RequestBase: prot.RequestBase{ ContainerID: req.ContainerID, }, }) diff --git a/internal/gcs/process.go b/internal/gcs/process.go index 87c5c29ae4..f5f013cd57 100644 --- a/internal/gcs/process.go +++ b/internal/gcs/process.go @@ -12,6 +12,7 @@ import ( "github.com/Microsoft/go-winio" "github.com/Microsoft/hcsshim/internal/cow" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/oc" @@ -29,7 +30,7 @@ type Process struct { cid string id uint32 waitCall *rpc - waitResp containerWaitForProcessResponse + waitResp prot.ContainerWaitForProcessResponse stdin, stdout, stderr *ioChannel stdinCloseWriteOnce sync.Once stdinCloseWriteErr error @@ -52,10 +53,10 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac return nil, err } - req := containerExecuteProcess{ - requestBase: makeRequest(ctx, cid), - Settings: executeProcessSettings{ - ProcessParameters: anyInString{params}, + req := prot.ContainerExecuteProcess{ + RequestBase: makeRequest(ctx, cid), + Settings: prot.ExecuteProcessSettings{ + ProcessParameters: prot.AnyInString{params}, }, } @@ -68,8 +69,8 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac // Construct the stdio channels. Windows guests expect hvsock service IDs // instead of vsock ports. - var hvsockSettings executeProcessStdioRelaySettings - var vsockSettings executeProcessVsockStdioRelaySettings + var hvsockSettings prot.ExecuteProcessStdioRelaySettings + var vsockSettings prot.ExecuteProcessVsockStdioRelaySettings if gc.os == "windows" { req.Settings.StdioRelaySettings = &hvsockSettings } else { @@ -100,20 +101,20 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac hvsockSettings.StdErr = &g } - var resp containerExecuteProcessResponse - err = gc.brdg.RPC(ctx, rpcExecuteProcess, &req, &resp, false) + var resp prot.ContainerExecuteProcessResponse + err = gc.brdg.RPC(ctx, prot.RpcExecuteProcess, &req, &resp, false) if err != nil { return nil, err } p.id = resp.ProcessID log.G(ctx).WithField("pid", p.id).Debug("created process pid") // Start a wait message. - waitReq := containerWaitForProcess{ - requestBase: makeRequest(ctx, cid), + waitReq := prot.ContainerWaitForProcess{ + RequestBase: makeRequest(ctx, cid), ProcessID: p.id, TimeoutInMs: 0xffffffff, } - p.waitCall, err = gc.brdg.AsyncRPC(ctx, rpcWaitForProcess, &waitReq, &p.waitResp) + p.waitCall, err = gc.brdg.AsyncRPC(ctx, prot.RpcWaitForProcess, &waitReq, &p.waitResp) if err != nil { return nil, fmt.Errorf("failed to wait on process, leaking process: %w", err) } @@ -220,14 +221,14 @@ func (p *Process) ResizeConsole(ctx context.Context, width, height uint16) (err trace.StringAttribute("cid", p.cid), trace.Int64Attribute("pid", int64(p.id))) - req := containerResizeConsole{ - requestBase: makeRequest(ctx, p.cid), + req := prot.ContainerResizeConsole{ + RequestBase: makeRequest(ctx, p.cid), ProcessID: p.id, Height: height, Width: width, } - var resp responseBase - return p.gc.brdg.RPC(ctx, rpcResizeConsole, &req, &resp, true) + var resp prot.ResponseBase + return p.gc.brdg.RPC(ctx, prot.RpcResizeConsole, &req, &resp, true) } // Signal sends a signal to the process, returning whether it was delivered. @@ -239,15 +240,15 @@ func (p *Process) Signal(ctx context.Context, options interface{}) (_ bool, err trace.StringAttribute("cid", p.cid), trace.Int64Attribute("pid", int64(p.id))) - req := containerSignalProcess{ - requestBase: makeRequest(ctx, p.cid), + req := prot.ContainerSignalProcess{ + RequestBase: makeRequest(ctx, p.cid), ProcessID: p.id, Options: options, } - var resp responseBase + var resp prot.ResponseBase // FUTURE: SIGKILL is idempotent and can safely be cancelled, but this interface // does currently make it easy to determine what signal is being sent. - err = p.gc.brdg.RPC(ctx, rpcSignalProcess, &req, &resp, false) + err = p.gc.brdg.RPC(ctx, prot.RpcSignalProcess, &req, &resp, false) if err != nil { if uint32(resp.Result) != hrNotFound { return false, err diff --git a/internal/gcs/protocol.go b/internal/gcs/prot/protocol.go similarity index 51% rename from internal/gcs/protocol.go rename to internal/gcs/prot/protocol.go index 7aeeb4991f..0555d71c5f 100644 --- a/internal/gcs/protocol.go +++ b/internal/gcs/prot/protocol.go @@ -1,6 +1,6 @@ //go:build windows -package gcs +package prot import ( "encoding/json" @@ -8,6 +8,7 @@ import ( "strconv" "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) @@ -33,97 +34,97 @@ var WindowsGcsHvHostID = guid.GUID{ Data4: [8]uint8{0x93, 0xfe, 0x42, 0x96, 0x9a, 0xe6, 0xd8, 0xd1}, } -type anyInString struct { +type AnyInString struct { Value interface{} } -func (a *anyInString) MarshalText() ([]byte, error) { +func (a *AnyInString) MarshalText() ([]byte, error) { return json.Marshal(a.Value) } -func (a *anyInString) UnmarshalText(b []byte) error { +func (a *AnyInString) UnmarshalText(b []byte) error { return json.Unmarshal(b, &a.Value) } -type rpcProc uint32 +type RpcProc uint32 const ( - rpcCreate rpcProc = (iota+1)<<8 | 1 - rpcStart - rpcShutdownGraceful - rpcShutdownForced - rpcExecuteProcess - rpcWaitForProcess - rpcSignalProcess - rpcResizeConsole - rpcGetProperties - rpcModifySettings - rpcNegotiateProtocol - rpcDumpStacks - rpcDeleteContainerState - rpcUpdateContainer - rpcLifecycleNotification + RpcCreate RpcProc = (iota+1)<<8 | 1 + RpcStart + RpcShutdownGraceful + RpcShutdownForced + RpcExecuteProcess + RpcWaitForProcess + RpcSignalProcess + RpcResizeConsole + RpcGetProperties + RpcModifySettings + RpcNegotiateProtocol + RpcDumpStacks + RpcDeleteContainerState + RpcUpdateContainer + RpcLifecycleNotification ) -func (rpc rpcProc) String() string { +func (rpc RpcProc) String() string { switch rpc { - case rpcCreate: + case RpcCreate: return "Create" - case rpcStart: + case RpcStart: return "Start" - case rpcShutdownGraceful: + case RpcShutdownGraceful: return "ShutdownGraceful" - case rpcShutdownForced: + case RpcShutdownForced: return "ShutdownForced" - case rpcExecuteProcess: + case RpcExecuteProcess: return "ExecuteProcess" - case rpcWaitForProcess: + case RpcWaitForProcess: return "WaitForProcess" - case rpcSignalProcess: + case RpcSignalProcess: return "SignalProcess" - case rpcResizeConsole: + case RpcResizeConsole: return "ResizeConsole" - case rpcGetProperties: + case RpcGetProperties: return "GetProperties" - case rpcModifySettings: + case RpcModifySettings: return "ModifySettings" - case rpcNegotiateProtocol: + case RpcNegotiateProtocol: return "NegotiateProtocol" - case rpcDumpStacks: + case RpcDumpStacks: return "DumpStacks" - case rpcDeleteContainerState: + case RpcDeleteContainerState: return "DeleteContainerState" - case rpcUpdateContainer: + case RpcUpdateContainer: return "UpdateContainer" - case rpcLifecycleNotification: + case RpcLifecycleNotification: return "LifecycleNotification" default: return "0x" + strconv.FormatUint(uint64(rpc), 16) } } -type msgType uint32 +type MsgType uint32 const ( - msgTypeRequest msgType = 0x10100000 - msgTypeResponse msgType = 0x20100000 - msgTypeNotify msgType = 0x30100000 - msgTypeMask msgType = 0xfff00000 + MsgTypeRequest MsgType = 0x10100000 + MsgTypeResponse MsgType = 0x20100000 + MsgTypeNotify MsgType = 0x30100000 + MsgTypeMask MsgType = 0xfff00000 - notifyContainer = 1<<8 | 1 + NotifyContainer = 1<<8 | 1 ) -func (typ msgType) String() string { +func (typ MsgType) String() string { var s string - switch typ & msgTypeMask { - case msgTypeRequest: + switch typ & MsgTypeMask { + case MsgTypeRequest: s = "Request(" - case msgTypeResponse: + case MsgTypeResponse: s = "Response(" - case msgTypeNotify: + case MsgTypeNotify: s = "Notify(" - switch typ - msgTypeNotify { - case notifyContainer: + switch typ - MsgTypeNotify { + case NotifyContainer: s += "Container" default: s += fmt.Sprintf("%#x", uint32(typ)) @@ -132,13 +133,13 @@ func (typ msgType) String() string { default: return fmt.Sprintf("%#x", uint32(typ)) } - s += rpcProc(typ &^ msgTypeMask).String() + s += RpcProc(typ &^ MsgTypeMask).String() return s + ")" } -// ocspancontext is the internal JSON representation of the OpenCensus +// Ocspancontext is the internal JSON representation of the OpenCensus // `trace.SpanContext` for fowarding to a GCS that supports it. -type ocspancontext struct { +type Ocspancontext struct { // TraceID is the `hex` encoded string of the OpenCensus // `SpanContext.TraceID` to propagate to the guest. TraceID string `json:",omitempty"` @@ -158,7 +159,7 @@ type ocspancontext struct { Tracestate string `json:",omitempty"` } -type requestBase struct { +type RequestBase struct { ContainerID string `json:"ContainerId"` ActivityID guid.GUID `json:"ActivityId"` @@ -168,155 +169,145 @@ type requestBase struct { // NOTE: This is not a part of the protocol but because its a JSON protocol // adding fields is a non-breaking change. If the guest supports it this is // just additive context. - OpenCensusSpanContext *ocspancontext `json:"ocsc,omitempty"` + OpenCensusSpanContext *Ocspancontext `json:"ocsc,omitempty"` } -func (req *requestBase) Base() *requestBase { +func (req *RequestBase) Base() *RequestBase { return req } -type responseBase struct { - Result int32 // HResult - ErrorMessage string `json:",omitempty"` - ActivityID guid.GUID `json:"ActivityId,omitempty"` - ErrorRecords []errorRecord `json:",omitempty"` +type ResponseBase struct { + Result int32 // HResult + ErrorMessage string `json:",omitempty"` + ActivityID guid.GUID `json:"ActivityId,omitempty"` + ErrorRecords []commonutils.ErrorRecord `json:",omitempty"` } -type errorRecord struct { - Result int32 // HResult - Message string - StackTrace string `json:",omitempty"` - ModuleName string - FileName string - Line uint32 - FunctionName string `json:",omitempty"` -} - -func (resp *responseBase) Base() *responseBase { +func (resp *ResponseBase) Base() *ResponseBase { return resp } -type negotiateProtocolRequest struct { - requestBase +type NegotiateProtocolRequest struct { + RequestBase MinimumVersion uint32 MaximumVersion uint32 } -type negotiateProtocolResponse struct { - responseBase +type NegotiateProtocolResponse struct { + ResponseBase Version uint32 `json:",omitempty"` - Capabilities gcsCapabilities `json:",omitempty"` + Capabilities GcsCapabilities `json:",omitempty"` } -type dumpStacksRequest struct { - requestBase +type DumpStacksRequest struct { + RequestBase } -type dumpStacksResponse struct { - responseBase +type DumpStacksResponse struct { + ResponseBase GuestStacks string } -type deleteContainerStateRequest struct { - requestBase +type DeleteContainerStateRequest struct { + RequestBase } -type containerCreate struct { - requestBase - ContainerConfig anyInString +type ContainerCreate struct { + RequestBase + ContainerConfig AnyInString } -type uvmConfig struct { +type UvmConfig struct { SystemType string // must be "Container" TimeZoneInformation *hcsschema.TimeZoneInformation } -type containerNotification struct { - requestBase +type ContainerNotification struct { + RequestBase Type string // Compute.System.NotificationType Operation string // Compute.System.ActiveOperation Result int32 // HResult - ResultInfo anyInString `json:",omitempty"` + ResultInfo AnyInString `json:",omitempty"` } -type containerExecuteProcess struct { - requestBase - Settings executeProcessSettings +type ContainerExecuteProcess struct { + RequestBase + Settings ExecuteProcessSettings } -type executeProcessSettings struct { - ProcessParameters anyInString - StdioRelaySettings *executeProcessStdioRelaySettings `json:",omitempty"` - VsockStdioRelaySettings *executeProcessVsockStdioRelaySettings `json:",omitempty"` +type ExecuteProcessSettings struct { + ProcessParameters AnyInString + StdioRelaySettings *ExecuteProcessStdioRelaySettings `json:",omitempty"` + VsockStdioRelaySettings *ExecuteProcessVsockStdioRelaySettings `json:",omitempty"` } -type executeProcessStdioRelaySettings struct { +type ExecuteProcessStdioRelaySettings struct { StdIn *guid.GUID `json:",omitempty"` StdOut *guid.GUID `json:",omitempty"` StdErr *guid.GUID `json:",omitempty"` } -type executeProcessVsockStdioRelaySettings struct { +type ExecuteProcessVsockStdioRelaySettings struct { StdIn uint32 `json:",omitempty"` StdOut uint32 `json:",omitempty"` StdErr uint32 `json:",omitempty"` } -type containerResizeConsole struct { - requestBase +type ContainerResizeConsole struct { + RequestBase ProcessID uint32 `json:"ProcessId"` Height uint16 Width uint16 } -type containerWaitForProcess struct { - requestBase +type ContainerWaitForProcess struct { + RequestBase ProcessID uint32 `json:"ProcessId"` TimeoutInMs uint32 } -type containerSignalProcess struct { - requestBase +type ContainerSignalProcess struct { + RequestBase ProcessID uint32 `json:"ProcessId"` Options interface{} `json:",omitempty"` } -type containerPropertiesQuery schema1.PropertyQuery +type ContainerPropertiesQuery schema1.PropertyQuery -func (q *containerPropertiesQuery) MarshalText() ([]byte, error) { +func (q *ContainerPropertiesQuery) MarshalText() ([]byte, error) { return json.Marshal((*schema1.PropertyQuery)(q)) } -func (q *containerPropertiesQuery) UnmarshalText(b []byte) error { +func (q *ContainerPropertiesQuery) UnmarshalText(b []byte) error { return json.Unmarshal(b, (*schema1.PropertyQuery)(q)) } -type containerPropertiesQueryV2 hcsschema.PropertyQuery +type ContainerPropertiesQueryV2 hcsschema.PropertyQuery -func (q *containerPropertiesQueryV2) MarshalText() ([]byte, error) { +func (q *ContainerPropertiesQueryV2) MarshalText() ([]byte, error) { return json.Marshal((*hcsschema.PropertyQuery)(q)) } -func (q *containerPropertiesQueryV2) UnmarshalText(b []byte) error { +func (q *ContainerPropertiesQueryV2) UnmarshalText(b []byte) error { return json.Unmarshal(b, (*hcsschema.PropertyQuery)(q)) } -type containerGetProperties struct { - requestBase - Query containerPropertiesQuery +type ContainerGetProperties struct { + RequestBase + Query ContainerPropertiesQuery } -type containerGetPropertiesV2 struct { - requestBase - Query containerPropertiesQueryV2 +type ContainerGetPropertiesV2 struct { + RequestBase + Query ContainerPropertiesQueryV2 } -type containerModifySettings struct { - requestBase +type ContainerModifySettings struct { + RequestBase Request interface{} } -type gcsCapabilities struct { +type GcsCapabilities struct { SendHostCreateMessage bool SendHostStartMessage bool HvSocketConfigOnStartup bool @@ -326,46 +317,46 @@ type gcsCapabilities struct { GuestDefinedCapabilities json.RawMessage } -type containerCreateResponse struct { - responseBase +type ContainerCreateResponse struct { + ResponseBase } -type containerExecuteProcessResponse struct { - responseBase +type ContainerExecuteProcessResponse struct { + ResponseBase ProcessID uint32 `json:"ProcessId"` } -type containerWaitForProcessResponse struct { - responseBase +type ContainerWaitForProcessResponse struct { + ResponseBase ExitCode uint32 } -type containerProperties schema1.ContainerProperties +type ContainerProperties schema1.ContainerProperties -func (p *containerProperties) MarshalText() ([]byte, error) { +func (p *ContainerProperties) MarshalText() ([]byte, error) { return json.Marshal((*schema1.ContainerProperties)(p)) } -func (p *containerProperties) UnmarshalText(b []byte) error { +func (p *ContainerProperties) UnmarshalText(b []byte) error { return json.Unmarshal(b, (*schema1.ContainerProperties)(p)) } -type containerPropertiesV2 hcsschema.Properties +type ContainerPropertiesV2 hcsschema.Properties -func (p *containerPropertiesV2) MarshalText() ([]byte, error) { +func (p *ContainerPropertiesV2) MarshalText() ([]byte, error) { return json.Marshal((*hcsschema.Properties)(p)) } -func (p *containerPropertiesV2) UnmarshalText(b []byte) error { +func (p *ContainerPropertiesV2) UnmarshalText(b []byte) error { return json.Unmarshal(b, (*hcsschema.Properties)(p)) } -type containerGetPropertiesResponse struct { - responseBase - Properties containerProperties +type ContainerGetPropertiesResponse struct { + ResponseBase + Properties ContainerProperties } -type containerGetPropertiesResponseV2 struct { - responseBase - Properties containerPropertiesV2 +type ContainerGetPropertiesResponseV2 struct { + ResponseBase + Properties ContainerPropertiesV2 } diff --git a/internal/guest/bridge/bridge.go b/internal/guest/bridge/bridge.go index f14663344f..024c7108b9 100644 --- a/internal/guest/bridge/bridge.go +++ b/internal/guest/bridge/bridge.go @@ -11,9 +11,7 @@ import ( "encoding/json" "fmt" "io" - "math" "os" - "strconv" "sync" "sync/atomic" "time" @@ -23,7 +21,8 @@ import ( "go.opencensus.io/trace" "go.opencensus.io/trace/tracestate" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/log" @@ -360,7 +359,7 @@ func (b *Bridge) ListenAndServe(bridgeIn io.ReadCloser, bridgeOut io.WriteCloser if span != nil { oc.SetSpanStatus(span, err) } - setErrorForResponseBase(resp.Base(), err) + setErrorForResponseBase(resp.Base(), err, "gcs" /* moduleName */) } br.response = resp b.responseChan <- br @@ -446,45 +445,9 @@ func (b *Bridge) PublishNotification(n *prot.ContainerNotification) { // setErrorForResponseBase modifies the passed-in MessageResponseBase to // contain information pertaining to the given error. -func setErrorForResponseBase(response *prot.MessageResponseBase, errForResponse error) { - errorMessage := errForResponse.Error() - stackString := "" - fileName := "" - // We use -1 as a sentinel if no line number found (or it cannot be parsed), - // but that will ultimately end up as [math.MaxUint32], so set it to that explicitly. - // (Still keep using -1 for backwards compatibility ...) - lineNumber := uint32(math.MaxUint32) - functionName := "" - if stack := gcserr.BaseStackTrace(errForResponse); stack != nil { - bottomFrame := stack[0] - stackString = fmt.Sprintf("%+v", stack) - fileName = fmt.Sprintf("%s", bottomFrame) - lineNumberStr := fmt.Sprintf("%d", bottomFrame) - if n, err := strconv.ParseUint(lineNumberStr, 10, 32); err == nil { - lineNumber = uint32(n) - } else { - logrus.WithFields(logrus.Fields{ - "line-number": lineNumberStr, - logrus.ErrorKey: err, - }).Error("opengcs::bridge::setErrorForResponseBase - failed to parse line number, using -1 instead") - } - functionName = fmt.Sprintf("%n", bottomFrame) - } - hresult, err := gcserr.GetHresult(errForResponse) - if err != nil { - // Default to using the generic failure HRESULT. - hresult = gcserr.HrFail - } +func setErrorForResponseBase(response *prot.MessageResponseBase, errForResponse error, moduleName string) { + hresult, errorMessage, newRecord := commonutils.SetErrorForResponseBaseUtil(errForResponse, moduleName) response.Result = int32(hresult) response.ErrorMessage = errorMessage - newRecord := prot.ErrorRecord{ - Result: int32(hresult), - Message: errorMessage, - StackTrace: stackString, - ModuleName: "gcs", - FileName: fileName, - Line: lineNumber, - FunctionName: functionName, - } response.ErrorRecords = append(response.ErrorRecords, newRecord) } diff --git a/internal/guest/bridge/bridge_unit_test.go b/internal/guest/bridge/bridge_unit_test.go index 67f583da05..613eaf326b 100644 --- a/internal/guest/bridge/bridge_unit_test.go +++ b/internal/guest/bridge/bridge_unit_test.go @@ -12,7 +12,7 @@ import ( "sync" "testing" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/transport" "github.com/pkg/errors" diff --git a/internal/guest/bridge/bridge_v2.go b/internal/guest/bridge/bridge_v2.go index f9712abc9d..800094e549 100644 --- a/internal/guest/bridge/bridge_v2.go +++ b/internal/guest/bridge/bridge_v2.go @@ -13,8 +13,8 @@ import ( "go.opencensus.io/trace" "golang.org/x/sys/unix" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" "github.com/Microsoft/hcsshim/internal/guest/stdio" diff --git a/internal/guest/commonutils/utilities.go b/internal/guest/commonutils/utilities.go deleted file mode 100644 index adcf70e6c2..0000000000 --- a/internal/guest/commonutils/utilities.go +++ /dev/null @@ -1,26 +0,0 @@ -package commonutils - -import ( - "encoding/json" - "io" - - "github.com/Microsoft/hcsshim/internal/guest/gcserr" -) - -// UnmarshalJSONWithHresult unmarshals the given data into the given interface, and -// wraps any error returned in an HRESULT error. -func UnmarshalJSONWithHresult(data []byte, v interface{}) error { - if err := json.Unmarshal(data, v); err != nil { - return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) - } - return nil -} - -// DecodeJSONWithHresult decodes the JSON from the given reader into the given -// interface, and wraps any error returned in an HRESULT error. -func DecodeJSONWithHresult(r io.Reader, v interface{}) error { - if err := json.NewDecoder(r).Decode(v); err != nil { - return gcserr.WrapHresult(err, gcserr.HrVmcomputeInvalidJSON) - } - return nil -} diff --git a/internal/guest/prot/protocol.go b/internal/guest/prot/protocol.go index 891891d510..88993ae563 100644 --- a/internal/guest/prot/protocol.go +++ b/internal/guest/prot/protocol.go @@ -11,7 +11,7 @@ import ( oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" @@ -601,26 +601,13 @@ func UnmarshalContainerModifySettings(b []byte) (*ContainerModifySettings, error return &request, nil } -// ErrorRecord represents a single error to be reported back to the HCS. It -// allows for specifying information about the source of the error, as well as -// an error message and stack trace. -type ErrorRecord struct { - Result int32 - Message string - StackTrace string `json:",omitempty"` - ModuleName string - FileName string - Line uint32 - FunctionName string `json:",omitempty"` -} - // MessageResponseBase is the base type embedded in all messages sent from the // GCS to the HCS except for ContainerNotification. type MessageResponseBase struct { Result int32 - ActivityID string `json:"ActivityId,omitempty"` - ErrorMessage string `json:",omitempty"` // Only used by hcsshim external bridge - ErrorRecords []ErrorRecord `json:",omitempty"` + ActivityID string `json:"ActivityId,omitempty"` + ErrorMessage string `json:",omitempty"` // Only used by hcsshim external bridge + ErrorRecords []commonutils.ErrorRecord `json:",omitempty"` } // Base returns the response base by reference. diff --git a/internal/guest/runtime/hcsv2/container.go b/internal/guest/runtime/hcsv2/container.go index f763f2615a..9627daf645 100644 --- a/internal/guest/runtime/hcsv2/container.go +++ b/internal/guest/runtime/hcsv2/container.go @@ -18,7 +18,7 @@ import ( "github.com/sirupsen/logrus" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime" specGuest "github.com/Microsoft/hcsshim/internal/guest/spec" diff --git a/internal/guest/runtime/hcsv2/network.go b/internal/guest/runtime/hcsv2/network.go index 136f544153..f6052e84f9 100644 --- a/internal/guest/runtime/hcsv2/network.go +++ b/internal/guest/runtime/hcsv2/network.go @@ -14,7 +14,7 @@ import ( "github.com/vishvananda/netns" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/network" "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" diff --git a/internal/guest/runtime/hcsv2/process.go b/internal/guest/runtime/hcsv2/process.go index e29e6e62f7..e94c9792f6 100644 --- a/internal/guest/runtime/hcsv2/process.go +++ b/internal/guest/runtime/hcsv2/process.go @@ -10,7 +10,7 @@ import ( "sync" "syscall" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/runtime" "github.com/Microsoft/hcsshim/internal/guest/stdio" "github.com/Microsoft/hcsshim/internal/log" diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 9e98cefc91..65a521d5da 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -30,9 +30,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" - + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/debug" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" "github.com/Microsoft/hcsshim/internal/guest/policy" "github.com/Microsoft/hcsshim/internal/guest/prot" "github.com/Microsoft/hcsshim/internal/guest/runtime" diff --git a/internal/guest/runtime/runc/runc.go b/internal/guest/runtime/runc/runc.go index 555fd17a7e..cd11cefdda 100644 --- a/internal/guest/runtime/runc/runc.go +++ b/internal/guest/runtime/runc/runc.go @@ -16,7 +16,7 @@ import ( "github.com/pkg/errors" "golang.org/x/sys/unix" - "github.com/Microsoft/hcsshim/internal/guest/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" "github.com/Microsoft/hcsshim/internal/guest/runtime" "github.com/Microsoft/hcsshim/internal/guest/stdio" ) diff --git a/internal/guest/runtime/runtime.go b/internal/guest/runtime/runtime.go index a8c5231cfc..db24459c27 100644 --- a/internal/guest/runtime/runtime.go +++ b/internal/guest/runtime/runtime.go @@ -8,7 +8,7 @@ import ( "io" "syscall" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/stdio" oci "github.com/opencontainers/runtime-spec/specs-go" ) diff --git a/internal/uvm/create_lcow.go b/internal/uvm/create_lcow.go index 1ab71e4e73..1bb9461a14 100644 --- a/internal/uvm/create_lcow.go +++ b/internal/uvm/create_lcow.go @@ -20,7 +20,7 @@ import ( "go.opencensus.io/trace" "github.com/Microsoft/hcsshim/internal/copyfile" - "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -438,7 +438,7 @@ func makeLCOWVMGSDoc(ctx context.Context, opts *OptionsLCOW, uvm *UtilityVM) (_ // entropyVsockPort - 1 is the entropy port, // linuxLogVsockPort - 109 used by vsockexec to log stdout/stderr logging, // 0x40000000 + 1 (LinuxGcsVsockPort + 1) is the bridge (see guestconnectiuon.go) - hvSockets := []uint32{entropyVsockPort, linuxLogVsockPort, gcs.LinuxGcsVsockPort, gcs.LinuxGcsVsockPort + 1} + hvSockets := []uint32{entropyVsockPort, linuxLogVsockPort, prot.LinuxGcsVsockPort, prot.LinuxGcsVsockPort + 1} hvSockets = append(hvSockets, opts.ExtraVSockPorts...) for _, whichSocket := range hvSockets { key := winio.VsockServiceID(whichSocket).String() @@ -988,7 +988,7 @@ func CreateLCOW(ctx context.Context, opts *OptionsLCOW) (_ *UtilityVM, err error if opts.UseGuestConnection { log.G(ctx).WithField("vmID", uvm.runtimeID).Debug("Using external GCS bridge") - l, err := uvm.listenVsock(gcs.LinuxGcsVsockPort) + l, err := uvm.listenVsock(prot.LinuxGcsVsockPort) if err != nil { return nil, err } diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 0b91e42cf2..2b6f8c8b2d 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -15,7 +15,7 @@ import ( "github.com/sirupsen/logrus" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" @@ -57,15 +57,6 @@ type OptionsWCOW struct { AdditionalRegistryKeys []hcsschema.RegistryValue } -// WindowsSidecarGcsHvsockServiceID is the hvsock service ID that the Windows GCS -// sidecar will connect to. This is only used in the confidential mode. -var windowsSidecarGcsHvsockServiceID = guid.GUID{ - Data1: 0xae8da506, - Data2: 0xa019, - Data3: 0x4553, - Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, -} - // NewDefaultOptionsWCOW creates the default options for a bootable version of // WCOW. The caller `MUST` set the `BootFiles` on the returned value. // @@ -502,9 +493,9 @@ func CreateWCOW(ctx context.Context, opts *OptionsWCOW) (_ *UtilityVM, err error return nil, fmt.Errorf("error while creating the compute system: %w", err) } - gcsServiceID := gcs.WindowsGcsHvsockServiceID + gcsServiceID := prot.WindowsGcsHvsockServiceID if opts.SecurityPolicyEnabled { - gcsServiceID = windowsSidecarGcsHvsockServiceID + gcsServiceID = prot.WindowsSidecarGcsHvsockServiceID } if err = uvm.startExternalGcsListener(ctx, gcsServiceID); err != nil { diff --git a/internal/uvm/start.go b/internal/uvm/start.go index ec8cdf1ee9..9ab8a9bdd5 100644 --- a/internal/uvm/start.go +++ b/internal/uvm/start.go @@ -18,6 +18,7 @@ import ( "golang.org/x/sys/windows" "github.com/Microsoft/hcsshim/internal/gcs" + "github.com/Microsoft/hcsshim/internal/gcs/prot" "github.com/Microsoft/hcsshim/internal/hcs" "github.com/Microsoft/hcsshim/internal/hcs/schema1" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" @@ -134,7 +135,7 @@ func (uvm *UtilityVM) configureHvSocketForGCS(ctx context.Context) (err error) { hvsocketAddress := &hcsschema.HvSocketAddress{ LocalAddress: uvm.runtimeID.String(), - ParentAddress: gcs.WindowsGcsHvHostID.String(), + ParentAddress: prot.WindowsGcsHvHostID.String(), } conSetupReq := &hcsschema.ModifySettingRequest{ From c3dcf0351ec88ec78a1ce3e771ac6a360307c1dd Mon Sep 17 00:00:00 2001 From: Kirtana Ashok Date: Mon, 21 Apr 2025 09:48:24 -0700 Subject: [PATCH 2/2] gcs-sidecar framework This commit makes the high level changes needed for gcs-sidecar - Starts sidecar as service - Dereferences the various valid rpc requests - Adds code to invoke refs formatter Note: This commit does not add invokers to the code for new ResourceTypes like SecurityPolicy, CWCOWBlockCIMs, Container scratch formatting etc. This will come in along with functional tests in later PRs. There are some TODO comments in the code which will be addressed in upcoming PRs as well. To make this initialization of the gcs-sidecar flow complete, certain high level code for the policy enforcement have been brought into this commit from Mahati's changes. Example: internal/gcs-sidecar/policy.go, internal/gcs-sidecar/host.go and helper functions in internal/gcs-sidecar/host.go. Hence adding her as co-author in this commit. The rest of the policy framework code will be brought in by Mahati as follow up PRs. Co-authored-by: Signed-off-by: Kirtana Ashok --- cmd/gcs-sidecar/main.go | 244 ++++++++ internal/fsformatter/formatter_driver.go | 283 +++++++++ internal/gcs-sidecar/bridge.go | 541 ++++++++++++++++++ internal/gcs-sidecar/handlers.go | 491 ++++++++++++++++ internal/gcs-sidecar/host.go | 64 +++ internal/gcs-sidecar/policy.go | 19 + internal/gcs-sidecar/uvm.go | 123 ++++ internal/gcs/bridge.go | 46 +- internal/gcs/bridge_test.go | 10 +- internal/gcs/container.go | 18 +- internal/gcs/guestconnection.go | 14 +- internal/gcs/guestconnection_test.go | 12 +- internal/gcs/process.go | 10 +- internal/gcs/prot/protocol.go | 109 ++-- internal/guest/prot/protocol.go | 8 +- internal/guest/runtime/hcsv2/uvm.go | 16 +- internal/protocol/guestresource/resources.go | 39 ++ .../securitypolicy_uvmpath_linux.go | 34 ++ .../securitypolicy_uvmpath_windows.go | 24 + pkg/securitypolicy/securitypolicyenforcer.go | 22 +- test/gcs/container_test.go | 2 +- 21 files changed, 1999 insertions(+), 130 deletions(-) create mode 100644 cmd/gcs-sidecar/main.go create mode 100644 internal/fsformatter/formatter_driver.go create mode 100644 internal/gcs-sidecar/bridge.go create mode 100644 internal/gcs-sidecar/handlers.go create mode 100644 internal/gcs-sidecar/host.go create mode 100644 internal/gcs-sidecar/policy.go create mode 100644 internal/gcs-sidecar/uvm.go create mode 100644 pkg/securitypolicy/securitypolicy_uvmpath_linux.go create mode 100644 pkg/securitypolicy/securitypolicy_uvmpath_windows.go diff --git a/cmd/gcs-sidecar/main.go b/cmd/gcs-sidecar/main.go new file mode 100644 index 0000000000..4cd0d70b34 --- /dev/null +++ b/cmd/gcs-sidecar/main.go @@ -0,0 +1,244 @@ +//go:build windows +// +build windows + +package main + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "time" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + shimlog "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/svc" + "golang.org/x/sys/windows/svc/debug" + + sidecar "github.com/Microsoft/hcsshim/internal/gcs-sidecar" +) + +var ( + defaultLogFile = "C:\\gcs-sidecar-logs.log" + defaultLogLevel = "trace" +) + +type handler struct { + fromsvc chan error +} + +// Accepts new connection and closes listener. +func acceptAndClose(ctx context.Context, l net.Listener) (net.Conn, error) { + var conn net.Conn + ch := make(chan error) + go func() { + var err error + conn, err = l.Accept() + ch <- err + }() + select { + case err := <-ch: + l.Close() + return conn, err + case <-ctx.Done(): + } + l.Close() + err := <-ch + if err == nil { + return conn, err + } + + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, err +} + +func (h *handler) Execute(args []string, r <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { + const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE) + + status <- svc.Status{State: svc.StartPending, Accepts: 0} + // unblock runService() + h.fromsvc <- nil + + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + +loop: + for c := range r { + switch c.Cmd { + case svc.Interrogate: + status <- c.CurrentStatus + case svc.Stop, svc.Shutdown: + logrus.Println("Shutting service...!") + break loop + case svc.Pause: + status <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted} + case svc.Continue: + status <- svc.Status{State: svc.Running, Accepts: cmdsAccepted} + default: + logrus.Printf("Unexpected service control request #%d", c) + } + } + + status <- svc.Status{State: svc.StopPending} + return false, 1 +} + +func runService(name string, isDebug bool) error { + h := &handler{ + fromsvc: make(chan error), + } + + var err error + go func() { + if isDebug { + err = debug.Run(name, h) + if err != nil { + logrus.Errorf("Error running service in debug mode.Err: %v", err) + } + } else { + err = svc.Run(name, h) + if err != nil { + logrus.Errorf("Error running service in Service Control mode.Err %v", err) + } + } + h.fromsvc <- err + }() + + // Wait for the first signal from the service handler. + logrus.Tracef("waiting for first signal from service handler\n") + return <-h.fromsvc +} + +func main() { + logLevel := flag.String("loglevel", + defaultLogLevel, + "Logging Level: trace, debug, info, warning, error, fatal, panic.") + logFile := flag.String("logfile", + defaultLogFile, + "Logging Target. Default is at C:\\gcs-sidecar-logs.log inside UVM") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "\nUsage of %s:\n", os.Args[0]) + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "Examples:\n") + fmt.Fprintf(os.Stderr, " %s -loglevel=trace -logfile=C:\\sidecarLogs.log \n", os.Args[0]) + } + + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + logFileHandle, err := os.OpenFile(*logFile, os.O_RDWR|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0666) + if err != nil { + fmt.Printf("error opening file: %v", err) + } + defer logFileHandle.Close() + + logrus.AddHook(shimlog.NewHook()) + + level, err := logrus.ParseLevel(*logLevel) + if err != nil { + logrus.Fatal(err) + } + logrus.SetLevel(level) + logrus.SetOutput(logFileHandle) + trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) + trace.RegisterExporter(&oc.LogrusExporter{}) + + if err := windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(logFileHandle.Fd())); err != nil { + logrus.WithError(err).Error("error redirecting handle") + return + } + os.Stderr = logFileHandle + + chsrv := make(chan error) + go func() { + defer close(chsrv) + + if err := runService("gcs-sidecar", false); err != nil { + logrus.Errorf("error starting gcs-sidecar service: %v", err) + } + + chsrv <- err + }() + + select { + case <-ctx.Done(): + logrus.Error("context deadline exceeded") + return + case r := <-chsrv: + if r != nil { + logrus.Error(r) + return + } + } + + // 1. Start external server to connect with inbox GCS + listener, err := winio.ListenHvsock(&winio.HvsockAddr{ + VMID: prot.HvGUIDLoopback, + ServiceID: prot.WindowsGcsHvsockServiceID, + }) + if err != nil { + logrus.WithError(err).Error("error starting listener for sidecar <-> inbox gcs communication") + return + } + + var gcsListener net.Listener = listener + gcsCon, err := acceptAndClose(ctx, gcsListener) + if err != nil { + logrus.WithError(err).Error("error accepting inbox GCS connection") + return + } + + // 2. Setup connection with external gcs connection started from hcsshim + hvsockAddr := &winio.HvsockAddr{ + VMID: prot.HvGUIDParent, + ServiceID: prot.WindowsSidecarGcsHvsockServiceID, + } + + logrus.WithFields(logrus.Fields{ + "hvsockAddr": hvsockAddr, + }).Tracef("Dialing to hcsshim external bridge at address %v", hvsockAddr) + shimCon, err := winio.Dial(ctx, hvsockAddr) + if err != nil { + logrus.WithError(err).Error("error dialing hcsshim external bridge") + return + } + + // gcs-sidecar can be used for non-confidentail hyperv wcow + // as well. So we do not always want to check for initialPolicyStance + var initialEnforcer securitypolicy.SecurityPolicyEnforcer + // TODO (kiashok/Mahati): The initialPolicyStance is set to allow + // only for dev. This will eventually be set to allow/deny depending on + // on whether SNP is supported or not. + initialPolicyStance := "allow" + switch initialPolicyStance { + case "allow": + initialEnforcer = &securitypolicy.OpenDoorSecurityPolicyEnforcer{} + logrus.Tracef("initial-policy-stance: allow") + case "deny": + initialEnforcer = &securitypolicy.ClosedDoorSecurityPolicyEnforcer{} + logrus.Tracef("initial-policy-stance: deny") + default: + logrus.Error("unknown initial-policy-stance") + } + + // 3. Create bridge and initializa + brdg := sidecar.NewBridge(shimCon, gcsCon, initialEnforcer) + brdg.AssignHandlers() + + // 3. Listen and serve for hcsshim requests. + err = brdg.ListenAndServeShimRequests() + if err != nil { + logrus.WithError(err).Error("failed to serve request") + } +} diff --git a/internal/fsformatter/formatter_driver.go b/internal/fsformatter/formatter_driver.go new file mode 100644 index 0000000000..14f743cf0b --- /dev/null +++ b/internal/fsformatter/formatter_driver.go @@ -0,0 +1,283 @@ +//go:build windows +// +build windows + +package fsformatter + +import ( + "context" + "encoding/binary" + "syscall" + "unicode/utf16" + "unsafe" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// This file contains all the supporting structures needed to make +// an ioctl call to RefsFormatter. +const ( + ioctlKernelFormatVolumeFormat = 0x40001000 + // This is used to construct the disk path that refsFormatter + // understands. `harddisk%d` here refers to the disk number + // associated with the corresponding lun of the attached + // scsi device. + VirtualDevObjectPathFormat = "\\device\\harddisk%d\\partition0" + checksumTypeSha256 = uint16(4) + refsChecksumType = checksumTypeSha256 + maxSizeOfKernelFormatVolumeFormatRefsParameters = 16 * 8 // 128 bytes + sizeOfWchar = int(unsafe.Sizeof(uint16(0))) + kernelFormatVolumeMaxVolumeLabelLength = uint32(33 * sizeOfWchar) + kernelFormatVolumeWin32DriverPath = "\\\\?\\KernelFSFormatter" + // Allocate large enough buffer for output from fsFormatter + maxSizeOfOutputBuffer = uint32(512) + + // KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS member offsets + clusterSizeOffset = 0 + checksumTypeOffset = 4 + useDataIntegrityOffset = 6 + majorVersionOffset = 8 + minorVersionOffset = 10 +) + +type kernelFormatVolumeFilesystemTypes uint32 + +const ( + kernelFormatVolumeFilesystemTypeInvalid = kernelFormatVolumeFilesystemTypes(iota) + kernelFormatVolumeFilesystemTypeRefs = kernelFormatVolumeFilesystemTypes(1) + kernelFormatVolumeFilesystemTypeMax = kernelFormatVolumeFilesystemTypes(2) +) + +// We only want to allow refs formatting +func (filesystemType kernelFormatVolumeFilesystemTypes) String() string { + switch filesystemType { + case kernelFormatVolumeFilesystemTypeRefs: + return "KERNEL_FORMAT_VOLUME_FILESYSTEM_TYPE_REFS" + default: + return "Unknown" + } +} + +type kernelFormatVolumeFormatInputBufferFlags uint32 + +const kernelFormatVolumeFormatInputBufferFlagNone = kernelFormatVolumeFormatInputBufferFlags(0x00000000) + +func (flag kernelFormatVolumeFormatInputBufferFlags) String() string { + switch flag { + case kernelFormatVolumeFormatInputBufferFlagNone: + return "kernelFormatVolumeFormatInputBufferFlagNone" + default: + return "Unknown" + } +} + +type KernelFormatVolumeFormatRefsParameters struct { + ClusterSize uint32 + MetadataChecksumType uint16 + UseDataIntegrity bool + MajorVersion uint16 + MinorVersion uint16 +} + +type KernelFormatVolumeFormatFsParameters struct { + FileSystemType kernelFormatVolumeFilesystemTypes + // Represents a WCHAR character array + VolumeLabel [kernelFormatVolumeMaxVolumeLabelLength / uint32(sizeOfWchar)]uint16 + // Length of volume label in bytes + VolumeLabelLength uint16 + // RefsFormatterParams represents the following union + /* + union { + + KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS RefsParameters; + + // + // This structure can't grow in size nor change in alignment. 16 ULONGLONGs + // should be more than enough for supporting other filesystems down the + // line. This also serves to enforce 8 byte alignment. + // + Reserved [16]uint64 + }; + */ + RefsFormatterParams [128]byte +} + +type KernelFormatVolumeFormatInputBuffer struct { + Size uint64 + FsParameters KernelFormatVolumeFormatFsParameters + Flags kernelFormatVolumeFormatInputBufferFlags + Reserved [4]uint32 + // Size of DiskPathBuffer in bytes + DiskPathLength uint16 + // DiskPathBuffer holds the disk path. It represents a + // variable size WCHAR character array + DiskPathBuffer []uint16 +} + +type kernelFormatVolumeFormatOutputBufferFlags uint32 + +const kernelFormatVolumeFormatOutputBufferFlagsNone = kernelFormatVolumeFormatOutputBufferFlags(0x00000000) + +func (flag kernelFormatVolumeFormatOutputBufferFlags) String() string { + switch flag { + case kernelFormatVolumeFormatOutputBufferFlagsNone: + return "kernelFormatVolumeFormatOutputBufferFlagsNone" + default: + return "Unknown" + } +} + +type KernelFormarVolumeFormatOutputBuffer struct { + Size uint32 + Flags kernelFormatVolumeFormatOutputBufferFlags + Reserved [4]uint32 + // VolumePathLength holds size of VolumePathBuffer + // in bytes + VolumePathLength uint16 + // VolumePathBuffer holds the mounted volume path + // as returned from refsFormatter. It represents + // a variable size WCHAR character array + VolumePathBuffer []uint16 +} + +// GetVolumePathBufferOffset gets offset to KernelFormarVolumeFormatOutputBuffer{}.VolumePathBuffer +func GetVolumePathBufferOffset() uint32 { + volPathBufferOffset := uint32(unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Size) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Flags) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.Reserved) + + unsafe.Sizeof(KernelFormarVolumeFormatOutputBuffer{}.VolumePathLength)) + + return volPathBufferOffset +} + +// getInputBufferSize gets the total size needed for input buffer +func getInputBufferSize(wcharDiskPathLength uint16) uint32 { + bufferSize := uint32(unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Size)+ + /* This is specifically for the union in KernelFormatVolumeFormatFsParameters */ + unsafe.Offsetof(KernelFormatVolumeFormatFsParameters{}.RefsFormatterParams)+ + maxSizeOfKernelFormatVolumeFormatRefsParameters+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Flags)+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Reserved)+ + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.DiskPathLength)) + + uint32(wcharDiskPathLength) + + return bufferSize +} + +// getInputBufferDiskPathBufferOffset gets offset to KernelFormatVolumeFormatInputBuffer{}.DiskPathBuffer +func getInputBufferDiskPathBufferOffset() uint32 { + diskPathBufferOffset := uint32(unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Size) + + unsafe.Offsetof(KernelFormatVolumeFormatFsParameters{}.RefsFormatterParams) + + maxSizeOfKernelFormatVolumeFormatRefsParameters + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Flags) + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.Reserved) + + unsafe.Sizeof(KernelFormatVolumeFormatInputBuffer{}.DiskPathLength)) + + return diskPathBufferOffset +} + +// KmFmtCreateFormatOutputBuffer formats an output buffer as expected +// by the fsFormatter driver +func KmFmtCreateFormatOutputBuffer() *KernelFormarVolumeFormatOutputBuffer { + buf := make([]uint16, maxSizeOfOutputBuffer) + outputBuffer := (*KernelFormarVolumeFormatOutputBuffer)(unsafe.Pointer(&buf[0])) + outputBuffer.Size = uint32(maxSizeOfOutputBuffer) + + return outputBuffer +} + +func toUTF16(s string) []uint16 { + return utf16.Encode([]rune(s)) +} + +// KmFmtCreateFormatInputBuffer formats an input buffer as expected +// by the refsFormatter driver. +// diskPath represents disk path in VirtualDevObjectPathFormat. +func KmFmtCreateFormatInputBuffer(diskPath string) *KernelFormatVolumeFormatInputBuffer { + refsParametersBuf := make([]byte, unsafe.Sizeof(KernelFormatVolumeFormatRefsParameters{})) + refsParameters := (*KernelFormatVolumeFormatRefsParameters)(unsafe.Pointer(&refsParametersBuf[0])) + + utf16DiskPath := toUTF16(diskPath) + wcharDiskPathLength := uint16(len(utf16DiskPath) * sizeOfWchar) + + refsParameters.ClusterSize = 0x1000 + refsParameters.MetadataChecksumType = refsChecksumType + refsParameters.UseDataIntegrity = true + refsParameters.MajorVersion = uint16(3) + refsParameters.MinorVersion = uint16(14) + + bufferSize := getInputBufferSize(wcharDiskPathLength) + buf := make([]byte, bufferSize) + inputBuffer := (*KernelFormatVolumeFormatInputBuffer)(unsafe.Pointer(&buf[0])) + + inputBuffer.Size = uint64(bufferSize) + inputBuffer.Flags = kernelFormatVolumeFormatInputBufferFlagNone + + inputBuffer.FsParameters.FileSystemType = kernelFormatVolumeFilesystemTypeRefs + inputBuffer.FsParameters.VolumeLabelLength = 0 + inputBuffer.FsParameters.VolumeLabel = [33]uint16{} + + // Write KERNEL_FORMAT_VOLUME_FORMAT_REFS_PARAMETERS + binary.LittleEndian.PutUint32(inputBuffer.FsParameters.RefsFormatterParams[clusterSizeOffset:], refsParameters.ClusterSize) + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[checksumTypeOffset:], refsParameters.MetadataChecksumType) + if refsParameters.UseDataIntegrity { + inputBuffer.FsParameters.RefsFormatterParams[useDataIntegrityOffset] = 1 + } else { + inputBuffer.FsParameters.RefsFormatterParams[useDataIntegrityOffset] = 0 + } + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[majorVersionOffset:], refsParameters.MajorVersion) + binary.LittleEndian.PutUint16(inputBuffer.FsParameters.RefsFormatterParams[minorVersionOffset:], refsParameters.MinorVersion) + + // Finally write the diskPathLength and diskPathBuffer with the input disk path + inputBuffer.DiskPathLength = wcharDiskPathLength + // DiskBuffer writing + ptr := unsafe.Add(unsafe.Pointer(inputBuffer), getInputBufferDiskPathBufferOffset()) + // Convert the string to UTF-16 slice + utf16Array := toUTF16(diskPath) + diskPathBuf := unsafe.Slice((*uint16)(ptr), len(utf16Array)) + copy(diskPathBuf, utf16Array) + + return inputBuffer +} + +// InvokeFsFormatter makes an ioctl call to the fsFormatter driver and returns +// a path to the mountedVolume +func InvokeFsFormatter(ctx context.Context, diskPath string) (string, error) { + // Prepare input and output buffers as expected by refsFormatter + inputBuffer := KmFmtCreateFormatInputBuffer(diskPath) + outputBuffer := KmFmtCreateFormatOutputBuffer() + + utf16DriverPath, _ := windows.UTF16PtrFromString(kernelFormatVolumeWin32DriverPath) + deviceHandle, err := windows.CreateFile(utf16DriverPath, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + 0, + nil, + windows.OPEN_EXISTING, + 0, + 0) + if err != nil { + return "", errors.Wrap(err, "failed to get handle to refsFormatter driver") + } + defer windows.Close(deviceHandle) + + // Ioctl to fsFormatter driver + var bytesReturned uint32 + if err := windows.DeviceIoControl( + deviceHandle, + ioctlKernelFormatVolumeFormat, + (*byte)(unsafe.Pointer(inputBuffer)), + uint32(inputBuffer.Size), + (*byte)(unsafe.Pointer(outputBuffer)), + outputBuffer.Size, + &bytesReturned, + nil, + ); err != nil { + return "", errors.Wrap(err, "ioctl to refsFormatter driver failed") + } + + // Read the returned volume path from the corresponding offset in outputBuffer + ptr := unsafe.Pointer(uintptr(unsafe.Pointer(outputBuffer)) + uintptr(GetVolumePathBufferOffset())) + utf16Data := unsafe.Slice((*uint16)(ptr), outputBuffer.VolumePathLength/2) + mountedVolumePath := syscall.UTF16ToString(utf16Data) + return mountedVolumePath, err +} diff --git a/internal/gcs-sidecar/bridge.go b/internal/gcs-sidecar/bridge.go new file mode 100644 index 0000000000..545d2b7eb3 --- /dev/null +++ b/internal/gcs-sidecar/bridge.go @@ -0,0 +1,541 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" + "go.opencensus.io/trace/tracestate" + "golang.org/x/sys/windows" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" +) + +type Bridge struct { + mu sync.Mutex + hostState *Host + // List of handlers for handling different rpc message requests. + rpcHandlerList map[prot.RPCProc]HandlerFunc + + // hcsshim and inbox GCS connections respectively. + shimConn io.ReadWriteCloser + inboxGCSConn io.ReadWriteCloser + + // Response channels to forward incoming requests to inbox GCS + // and send responses back to hcsshim respectively. + sendToGCSCh chan request + sendToShimCh chan bridgeResponse +} + +// SequenceID is used to correlate requests and responses. +type sequenceID uint64 + +// messageHeader is the common header present in all communications messages. +type messageHeader struct { + Type prot.MsgType + Size uint32 + ID sequenceID +} + +type bridgeResponse struct { + ctx context.Context + header messageHeader + response []byte +} + +type request struct { + // Context created once received from the bridge. + ctx context.Context + // header is the wire format message header that preceded the message for + // this request. + header messageHeader + // activityID is the id of the specific activity for this request. + activityID guid.GUID + // message is the portion of the request that follows the `Header`. + message []byte +} + +func NewBridge(shimConn io.ReadWriteCloser, inboxGCSConn io.ReadWriteCloser, initialEnforcer securitypolicy.SecurityPolicyEnforcer) *Bridge { + hostState := NewHost(initialEnforcer) + return &Bridge{ + rpcHandlerList: make(map[prot.RPCProc]HandlerFunc), + hostState: hostState, + shimConn: shimConn, + inboxGCSConn: inboxGCSConn, + sendToGCSCh: make(chan request), + sendToShimCh: make(chan bridgeResponse), + } +} + +func NewPolicyEnforcer(initialEnforcer securitypolicy.SecurityPolicyEnforcer) *SecurityPolicyEnforcer { + return &SecurityPolicyEnforcer{ + securityPolicyEnforcerSet: false, + securityPolicyEnforcer: initialEnforcer, + } +} + +// UnknownMessage represents the default handler logic for an unmatched request +// type sent from the bridge. +func UnknownMessage(r *request) error { + log.G(r.ctx).Debugf("bridge: function not supported, header type %v", prot.MsgType(r.header.Type).String()) + return gcserr.WrapHresult(errors.Errorf("bridge: function not supported, header type: %v", r.header.Type), gcserr.HrNotImpl) +} + +// HandlerFunc is an adapter to use functions as handlers. +type HandlerFunc func(*request) error + +func (b *Bridge) getRequestHandler(r *request) (HandlerFunc, error) { + b.mu.Lock() + defer b.mu.Unlock() + + var handler HandlerFunc + var ok bool + messageType := r.header.Type + rpcProcID := prot.RPCProc(messageType &^ prot.MsgTypeMask) + if handler, ok = b.rpcHandlerList[rpcProcID]; !ok { + return nil, UnknownMessage(r) + } + return handler, nil +} + +// ServeMsg serves request by calling appropriate handler functions. +func (b *Bridge) ServeMsg(r *request) error { + if r == nil { + panic("bridge: nil request to handler") + } + + var handler HandlerFunc + var err error + if handler, err = b.getRequestHandler(r); err != nil { + return UnknownMessage(r) + } + return handler(r) +} + +// Handle registers the handler for the given message id and protocol version. +func (b *Bridge) Handle(rpcProcID prot.RPCProc, handlerFunc HandlerFunc) { + b.mu.Lock() + defer b.mu.Unlock() + + if handlerFunc == nil { + panic("empty function handler") + } + + if _, ok := b.rpcHandlerList[rpcProcID]; ok { + logrus.WithFields(logrus.Fields{ + "message-type": rpcProcID.String(), + }).Warn("overwriting bridge handler") + } + + b.rpcHandlerList[rpcProcID] = handlerFunc +} + +func (b *Bridge) HandleFunc(rpcProcID prot.RPCProc, handler func(*request) error) { + if handler == nil { + panic("bridge: nil handler func") + } + + b.Handle(rpcProcID, HandlerFunc(handler)) +} + +// AssignHandlers creates and assigns appropriate event handlers +// for the different bridge message types. +func (b *Bridge) AssignHandlers() { + b.HandleFunc(prot.RPCCreate, b.createContainer) + b.HandleFunc(prot.RPCStart, b.startContainer) + b.HandleFunc(prot.RPCShutdownGraceful, b.shutdownGraceful) + b.HandleFunc(prot.RPCShutdownForced, b.shutdownForced) + b.HandleFunc(prot.RPCExecuteProcess, b.executeProcess) + b.HandleFunc(prot.RPCWaitForProcess, b.waitForProcess) + b.HandleFunc(prot.RPCSignalProcess, b.signalProcess) + b.HandleFunc(prot.RPCResizeConsole, b.resizeConsole) + b.HandleFunc(prot.RPCGetProperties, b.getProperties) + b.HandleFunc(prot.RPCModifySettings, b.modifySettings) + b.HandleFunc(prot.RPCNegotiateProtocol, b.negotiateProtocol) + b.HandleFunc(prot.RPCDumpStacks, b.dumpStacks) + b.HandleFunc(prot.RPCDeleteContainerState, b.deleteContainerState) + b.HandleFunc(prot.RPCUpdateContainer, b.updateContainer) + b.HandleFunc(prot.RPCLifecycleNotification, b.lifecycleNotification) +} + +// readMessage reads the message from io.Reader +func readMessage(r io.Reader) (messageHeader, []byte, error) { + var h [prot.HdrSize]byte + _, err := io.ReadFull(r, h[:]) + if err != nil { + return messageHeader{}, nil, err + } + var header messageHeader + buf := bytes.NewReader(h[:]) + err = binary.Read(buf, binary.LittleEndian, &header) + if err != nil { + logrus.WithError(err).Errorf("error reading message header") + return messageHeader{}, nil, err + } + + n := header.Size + if n < prot.HdrSize || n > prot.MaxMsgSize { + logrus.Errorf("invalid message size %d", n) + return messageHeader{}, nil, fmt.Errorf("invalid message size %d: %w", n, err) + } + + n -= prot.HdrSize + msg := make([]byte, n) + _, err = io.ReadFull(r, msg) + if err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return messageHeader{}, nil, err + } + + return header, msg, nil +} + +func isLocalDisconnectError(err error) bool { + return errors.Is(err, windows.WSAECONNABORTED) +} + +// Sends request to the inbox GCS channel +func (b *Bridge) forwardRequestToGcs(req *request) { + b.sendToGCSCh <- *req +} + +func getContextAndSpan(baseSpanCtx *prot.Ocspancontext) (context.Context, *trace.Span) { + var ctx context.Context + var span *trace.Span + if baseSpanCtx != nil { + sc := trace.SpanContext{} + if bytes, err := hex.DecodeString(baseSpanCtx.TraceID); err == nil { + copy(sc.TraceID[:], bytes) + } + if bytes, err := hex.DecodeString(baseSpanCtx.SpanID); err == nil { + copy(sc.SpanID[:], bytes) + } + sc.TraceOptions = trace.TraceOptions(baseSpanCtx.TraceOptions) + if baseSpanCtx.Tracestate != "" { + if bytes, err := base64.StdEncoding.DecodeString(baseSpanCtx.Tracestate); err == nil { + var entries []tracestate.Entry + if err := json.Unmarshal(bytes, &entries); err == nil { + if ts, err := tracestate.New(nil, entries...); err == nil { + sc.Tracestate = ts + } + } + } + } + ctx, span = oc.StartSpanWithRemoteParent( + context.Background(), + "sidecar::request", + sc, + oc.WithServerSpanKind, + ) + } else { + ctx, span = oc.StartSpan( + context.Background(), + "sidecar::request", + oc.WithServerSpanKind, + ) + } + + return ctx, span +} + +func sendWithContextCancel[T any](ctx context.Context, sendCh chan<- T, msg T) error { + select { + case sendCh <- msg: + // Sent successfully + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Sends response to the hcsshim channel +func (b *Bridge) sendResponseToShim(ctx context.Context, rpcProcType prot.RPCProc, id sequenceID, response interface{}) error { + respType := prot.MsgTypeResponse | prot.MsgType(rpcProcType) + msgb, err := json.Marshal(response) + if err != nil { + return err + } + msgHeader := messageHeader{ + Type: respType, + Size: uint32(len(msgb) + prot.HdrSize), + ID: id, + } + + resp := bridgeResponse{ + ctx: ctx, + header: msgHeader, + response: msgb, + } + + return sendWithContextCancel(ctx, b.sendToShimCh, resp) +} + +// ListenAndServeShimRequests listens to messages on the hcsshim +// and inbox GCS connections and schedules them for processing. +// After processing, messages are forwarded to inbox GCS on success +// and responses from inbox GCS or error messages are sent back +// to hcsshim via bridge connection. +func (b *Bridge) ListenAndServeShimRequests() error { + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + shimRequestChan := make(chan request) + // goroutines share the error channel to send error responses + // back to hcsshim. Therefore use buffered error channel. + sidecarErrChan := make(chan error, 5) + + // Goroutine 1: Listen to requests from hcsshim + wg.Add(1) + go func() { + defer wg.Done() + defer close(shimRequestChan) + + var recverr error + br := bufio.NewReader(b.shimConn) + for { + select { + case <-ctx.Done(): + return + default: + header, msg, err := readMessage(br) + if err != nil { + if errors.Is(err, io.EOF) || isLocalDisconnectError(err) { + // hcsshim handles these errors explicitly. Therefore, + // returning the exact same error. + recverr = err + } else { + recverr = fmt.Errorf("bridge read from shim connection failed: %w", err) + } + logrus.Error(recverr) + _ = sendWithContextCancel(ctx, sidecarErrChan, recverr) + return + } + + var msgBase prot.RequestBase + _ = json.Unmarshal(msg, &msgBase) + reqCtx, span := getContextAndSpan(msgBase.OpenCensusSpanContext) + span.AddAttributes( + trace.Int64Attribute("message-id", int64(header.ID)), + trace.StringAttribute("message-type", header.Type.String()), + trace.StringAttribute("activityID", msgBase.ActivityID.String()), + trace.StringAttribute("containerID", msgBase.ContainerID)) + + req := request{ + ctx: reqCtx, + activityID: msgBase.ActivityID, + header: header, + message: msg, + } + if err := sendWithContextCancel(ctx, shimRequestChan, req); err != nil { + log.G(ctx).WithError(err).Error("failed to send request to shimRequestChan") + return + } + } + } + }() + // Goroutine 2: Process each bridge request received from shim asynchronously. + wg.Add(1) + go func() { + defer wg.Done() + + for { + // Requests are served sequentially to avoid + // racing/reordering of incoming message order. + // This becomes important for confidential cases + // where the shim could be compromised and replay + // messages out of order. + select { + case <-ctx.Done(): + return + case req, ok := <-shimRequestChan: + if !ok { + return + } + if err := b.ServeMsg(&req); err != nil { + log.G(req.ctx).WithError(err).Errorf("failed to serve request: %v", req.header.Type.String()) + // In case of error, create appropriate response message to + // be sent to hcsshim. + resp := &prot.ResponseBase{ + Result: int32(windows.ERROR_GEN_FAILURE), + ErrorMessage: err.Error(), + ActivityID: req.activityID, + } + setErrorForResponseBase(resp, err, "gcs-sidecar" /* moduleName */) + // Prepares and sends message to hcsshim via b.sendToShimCh + responseErr := b.sendResponseToShim(req.ctx, prot.RPCProc(prot.MsgTypeResponse), req.header.ID, resp) + if responseErr != nil { + log.G(req.ctx).WithError(err).Errorf("failed to send response to shim") + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + } + }() + // Goroutine 3: Forrward requests to inbox GCS + wg.Add(1) + go func() { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + case req, ok := <-b.sendToGCSCh: + if !ok { + return + } + + // Forward message to gcs + log.G(req.ctx).Tracef("bridge send to gcs, req %v, %v", req.header.Type.String(), string(req.message)) + buffer, err := b.prepareResponseMessage(req.header, req.message) + if err != nil { + responseErr := fmt.Errorf("error preparing response: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + + _, err = buffer.WriteTo(b.inboxGCSConn) + if err != nil { + responseErr := fmt.Errorf("err forwarding shim req to inbox GCS: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + }() + // Goroutine 4: Receive response from gcs and forward to hcsshim + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + default: + header, message, err := readMessage(b.inboxGCSConn) + if err != nil { + var recverr error + if errors.Is(err, io.EOF) || isLocalDisconnectError(err) { + // hcsshim handles these errors explicitly. Therefore, + // returning the exact same error. + recverr = err + } else { + recverr = fmt.Errorf("bridge read from gcs failed: %w", err) + } + logrus.Error(recverr) + _ = sendWithContextCancel(ctx, sidecarErrChan, recverr) + return + } + + // Forward to shim + resp := bridgeResponse{ + ctx: context.Background(), + header: header, + response: message, + } + if err := sendWithContextCancel(ctx, b.sendToShimCh, resp); err != nil { + log.G(ctx).WithError(err).Error("failed to send request to b.sendToShimCh") + return + } + } + } + }() + // Goroutine 5: Send response to hcsshim + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + case resp, ok := <-b.sendToShimCh: + if !ok { + return + } + + // Send response to shim + logrus.Tracef("Send response to shim. Header:{ID: %v, Type: %v, Size: %v} msg: %v", resp.header.ID, + resp.header.Type, resp.header.Size, string(resp.response)) + buffer, err := b.prepareResponseMessage(resp.header, resp.response) + if err != nil { + responseErr := fmt.Errorf("error preparing response: %w", err) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + _, sendErr := buffer.WriteTo(b.shimConn) + if sendErr != nil { + responseErr := fmt.Errorf("err sending response to shim: %w", sendErr) + _ = sendWithContextCancel(ctx, sidecarErrChan, responseErr) + return + } + } + } + }() + + var finalErr error + select { + case finalErr = <-sidecarErrChan: + cancel() + case <-ctx.Done(): + finalErr = ctx.Err() + cancel() + } + + // Close read b.ShimConn and b.inboxGCSConn + _ = b.shimConn.Close() + _ = b.inboxGCSConn.Close() + + // Close sidecarErrChan after all the goroutines have finished + // sending to it. + wg.Wait() + close(sidecarErrChan) + + return finalErr +} + +// Prepare response message +func (b *Bridge) prepareResponseMessage(header messageHeader, message []byte) (bytes.Buffer, error) { + // Create a buffer to hold the serialized header data + var headerBuf bytes.Buffer + err := binary.Write(&headerBuf, binary.LittleEndian, header) + if err != nil { + return headerBuf, err + } + + // Write message header followed by actual payload. + var buf bytes.Buffer + buf.Write(headerBuf.Bytes()) + buf.Write(message[:]) + return buf, nil +} + +// setErrorForResponseBase modifies the passed-in ResponseBase to +// contain information pertaining to the given error. +func setErrorForResponseBase(response *prot.ResponseBase, errForResponse error, moduleName string) { + hresult, errorMessage, newRecord := commonutils.SetErrorForResponseBaseUtil(errForResponse, moduleName) + response.Result = int32(hresult) + response.ErrorMessage = errorMessage + response.ErrorRecords = append(response.ErrorRecords, newRecord) +} diff --git a/internal/gcs-sidecar/handlers.go b/internal/gcs-sidecar/handlers.go new file mode 100644 index 0000000000..e321d4e092 --- /dev/null +++ b/internal/gcs-sidecar/handlers.go @@ -0,0 +1,491 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/fsformatter" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/internal/windevice" + "github.com/Microsoft/hcsshim/pkg/cimfs" +) + +const ( + sandboxStateDirName = "WcSandboxState" + hivesDirName = "Hives" + devPathFormat = "\\\\.\\PHYSICALDRIVE%d" +) + +// - Handler functions handle the incoming message requests. It +// also enforces security policy for confidential cwcow containers. +// - These handler functions may do some additional processing before +// forwarding requests to inbox GCS or send responses back to hcsshim. +// - In case of any error encountered during processing, appropriate error +// messages are returned and responses are sent back to hcsshim from ListenAndServer(). +// TODO (kiashok): Verbose logging is for WIP and will be removed eventually. +func (b *Bridge) createContainer(req *request) (err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::createContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerCreate + var containerConfig json.RawMessage + r.ContainerConfig.Value = &containerConfig + if err = commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal createContainer: %w", err) + } + + // containerConfig can be of type uvnConfig or hcsschema.HostedSystem + var ( + uvmConfig prot.UvmConfig + hostedSystemConfig hcsschema.HostedSystem + ) + if err = commonutils.UnmarshalJSONWithHresult(containerConfig, &uvmConfig); err == nil && + uvmConfig.SystemType != "" { + systemType := uvmConfig.SystemType + timeZoneInformation := uvmConfig.TimeZoneInformation + log.G(ctx).Tracef("createContainer: uvmConfig: {systemType: %v, timeZoneInformation: %v}}", systemType, timeZoneInformation) + } else if err = commonutils.UnmarshalJSONWithHresult(containerConfig, &hostedSystemConfig); err == nil && + hostedSystemConfig.SchemaVersion != nil && hostedSystemConfig.Container != nil { + schemaVersion := hostedSystemConfig.SchemaVersion + container := hostedSystemConfig.Container + log.G(ctx).Tracef("createContainer: HostedSystemConfig: {schemaVersion: %v, container: %v}}", schemaVersion, container) + } else { + return fmt.Errorf("invalid request to createContainer") + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) startContainer(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::startContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal startContainer: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) shutdownGraceful(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::shutdownGraceful") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal shutdownGraceful: %w", err) + } + + // TODO (kiashok/Mahati): Since gcs-sidecar can be used for all types of windows + // containers, it is important to check if we want to + // enforce policy or not. + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) shutdownForced(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::shutdownForced") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.RequestBase + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal shutdownForced: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) executeProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::executeProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerExecuteProcess + var processParamSettings json.RawMessage + r.Settings.ProcessParameters.Value = &processParamSettings + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal executeProcess: %w", err) + } + + var processParams hcsschema.ProcessParameters + if err := commonutils.UnmarshalJSONWithHresult(processParamSettings, &processParams); err != nil { + return fmt.Errorf("executeProcess: invalid params type for request: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) waitForProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::waitForProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerWaitForProcess + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal waitForProcess: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) signalProcess(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::signalProcess") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerSignalProcess + var rawOpts json.RawMessage + r.Options = &rawOpts + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal signalProcess: %w", err) + } + + var wcowOptions guestresource.SignalProcessOptionsWCOW + if rawOpts != nil { + if err := commonutils.UnmarshalJSONWithHresult(rawOpts, &wcowOptions); err != nil { + return fmt.Errorf("signalProcess: invalid Options type for request: %w", err) + } + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) resizeConsole(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::resizeConsole") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerResizeConsole + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal resizeConsole: %v", req) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) getProperties(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::getProperties") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var getPropReqV2 prot.ContainerGetPropertiesV2 + if err := commonutils.UnmarshalJSONWithHresult(req.message, &getPropReqV2); err != nil { + return fmt.Errorf("failed to unmarshal getProperties: %v: %w", string(req.message), err) + } + log.G(req.ctx).Tracef("getProperties query: %v", getPropReqV2.Query.PropertyTypes) + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) negotiateProtocol(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::negotiateProtocol") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.NegotiateProtocolRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal negotiateProtocol") + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) dumpStacks(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::dumpStacks") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.DumpStacksRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal dumpStacks: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) deleteContainerState(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::deleteContainerState") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.DeleteContainerStateRequest + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return fmt.Errorf("failed to unmarshal deleteContainerState: %w", err) + } + + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) updateContainer(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::updateContainer") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + // No callers in the code for rpcUpdateContainer + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) lifecycleNotification(req *request) (err error) { + _, span := oc.StartSpan(req.ctx, "sidecar::lifecycleNotification") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + // No callers in the code for rpcLifecycleNotification + b.forwardRequestToGcs(req) + return nil +} + +func (b *Bridge) modifySettings(req *request) (err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::modifySettings") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + log.G(ctx).Tracef("modifySettings: MsgType: %v, Payload: %v", req.header.Type, string(req.message)) + modifyRequest, err := unmarshalContainerModifySettings(req) + if err != nil { + return err + } + modifyGuestSettingsRequest := modifyRequest.Request.(*guestrequest.ModificationRequest) + guestResourceType := modifyGuestSettingsRequest.ResourceType + guestRequestType := modifyGuestSettingsRequest.RequestType + log.G(ctx).Tracef("modifySettings: resourceType: %v, requestType: %v", guestResourceType, guestRequestType) + + if guestRequestType == "" { + guestRequestType = guestrequest.RequestTypeAdd + } + + switch guestRequestType { + case guestrequest.RequestTypeAdd: + case guestrequest.RequestTypeRemove: + case guestrequest.RequestTypePreAdd: + case guestrequest.RequestTypeUpdate: + default: + return fmt.Errorf("invald guestRequestType %v", guestRequestType) + } + + if guestResourceType != "" { + switch guestResourceType { + case guestresource.ResourceTypeCombinedLayers: + settings := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWCombinedLayers) + log.G(ctx).Tracef("WCOWCombinedLayers: {%v}", settings) + + case guestresource.ResourceTypeNetworkNamespace: + settings := modifyGuestSettingsRequest.Settings.(*hcn.HostComputeNamespace) + log.G(ctx).Tracef("HostComputeNamespaces { %v}", settings) + + case guestresource.ResourceTypeNetwork: + settings := modifyGuestSettingsRequest.Settings.(*guestrequest.NetworkModifyRequest) + log.G(ctx).Tracef("NetworkModifyRequest { %v}", settings) + + case guestresource.ResourceTypeMappedVirtualDisk: + wcowMappedVirtualDisk := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWMappedVirtualDisk) + log.G(ctx).Tracef("wcowMappedVirtualDisk { %v}", wcowMappedVirtualDisk) + + case guestresource.ResourceTypeHvSocket: + hvSocketAddress := modifyGuestSettingsRequest.Settings.(*hcsschema.HvSocketAddress) + log.G(ctx).Tracef("hvSocketAddress { %v }", hvSocketAddress) + + case guestresource.ResourceTypeMappedDirectory: + settings := modifyGuestSettingsRequest.Settings.(*hcsschema.MappedDirectory) + log.G(ctx).Tracef("hcsschema.MappedDirectory { %v }", settings) + + case guestresource.ResourceTypeSecurityPolicy: + securityPolicyRequest := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWConfidentialOptions) + log.G(ctx).Tracef("WCOWConfidentialOptions: { %v}", securityPolicyRequest) + _ = b.hostState.SetWCOWConfidentialUVMOptions(securityPolicyRequest) + + // Send response back to shim + resp := &prot.ResponseBase{ + Result: 0, // 0 means success + ActivityID: req.activityID, + } + err := b.sendResponseToShim(req.ctx, prot.RPCModifySettings, req.header.ID, resp) + if err != nil { + return fmt.Errorf("error sending response to hcsshim: %w", err) + } + return nil + + case guestresource.ResourceTypeWCOWBlockCims: + // This is request to mount the merged cim at given volumeGUID + wcowBlockCimMounts := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWBlockCIMMounts) + log.G(ctx).Tracef("WCOWBlockCIMMounts { %v}", wcowBlockCimMounts) + + // The block device takes some time to show up. Wait for a few seconds. + time.Sleep(2 * time.Second) + + var layerCIMs []*cimfs.BlockCIM + for _, blockCimDevice := range wcowBlockCimMounts.BlockCIMs { + // Get the scsi device path for the blockCim lun + devNumber, err := windevice.GetDeviceNumberFromControllerLUN( + ctx, + 0, /* controller is always 0 for wcow */ + uint8(blockCimDevice.Lun)) + if err != nil { + return fmt.Errorf("err getting scsiDevPath: %w", err) + } + layerCim := cimfs.BlockCIM{ + Type: cimfs.BlockCIMTypeDevice, + BlockPath: fmt.Sprintf(devPathFormat, devNumber), + CimName: blockCimDevice.CimName, + } + layerCIMs = append(layerCIMs, &layerCim) + } + if len(layerCIMs) > 1 { + // Get the topmost merge CIM and invoke the MountMergedBlockCIMs + _, err := cimfs.MountMergedBlockCIMs(layerCIMs[0], layerCIMs[1:], wcowBlockCimMounts.MountFlags, wcowBlockCimMounts.VolumeGUID) + if err != nil { + return fmt.Errorf("error mounting multilayer block cims: %w", err) + } + } else { + _, err := cimfs.Mount(filepath.Join(layerCIMs[0].BlockPath, layerCIMs[0].CimName), wcowBlockCimMounts.VolumeGUID, wcowBlockCimMounts.MountFlags) + if err != nil { + return fmt.Errorf("error mounting merged block cims: %w", err) + } + } + + // Send response back to shim + resp := &prot.ResponseBase{ + Result: 0, // 0 means success + ActivityID: req.activityID, + } + err = b.sendResponseToShim(req.ctx, prot.RPCModifySettings, req.header.ID, resp) + if err != nil { + return fmt.Errorf("error sending response to hcsshim: %w", err) + } + return nil + + case guestresource.ResourceTypeCWCOWCombinedLayers: + settings := modifyGuestSettingsRequest.Settings.(*guestresource.CWCOWCombinedLayers) + containerID := settings.ContainerID + log.G(ctx).Tracef("CWCOWCombinedLayers:: ContainerID: %v, ContainerRootPath: %v, Layers: %v, ScratchPath: %v", + containerID, settings.CombinedLayers.ContainerRootPath, settings.CombinedLayers.Layers, settings.CombinedLayers.ScratchPath) + + // TODO: Update modifyCombinedLayers with verified CimFS API + + // The following two folders are expected to be present in the scratch. + // But since we have just formatted the scratch we would need to + // create them manually. + sandboxStateDirectory := filepath.Join(settings.CombinedLayers.ContainerRootPath, sandboxStateDirName) + err = os.Mkdir(sandboxStateDirectory, 0777) + if err != nil { + return fmt.Errorf("failed to create sandboxStateDirectory: %w", err) + } + + hivesDirectory := filepath.Join(settings.CombinedLayers.ContainerRootPath, hivesDirName) + err = os.Mkdir(hivesDirectory, 0777) + if err != nil { + return fmt.Errorf("failed to create hivesDirectory: %w", err) + } + + // Reconstruct WCOWCombinedLayers{} req before forwarding to GCS + // as GCS does not understand ResourceTypeCWCOWCombinedLayers + modifyGuestSettingsRequest.ResourceType = guestresource.ResourceTypeCombinedLayers + modifyGuestSettingsRequest.Settings = settings.CombinedLayers + modifyRequest.Request = modifyGuestSettingsRequest + buf, err := json.Marshal(modifyRequest) + if err != nil { + return fmt.Errorf("failed to marshal rpcModifySettings: %w", err) + } + var newRequest request + newRequest.ctx = req.ctx + newRequest.header = req.header + newRequest.header.Size = uint32(len(buf)) + prot.HdrSize + newRequest.message = buf + req = &newRequest + + case guestresource.ResourceTypeMappedVirtualDiskForContainerScratch: + wcowMappedVirtualDisk := modifyGuestSettingsRequest.Settings.(*guestresource.WCOWMappedVirtualDisk) + log.G(ctx).Tracef("ResourceTypeMappedVirtualDiskForContainerScratch: { %v }", wcowMappedVirtualDisk) + + // 1. TODO (Mahati): Need to enforce policy before calling into fsFormatter + // 2. Call fsFormatter to format the scratch disk. + // This will return the volume path of the mounted scratch. + // Scratch disk should be >= 30 GB for refs formatter to work. + + // fsFormatter understands only virtualDevObjectPathFormat. Therefore fetch the + // disk number for the corresponding lun + var devNumber uint32 + // It could take a few seconds for the attached scsi disk + // to show up inside the UVM. Therefore adding retry logic + // with delay here. + for try := 0; try < 5; try++ { + time.Sleep(1 * time.Second) + devNumber, err = windevice.GetDeviceNumberFromControllerLUN(req.ctx, + 0, /* Only one controller allowed in wcow hyperv */ + uint8(wcowMappedVirtualDisk.Lun)) + if err != nil { + if try == 4 { + // bail out + return fmt.Errorf("error getting diskNumber for LUN %d: %w", wcowMappedVirtualDisk.Lun, err) + } + continue + } else { + log.G(ctx).Tracef("DiskNumber of lun %d is: %d", wcowMappedVirtualDisk.Lun, devNumber) + break + } + } + diskPath := fmt.Sprintf(fsformatter.VirtualDevObjectPathFormat, devNumber) + log.G(ctx).Tracef("diskPath: %v, diskNumber: %v ", diskPath, devNumber) + mountedVolumePath, err := fsformatter.InvokeFsFormatter(req.ctx, diskPath) + if err != nil { + return fmt.Errorf("failed to invoke refsFormatter: %w", err) + } + log.G(ctx).Tracef("mountedVolumePath returned from InvokeFsFormatter: %v", mountedVolumePath) + + // Forward the req as is to inbox gcs and let it retreive the volume. + // While forwarding request to inbox gcs, make sure to replace the + // resourceType to ResourceTypeMappedVirtualDisk that inbox GCS + // understands. + modifyGuestSettingsRequest.ResourceType = guestresource.ResourceTypeMappedVirtualDisk + modifyRequest.Request = modifyGuestSettingsRequest + buf, err := json.Marshal(modifyRequest) + if err != nil { + return fmt.Errorf("failed to marshal WCOWMappedVirtualDisk: %w", err) + } + var newRequest request + newRequest.ctx = req.ctx + newRequest.header = req.header + newRequest.header.Size = uint32(len(buf)) + prot.HdrSize + newRequest.message = buf + req = &newRequest + + default: + // Invalid request + return fmt.Errorf("invald modifySettingsRequest: %v", guestResourceType) + } + } + + b.forwardRequestToGcs(req) + return nil +} diff --git a/internal/gcs-sidecar/host.go b/internal/gcs-sidecar/host.go new file mode 100644 index 0000000000..b17cd38448 --- /dev/null +++ b/internal/gcs-sidecar/host.go @@ -0,0 +1,64 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "errors" + "fmt" + "sync" + + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" +) + +type Host struct { + // state required for the security policy enforcement + policyMutex sync.Mutex + securityPolicyEnforcer securitypolicy.SecurityPolicyEnforcer + securityPolicyEnforcerSet bool +} + +type SecurityPolicyEnforcer struct { + // State required for the security policy enforcement + securityPolicyEnforcer securitypolicy.SecurityPolicyEnforcer + securityPolicyEnforcerSet bool +} + +func NewHost(initialEnforcer securitypolicy.SecurityPolicyEnforcer) *Host { + return &Host{ + securityPolicyEnforcer: initialEnforcer, + securityPolicyEnforcerSet: false, + } +} + +func (h *Host) SetWCOWConfidentialUVMOptions(securityPolicyRequest *guestresource.WCOWConfidentialOptions) error { + h.policyMutex.Lock() + defer h.policyMutex.Unlock() + + if h.securityPolicyEnforcerSet { + return errors.New("security policy has already been set") + } + + // This limit ensures messages are below the character truncation limit that + // can be imposed by an orchestrator + maxErrorMessageLength := 3 * 1024 + + // Initialize security policy enforcer for a given enforcer type and + // encoded security policy. + p, err := securitypolicy.CreateSecurityPolicyEnforcer( + "rego", + securityPolicyRequest.EncodedSecurityPolicy, + DefaultCRIMounts(), + DefaultCRIPrivilegedMounts(), + maxErrorMessageLength, + ) + if err != nil { + return fmt.Errorf("error creating security policy enforcer: %w", err) + } + + h.securityPolicyEnforcer = p + h.securityPolicyEnforcerSet = true + + return nil +} diff --git a/internal/gcs-sidecar/policy.go b/internal/gcs-sidecar/policy.go new file mode 100644 index 0000000000..13b96ce64d --- /dev/null +++ b/internal/gcs-sidecar/policy.go @@ -0,0 +1,19 @@ +//go:build windows +// +build windows + +package bridge + +import ( + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// DefaultCRIMounts returns default mounts added to windows spec by containerD. +func DefaultCRIMounts() []oci.Mount { + return []oci.Mount{} +} + +// DefaultCRIPrivilegedMounts returns a slice of mounts which are added to the +// windows container spec when a container runs in a privileged mode. +func DefaultCRIPrivilegedMounts() []oci.Mount { + return []oci.Mount{} +} diff --git a/internal/gcs-sidecar/uvm.go b/internal/gcs-sidecar/uvm.go new file mode 100644 index 0000000000..4fccdf3df6 --- /dev/null +++ b/internal/gcs-sidecar/uvm.go @@ -0,0 +1,123 @@ +//go:build windows +// +build windows + +package bridge + +import ( + "encoding/json" + "fmt" + + "github.com/Microsoft/hcsshim/hcn" + "github.com/Microsoft/hcsshim/internal/bridgeutils/commonutils" + "github.com/Microsoft/hcsshim/internal/gcs/prot" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" +) + +func unmarshalContainerModifySettings(req *request) (_ *prot.ContainerModifySettings, err error) { + ctx, span := oc.StartSpan(req.ctx, "sidecar::unmarshalContainerModifySettings") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + var r prot.ContainerModifySettings + var requestRawSettings json.RawMessage + r.Request = &requestRawSettings + if err := commonutils.UnmarshalJSONWithHresult(req.message, &r); err != nil { + return nil, fmt.Errorf("failed to unmarshal rpcModifySettings: %w", err) + } + + var modifyGuestSettingsRequest guestrequest.ModificationRequest + var rawGuestRequest json.RawMessage + modifyGuestSettingsRequest.Settings = &rawGuestRequest + if err := commonutils.UnmarshalJSONWithHresult(requestRawSettings, &modifyGuestSettingsRequest); err != nil { + return nil, fmt.Errorf("invalid rpcModifySettings ModificationRequest: %w", err) + } + + if modifyGuestSettingsRequest.RequestType == "" { + modifyGuestSettingsRequest.RequestType = guestrequest.RequestTypeAdd + } + + if modifyGuestSettingsRequest.ResourceType != "" { + switch modifyGuestSettingsRequest.ResourceType { + case guestresource.ResourceTypeCWCOWCombinedLayers: + settings := &guestresource.CWCOWCombinedLayers{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeCWCOWCombinedLayers request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeCombinedLayers: + settings := &guestresource.WCOWCombinedLayers{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeCombinedLayers request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeNetworkNamespace: + settings := &hcn.HostComputeNamespace{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeNetworkNamespace request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeNetwork: + settings := &guestrequest.NetworkModifyRequest{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeNetwork request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeMappedVirtualDisk: + wcowMappedVirtualDisk := &guestresource.WCOWMappedVirtualDisk{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowMappedVirtualDisk); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedVirtualDisk request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk + + case guestresource.ResourceTypeHvSocket: + hvSocketAddress := &hcsschema.HvSocketAddress{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, hvSocketAddress); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeHvSocket request: %w", err) + } + modifyGuestSettingsRequest.Settings = hvSocketAddress + + case guestresource.ResourceTypeMappedDirectory: + settings := &hcsschema.MappedDirectory{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, settings); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedDirectory request: %w", err) + } + modifyGuestSettingsRequest.Settings = settings + + case guestresource.ResourceTypeSecurityPolicy: + securityPolicyRequest := &guestresource.WCOWConfidentialOptions{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, securityPolicyRequest); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeSecurityPolicy request: %w", err) + } + modifyGuestSettingsRequest.Settings = securityPolicyRequest + + case guestresource.ResourceTypeMappedVirtualDiskForContainerScratch: + wcowMappedVirtualDisk := &guestresource.WCOWMappedVirtualDisk{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowMappedVirtualDisk); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeMappedVirtualDiskForContainerScratch request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowMappedVirtualDisk + + case guestresource.ResourceTypeWCOWBlockCims: + wcowBlockCimMounts := &guestresource.WCOWBlockCIMMounts{} + if err := commonutils.UnmarshalJSONWithHresult(rawGuestRequest, wcowBlockCimMounts); err != nil { + return nil, fmt.Errorf("invalid ResourceTypeWCOWBlockCims request: %w", err) + } + modifyGuestSettingsRequest.Settings = wcowBlockCimMounts + + default: + // Invalid request + log.G(ctx).Errorf("Invald modifySettingsRequest: %v", modifyGuestSettingsRequest.ResourceType) + return nil, fmt.Errorf("invald modifySettingsRequest") + } + } + r.Request = &modifyGuestSettingsRequest + return &r, nil +} diff --git a/internal/gcs/bridge.go b/internal/gcs/bridge.go index 0ed38a4896..c1e026778d 100644 --- a/internal/gcs/bridge.go +++ b/internal/gcs/bridge.go @@ -24,20 +24,6 @@ import ( "github.com/Microsoft/hcsshim/internal/oc" ) -const ( - hdrSize = 16 - hdrOffType = 0 - hdrOffSize = 4 - hdrOffID = 8 - - // maxMsgSize is the maximum size of an incoming message. This is not - // enforced by the guest today but some maximum must be set to avoid - // unbounded allocations. - // - // Matches HCS limitions on maximum (sent and received) message size. - maxMsgSize = 0x10000 -) - type requestMessage interface { Base() *prot.RequestBase } @@ -48,7 +34,7 @@ type responseMessage interface { // rpc represents an outstanding rpc request to the guest type rpc struct { - proc prot.RpcProc + proc prot.RPCProc id int64 req requestMessage resp responseMessage @@ -144,7 +130,7 @@ func (brdg *bridge) Wait() error { // AsyncRPC sends an RPC request to the guest but does not wait for a response. // If the message cannot be sent before the context is done, then an error is // returned. -func (brdg *bridge) AsyncRPC(ctx context.Context, proc prot.RpcProc, req requestMessage, resp responseMessage) (*rpc, error) { +func (brdg *bridge) AsyncRPC(ctx context.Context, proc prot.RPCProc, req requestMessage, resp responseMessage) (*rpc, error) { call := &rpc{ ch: make(chan struct{}), proc: proc, @@ -225,7 +211,7 @@ func (call *rpc) Wait() { // If allowCancel is set and the context becomes done, returns an error without // waiting for a response. Avoid this on messages that are not idempotent or // otherwise safe to ignore the response of. -func (brdg *bridge) RPC(ctx context.Context, proc prot.RpcProc, req requestMessage, resp responseMessage, allowCancel bool) error { +func (brdg *bridge) RPC(ctx context.Context, proc prot.RPCProc, req requestMessage, resp responseMessage, allowCancel bool) error { call, err := brdg.AsyncRPC(ctx, proc, req, resp) if err != nil { return err @@ -266,22 +252,22 @@ func readMessage(r io.Reader) (int64, prot.MsgType, []byte, error) { _, span := oc.StartSpan(context.Background(), "bridge receive read message", oc.WithClientSpanKind) defer span.End() - var h [hdrSize]byte + var h [prot.HdrSize]byte _, err := io.ReadFull(r, h[:]) if err != nil { return 0, 0, nil, fmt.Errorf("header read: %w", err) } - typ := prot.MsgType(binary.LittleEndian.Uint32(h[hdrOffType:])) - n := binary.LittleEndian.Uint32(h[hdrOffSize:]) - id := int64(binary.LittleEndian.Uint64(h[hdrOffID:])) + typ := prot.MsgType(binary.LittleEndian.Uint32(h[prot.HdrOffType:])) + n := binary.LittleEndian.Uint32(h[prot.HdrOffSize:]) + id := int64(binary.LittleEndian.Uint64(h[prot.HdrOffID:])) span.AddAttributes( trace.StringAttribute("type", typ.String()), trace.Int64Attribute("message-id", id)) - if n < hdrSize || n > maxMsgSize { + if n < prot.HdrSize || n > prot.MaxMsgSize { return 0, 0, nil, fmt.Errorf("invalid message size %d", n) } - n -= hdrSize + n -= prot.HdrSize b := make([]byte, n) _, err = io.ReadFull(r, b) if err != nil { @@ -394,24 +380,24 @@ func (brdg *bridge) writeMessage(buf *bytes.Buffer, enc *json.Encoder, typ prot. trace.Int64Attribute("message-id", id)) // Prepare the buffer with the message. - var h [hdrSize]byte - binary.LittleEndian.PutUint32(h[hdrOffType:], uint32(typ)) - binary.LittleEndian.PutUint64(h[hdrOffID:], uint64(id)) + var h [prot.HdrSize]byte + binary.LittleEndian.PutUint32(h[prot.HdrOffType:], uint32(typ)) + binary.LittleEndian.PutUint64(h[prot.HdrOffID:], uint64(id)) buf.Write(h[:]) err = enc.Encode(req) if err != nil { return fmt.Errorf("bridge encode: %w", err) } // Update the message header with the size. - binary.LittleEndian.PutUint32(buf.Bytes()[hdrOffSize:], uint32(buf.Len())) + binary.LittleEndian.PutUint32(buf.Bytes()[prot.HdrOffSize:], uint32(buf.Len())) if brdg.log.Logger.GetLevel() > logrus.DebugLevel { - b := buf.Bytes()[hdrSize:] + b := buf.Bytes()[prot.HdrSize:] switch typ { // container environment vars are in rpCreate for linux; rpcExecuteProcess for windows - case prot.MsgType(prot.RpcCreate) | prot.MsgTypeRequest: + case prot.MsgType(prot.RPCCreate) | prot.MsgTypeRequest: b, err = log.ScrubBridgeCreate(b) - case prot.MsgType(prot.RpcExecuteProcess) | prot.MsgTypeRequest: + case prot.MsgType(prot.RPCExecuteProcess) | prot.MsgTypeRequest: b, err = log.ScrubBridgeExecProcess(b) } if err != nil { diff --git a/internal/gcs/bridge_test.go b/internal/gcs/bridge_test.go index 5a6ab368f1..3da35e9d9d 100644 --- a/internal/gcs/bridge_test.go +++ b/internal/gcs/bridge_test.go @@ -93,7 +93,7 @@ func TestBridgeRPC(t *testing.T) { defer b.Close() req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), prot.RpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RPCCreate, &req, &resp, false) if err != nil { t.Fatal(err) } @@ -108,7 +108,7 @@ func TestBridgeRPCResponseTimeout(t *testing.T) { b.Timeout = time.Millisecond * 100 req := testReq{X: 5} var resp testResp - err := b.RPC(context.Background(), prot.RpcCreate, &req, &resp, false) + err := b.RPC(context.Background(), prot.RPCCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -122,7 +122,7 @@ func TestBridgeRPCContextDone(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, prot.RpcCreate, &req, &resp, true) + err := b.RPC(ctx, prot.RPCCreate, &req, &resp, true) if err != context.DeadlineExceeded { //nolint:errorlint t.Fatalf("expected deadline exceeded, got %s", err) } @@ -136,7 +136,7 @@ func TestBridgeRPCContextDoneNoCancel(t *testing.T) { defer cancel() req := testReq{X: 5} var resp testResp - err := b.RPC(ctx, prot.RpcCreate, &req, &resp, false) + err := b.RPC(ctx, prot.RPCCreate, &req, &resp, false) if err == nil || !strings.Contains(err.Error(), "bridge closed") { t.Fatalf("expected bridge disconnection, got %s", err) } @@ -146,7 +146,7 @@ func TestBridgeRPCBridgeClosed(t *testing.T) { b := startReflectedBridge(t, 0) eerr := errors.New("forcibly terminated") b.kill(eerr) - err := b.RPC(context.Background(), prot.RpcCreate, nil, nil, false) + err := b.RPC(context.Background(), prot.RPCCreate, nil, nil, false) if err != eerr { //nolint:errorlint t.Fatal("unexpected: ", err) } diff --git a/internal/gcs/container.go b/internal/gcs/container.go index 728f38a43a..549abd35a2 100644 --- a/internal/gcs/container.go +++ b/internal/gcs/container.go @@ -56,10 +56,10 @@ func (gc *GuestConnection) CreateContainer(ctx context.Context, cid string, conf } req := prot.ContainerCreate{ RequestBase: makeRequest(ctx, cid), - ContainerConfig: prot.AnyInString{config}, + ContainerConfig: prot.AnyInString{Value: config}, } var resp prot.ContainerCreateResponse - err = gc.brdg.RPC(ctx, prot.RpcCreate, &req, &resp, false) + err = gc.brdg.RPC(ctx, prot.RPCCreate, &req, &resp, false) if err != nil { return nil, err } @@ -135,7 +135,7 @@ func (c *Container) Modify(ctx context.Context, config interface{}) (err error) Request: config, } var resp prot.ResponseBase - return c.gc.brdg.RPC(ctx, prot.RpcModifySettings, &req, &resp, false) + return c.gc.brdg.RPC(ctx, prot.RPCModifySettings, &req, &resp, false) } // Properties returns the requested container properties targeting a V1 schema prot.Container. @@ -150,7 +150,7 @@ func (c *Container) Properties(ctx context.Context, types ...schema1.PropertyTyp Query: prot.ContainerPropertiesQuery{PropertyTypes: types}, } var resp prot.ContainerGetPropertiesResponse - err = c.gc.brdg.RPC(ctx, prot.RpcGetProperties, &req, &resp, true) + err = c.gc.brdg.RPC(ctx, prot.RPCGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -169,7 +169,7 @@ func (c *Container) PropertiesV2(ctx context.Context, types ...hcsschema.Propert Query: prot.ContainerPropertiesQueryV2{PropertyTypes: types}, } var resp prot.ContainerGetPropertiesResponseV2 - err = c.gc.brdg.RPC(ctx, prot.RpcGetProperties, &req, &resp, true) + err = c.gc.brdg.RPC(ctx, prot.RPCGetProperties, &req, &resp, true) if err != nil { return nil, err } @@ -185,10 +185,10 @@ func (c *Container) Start(ctx context.Context) (err error) { req := makeRequest(ctx, c.id) var resp prot.ResponseBase - return c.gc.brdg.RPC(ctx, prot.RpcStart, &req, &resp, false) + return c.gc.brdg.RPC(ctx, prot.RPCStart, &req, &resp, false) } -func (c *Container) shutdown(ctx context.Context, proc prot.RpcProc) error { +func (c *Container) shutdown(ctx context.Context, proc prot.RPCProc) error { req := makeRequest(ctx, c.id) var resp prot.ResponseBase err := c.gc.brdg.RPC(ctx, proc, &req, &resp, true) @@ -216,7 +216,7 @@ func (c *Container) Shutdown(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, prot.RpcShutdownGraceful) + return c.shutdown(ctx, prot.RPCShutdownGraceful) } // Terminate sends a forceful terminate request to the container. The container @@ -230,7 +230,7 @@ func (c *Container) Terminate(ctx context.Context) (err error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return c.shutdown(ctx, prot.RpcShutdownForced) + return c.shutdown(ctx, prot.RPCShutdownForced) } func (c *Container) WaitChannel() <-chan struct{} { diff --git a/internal/gcs/guestconnection.go b/internal/gcs/guestconnection.go index 9607c61261..9107dd4d3b 100644 --- a/internal/gcs/guestconnection.go +++ b/internal/gcs/guestconnection.go @@ -123,7 +123,7 @@ func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGu MaximumVersion: protocolVersion, } var resp prot.NegotiateProtocolResponse - err = gc.brdg.RPC(ctx, prot.RpcNegotiateProtocol, &req, &resp, true) + err = gc.brdg.RPC(ctx, prot.RPCNegotiateProtocol, &req, &resp, true) if err != nil { return err } @@ -150,17 +150,17 @@ func (gc *GuestConnection) connect(ctx context.Context, isColdStart bool, initGu } createReq := prot.ContainerCreate{ RequestBase: makeRequest(ctx, nullContainerID), - ContainerConfig: prot.AnyInString{conf}, + ContainerConfig: prot.AnyInString{Value: conf}, } var createResp prot.ResponseBase - err = gc.brdg.RPC(ctx, prot.RpcCreate, &createReq, &createResp, true) + err = gc.brdg.RPC(ctx, prot.RPCCreate, &createReq, &createResp, true) if err != nil { return err } if resp.Capabilities.SendHostStartMessage { startReq := makeRequest(ctx, nullContainerID) var startResp prot.ResponseBase - err = gc.brdg.RPC(ctx, prot.RpcStart, &startReq, &startResp, true) + err = gc.brdg.RPC(ctx, prot.RPCStart, &startReq, &startResp, true) if err != nil { return err } @@ -181,7 +181,7 @@ func (gc *GuestConnection) Modify(ctx context.Context, settings interface{}) (er Request: settings, } var resp prot.ResponseBase - return gc.brdg.RPC(ctx, prot.RpcModifySettings, &req, &resp, false) + return gc.brdg.RPC(ctx, prot.RPCModifySettings, &req, &resp, false) } func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err error) { @@ -193,7 +193,7 @@ func (gc *GuestConnection) DumpStacks(ctx context.Context) (response string, err RequestBase: makeRequest(ctx, nullContainerID), } var resp prot.DumpStacksResponse - err = gc.brdg.RPC(ctx, prot.RpcDumpStacks, &req, &resp, false) + err = gc.brdg.RPC(ctx, prot.RPCDumpStacks, &req, &resp, false) return resp.GuestStacks, err } @@ -207,7 +207,7 @@ func (gc *GuestConnection) DeleteContainerState(ctx context.Context, cid string) RequestBase: makeRequest(ctx, cid), } var resp prot.ResponseBase - return gc.brdg.RPC(ctx, prot.RpcDeleteContainerState, &req, &resp, false) + return gc.brdg.RPC(ctx, prot.RPCDeleteContainerState, &req, &resp, false) } // Close terminates the guest connection. It is undefined to call any other diff --git a/internal/gcs/guestconnection_test.go b/internal/gcs/guestconnection_test.go index 8c505be54b..6a72cb8a3f 100644 --- a/internal/gcs/guestconnection_test.go +++ b/internal/gcs/guestconnection_test.go @@ -56,8 +56,8 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { } return err } - switch proc := prot.RpcProc(typ &^ prot.MsgTypeRequest); proc { - case prot.RpcNegotiateProtocol: + switch proc := prot.RPCProc(typ &^ prot.MsgTypeRequest); proc { + case prot.RPCNegotiateProtocol: err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.NegotiateProtocolResponse{ Version: protocolVersion, Capabilities: prot.GcsCapabilities{ @@ -67,12 +67,12 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { if err != nil { return err } - case prot.RpcCreate: + case prot.RPCCreate: err := sendJSON(t, rw, prot.MsgTypeResponse|prot.MsgType(proc), id, &prot.ContainerCreateResponse{}) if err != nil { return err } - case prot.RpcExecuteProcess: + case prot.RPCExecuteProcess: var req prot.ContainerExecuteProcess var params baseProcessParams req.Settings.ProcessParameters.Value = ¶ms @@ -118,9 +118,9 @@ func simpleGcsLoop(t *testing.T, rw io.ReadWriter) error { if err != nil { return err } - case prot.RpcWaitForProcess: + case prot.RPCWaitForProcess: // nothing - case prot.RpcShutdownForced: + case prot.RPCShutdownForced: var req prot.RequestBase err = json.Unmarshal(b, &req) if err != nil { diff --git a/internal/gcs/process.go b/internal/gcs/process.go index f5f013cd57..91d3a87faa 100644 --- a/internal/gcs/process.go +++ b/internal/gcs/process.go @@ -56,7 +56,7 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac req := prot.ContainerExecuteProcess{ RequestBase: makeRequest(ctx, cid), Settings: prot.ExecuteProcessSettings{ - ProcessParameters: prot.AnyInString{params}, + ProcessParameters: prot.AnyInString{Value: params}, }, } @@ -102,7 +102,7 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac } var resp prot.ContainerExecuteProcessResponse - err = gc.brdg.RPC(ctx, prot.RpcExecuteProcess, &req, &resp, false) + err = gc.brdg.RPC(ctx, prot.RPCExecuteProcess, &req, &resp, false) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (gc *GuestConnection) exec(ctx context.Context, cid string, params interfac ProcessID: p.id, TimeoutInMs: 0xffffffff, } - p.waitCall, err = gc.brdg.AsyncRPC(ctx, prot.RpcWaitForProcess, &waitReq, &p.waitResp) + p.waitCall, err = gc.brdg.AsyncRPC(ctx, prot.RPCWaitForProcess, &waitReq, &p.waitResp) if err != nil { return nil, fmt.Errorf("failed to wait on process, leaking process: %w", err) } @@ -228,7 +228,7 @@ func (p *Process) ResizeConsole(ctx context.Context, width, height uint16) (err Width: width, } var resp prot.ResponseBase - return p.gc.brdg.RPC(ctx, prot.RpcResizeConsole, &req, &resp, true) + return p.gc.brdg.RPC(ctx, prot.RPCResizeConsole, &req, &resp, true) } // Signal sends a signal to the process, returning whether it was delivered. @@ -248,7 +248,7 @@ func (p *Process) Signal(ctx context.Context, options interface{}) (_ bool, err var resp prot.ResponseBase // FUTURE: SIGKILL is idempotent and can safely be cancelled, but this interface // does currently make it easy to determine what signal is being sent. - err = p.gc.brdg.RPC(ctx, prot.RpcSignalProcess, &req, &resp, false) + err = p.gc.brdg.RPC(ctx, prot.RPCSignalProcess, &req, &resp, false) if err != nil { if uint32(resp.Result) != hrNotFound { return false, err diff --git a/internal/gcs/prot/protocol.go b/internal/gcs/prot/protocol.go index 0555d71c5f..6be8f95482 100644 --- a/internal/gcs/prot/protocol.go +++ b/internal/gcs/prot/protocol.go @@ -13,9 +13,37 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) -// LinuxGcsVsockPort is the vsock port number that the Linux GCS will -// connect to. -const LinuxGcsVsockPort = 0x40000000 +const ( + HdrSize = 16 + HdrOffType = 0 + HdrOffSize = 4 + HdrOffID = 8 + + // maxMsgSize is the maximum size of an incoming message. This is not + // enforced by the guest today but some maximum must be set to avoid + // unbounded allocations. + MaxMsgSize = 0x10000 + + // LinuxGcsVsockPort is the vsock port number that the Linux GCS will + // connect to. + LinuxGcsVsockPort = 0x40000000 +) + +// e0e16197-dd56-4a10-9195-5ee7a155a838 +var HvGUIDLoopback = guid.GUID{ + Data1: 0xe0e16197, + Data2: 0xdd56, + Data3: 0x4a10, + Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38}, +} + +// a42e7cda-d03f-480c-9cc2-a4de20abb878 +var HvGUIDParent = guid.GUID{ + Data1: 0xa42e7cda, + Data2: 0xd03f, + Data3: 0x480c, + Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78}, +} // WindowsGcsHvsockServiceID is the hvsock service ID that the Windows GCS // will connect to. @@ -26,6 +54,15 @@ var WindowsGcsHvsockServiceID = guid.GUID{ Data4: [8]uint8{0x85, 0x6b, 0x62, 0x45, 0xe6, 0x9f, 0x46, 0x20}, } +// WindowsSidecarGcsHvsockServiceID is the hvsock service ID that the Windows GCS +// sidecar will connect to. This is only used in the confidential mode. +var WindowsSidecarGcsHvsockServiceID = guid.GUID{ + Data1: 0xae8da506, + Data2: 0xa019, + Data3: 0x4553, + Data4: [8]uint8{0xa5, 0x2b, 0x90, 0x2b, 0xc0, 0xfa, 0x04, 0x11}, +} + // WindowsGcsHvHostID is the hvsock address for the parent of the VM running the GCS var WindowsGcsHvHostID = guid.GUID{ Data1: 0x894cc2d6, @@ -46,57 +83,57 @@ func (a *AnyInString) UnmarshalText(b []byte) error { return json.Unmarshal(b, &a.Value) } -type RpcProc uint32 +type RPCProc uint32 const ( - RpcCreate RpcProc = (iota+1)<<8 | 1 - RpcStart - RpcShutdownGraceful - RpcShutdownForced - RpcExecuteProcess - RpcWaitForProcess - RpcSignalProcess - RpcResizeConsole - RpcGetProperties - RpcModifySettings - RpcNegotiateProtocol - RpcDumpStacks - RpcDeleteContainerState - RpcUpdateContainer - RpcLifecycleNotification + RPCCreate RPCProc = (iota+1)<<8 | 1 + RPCStart + RPCShutdownGraceful + RPCShutdownForced + RPCExecuteProcess + RPCWaitForProcess + RPCSignalProcess + RPCResizeConsole + RPCGetProperties + RPCModifySettings + RPCNegotiateProtocol + RPCDumpStacks + RPCDeleteContainerState + RPCUpdateContainer + RPCLifecycleNotification ) -func (rpc RpcProc) String() string { +func (rpc RPCProc) String() string { switch rpc { - case RpcCreate: + case RPCCreate: return "Create" - case RpcStart: + case RPCStart: return "Start" - case RpcShutdownGraceful: + case RPCShutdownGraceful: return "ShutdownGraceful" - case RpcShutdownForced: + case RPCShutdownForced: return "ShutdownForced" - case RpcExecuteProcess: + case RPCExecuteProcess: return "ExecuteProcess" - case RpcWaitForProcess: + case RPCWaitForProcess: return "WaitForProcess" - case RpcSignalProcess: + case RPCSignalProcess: return "SignalProcess" - case RpcResizeConsole: + case RPCResizeConsole: return "ResizeConsole" - case RpcGetProperties: + case RPCGetProperties: return "GetProperties" - case RpcModifySettings: + case RPCModifySettings: return "ModifySettings" - case RpcNegotiateProtocol: + case RPCNegotiateProtocol: return "NegotiateProtocol" - case RpcDumpStacks: + case RPCDumpStacks: return "DumpStacks" - case RpcDeleteContainerState: + case RPCDeleteContainerState: return "DeleteContainerState" - case RpcUpdateContainer: + case RPCUpdateContainer: return "UpdateContainer" - case RpcLifecycleNotification: + case RPCLifecycleNotification: return "LifecycleNotification" default: return "0x" + strconv.FormatUint(uint64(rpc), 16) @@ -133,7 +170,7 @@ func (typ MsgType) String() string { default: return fmt.Sprintf("%#x", uint32(typ)) } - s += RpcProc(typ &^ MsgTypeMask).String() + s += RPCProc(typ &^ MsgTypeMask).String() return s + ")" } diff --git a/internal/guest/prot/protocol.go b/internal/guest/prot/protocol.go index 88993ae563..576ac5e5f1 100644 --- a/internal/guest/prot/protocol.go +++ b/internal/guest/prot/protocol.go @@ -501,9 +501,9 @@ type ResourceModificationRequestResponse struct { Settings interface{} `json:",omitempty"` } -// ContainerModifySettings is the message from the HCS specifying how a certain +// containerModifySettings is the message from the HCS specifying how a certain // container resource should be modified. -type ContainerModifySettings struct { +type containerModifySettings struct { MessageBase Request interface{} } @@ -512,9 +512,9 @@ type ContainerModifySettings struct { // ContainerModifySettings message. This function is required because properties // such as `Settings` can be of many types identified by the `ResourceType` and // require dynamic unmarshalling. -func UnmarshalContainerModifySettings(b []byte) (*ContainerModifySettings, error) { +func UnmarshalContainerModifySettings(b []byte) (*containerModifySettings, error) { // Unmarshal the message. - var request ContainerModifySettings + var request containerModifySettings var requestRawSettings json.RawMessage request.Request = &requestRawSettings if err := commonutils.UnmarshalJSONWithHresult(b, &request); err != nil { diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 65a521d5da..6dddc6f41d 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -22,14 +22,6 @@ import ( "github.com/Microsoft/cosesign1go/pkg/cosesign1" didx509resolver "github.com/Microsoft/didx509go/pkg/did-x509-resolver" - "github.com/Microsoft/hcsshim/pkg/annotations" - "github.com/Microsoft/hcsshim/pkg/securitypolicy" - cgroup1stats "github.com/containerd/cgroups/v3/cgroup1/stats" - "github.com/mattn/go-shellwords" - "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/debug" "github.com/Microsoft/hcsshim/internal/guest/policy" @@ -49,6 +41,14 @@ import ( "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/verity" + "github.com/Microsoft/hcsshim/pkg/annotations" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + cgroup1stats "github.com/containerd/cgroups/v3/cgroup1/stats" + "github.com/mattn/go-shellwords" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // UVMContainerID is the ContainerID that will be sent on any prot.MessageBase diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index 89c2003d7e..b848017c13 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -4,6 +4,7 @@ import ( "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/Microsoft/go-winio/pkg/guid" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) @@ -33,6 +34,10 @@ const ( // ResourceTypeCombinedLayers is the modify resource type for combined // layers ResourceTypeCombinedLayers guestrequest.ResourceType = "CombinedLayers" + // ResourceTypeCWCOWCombinedLayers is the modify resource type for combined + // layers call for cwcow cases. This resource type wraps containerID around + // ResourceTypeCombinedLayers. + ResourceTypeCWCOWCombinedLayers guestrequest.ResourceType = "CWCOWCombinedLayers" // ResourceTypeVPMemDevice is the modify resource type for VPMem devices ResourceTypeVPMemDevice guestrequest.ResourceType = "VPMemDevice" // ResourceTypeVPCIDevice is the modify resource type for vpci devices @@ -46,6 +51,12 @@ const ( ResourceTypeSecurityPolicy guestrequest.ResourceType = "SecurityPolicy" // ResourceTypePolicyFragment is the modify resource type for injecting policy fragments. ResourceTypePolicyFragment guestrequest.ResourceType = "SecurityPolicyFragment" + // ResourceTypeWCOWBlockCims is the modify resource type for mounting block cims for hyperv + // wcow containers. + ResourceTypeWCOWBlockCims guestrequest.ResourceType = "WCOWBlockCims" + // ResourceTypeMappedVirtualDiskForContainerScratch is the modify resource type + // specifically for refs formatting and mounting scratch vhds for c-wcow cases only. + ResourceTypeMappedVirtualDiskForContainerScratch guestrequest.ResourceType = "MappedVirtualDiskForContainerScratch" ) // This class is used by a modify request to add or remove a combined layers @@ -67,6 +78,11 @@ type WCOWCombinedLayers struct { ScratchPath string `json:"ScratchPath,omitempty"` } +type CWCOWCombinedLayers struct { + ContainerID string `json:"ContainerID,omitempty"` + CombinedLayers WCOWCombinedLayers `json:"CombinedLayers,omitempty"` +} + // Defines the schema for hosted settings passed to GCS and/or OpenGCS // SCSIDevice represents a SCSI device that is attached to the system. @@ -92,6 +108,18 @@ type LCOWMappedVirtualDisk struct { Filesystem string `json:"Filesystem,omitempty"` } +type BlockCIMDevice struct { + CimName string + Lun int32 +} + +type WCOWBlockCIMMounts struct { + // BlockCIMs should be ordered from merged CIM followed by Layer n .. layer 1 + BlockCIMs []BlockCIMDevice `json:"BlockCIMs,omitempty"` + VolumeGUID guid.GUID `json:"VolumeGUID,omitempty"` + MountFlags uint32 `json:"MountFlags,omitempty"` +} + type WCOWMappedVirtualDisk struct { ContainerPath string `json:"ContainerPath,omitempty"` Lun int32 `json:"Lun,omitempty"` @@ -207,3 +235,14 @@ type LCOWConfidentialOptions struct { type LCOWSecurityPolicyFragment struct { Fragment string `json:"Fragment,omitempty"` } + +type WCOWConfidentialOptions struct { + EnforcerType string `json:"EnforcerType,omitempty"` + EncodedSecurityPolicy string `json:"EncodedSecurityPolicy,omitempty"` + // Optional security policy + WCOWSecurityPolicy string + // Set when there is a security policy to apply on actual SNP hardware, use this rathen than checking the string length + WCOWSecurityPolicyEnabled bool + // Set which security policy enforcer to use (open door or rego). This allows for better fallback mechanic. + WCOWSecurityPolicyEnforcer string +} diff --git a/pkg/securitypolicy/securitypolicy_uvmpath_linux.go b/pkg/securitypolicy/securitypolicy_uvmpath_linux.go new file mode 100644 index 0000000000..4fdfaafdcc --- /dev/null +++ b/pkg/securitypolicy/securitypolicy_uvmpath_linux.go @@ -0,0 +1,34 @@ +//go:build linux +// +build linux + +package securitypolicy + +import ( + "strings" + + specInternal "github.com/Microsoft/hcsshim/internal/guest/spec" + "github.com/Microsoft/hcsshim/internal/guestpath" +) + +// This is being used by StandEnforcer. +// substituteUVMPath substitutes mount prefix to an appropriate path inside +// UVM. At policy generation time, it's impossible to tell what the sandboxID +// will be, so the prefix substitution needs to happen during runtime. +func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { + if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { + m.Source = specInternal.SandboxMountSource(sandboxID, m.Source) + } else if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { + m.Source = specInternal.HugePagesMountSource(sandboxID, m.Source) + } + return m +} + +// SandboxMountsDir returns sandbox mounts directory inside UVM/host. +func SandboxMountsDir(sandboxID string) string { + return specInternal.SandboxMountsDir((sandboxID)) +} + +// HugePagesMountsDir returns hugepages mounts directory inside UVM. +func HugePagesMountsDir(sandboxID string) string { + return specInternal.HugePagesMountsDir(sandboxID) +} diff --git a/pkg/securitypolicy/securitypolicy_uvmpath_windows.go b/pkg/securitypolicy/securitypolicy_uvmpath_windows.go new file mode 100644 index 0000000000..cc6949fa68 --- /dev/null +++ b/pkg/securitypolicy/securitypolicy_uvmpath_windows.go @@ -0,0 +1,24 @@ +//go:build windows +// +build windows + +package securitypolicy + +// This is being used by StandEnforcer and is a no-op for windows. +// substituteUVMPath substitutes mount prefix to an appropriate path inside +// UVM. At policy generation time, it's impossible to tell what the sandboxID +// will be, so the prefix substitution needs to happen during runtime. +func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { + //no-op for windows + _ = sandboxID + return m +} + +// SandboxMountsDir returns sandbox mounts directory inside UVM/host. +func SandboxMountsDir(sandboxID string) string { + return "" +} + +// HugePagesMountsDir returns hugepages mounts directory inside UVM. +func HugePagesMountsDir(sandboxID string) string { + return "" +} diff --git a/pkg/securitypolicy/securitypolicyenforcer.go b/pkg/securitypolicy/securitypolicyenforcer.go index 5806eb7544..8792966229 100644 --- a/pkg/securitypolicy/securitypolicyenforcer.go +++ b/pkg/securitypolicy/securitypolicyenforcer.go @@ -1,6 +1,3 @@ -//go:build linux -// +build linux - package securitypolicy import ( @@ -10,15 +7,11 @@ import ( "fmt" "regexp" "strconv" - "strings" "sync" "syscall" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" - - specGuest "github.com/Microsoft/hcsshim/internal/guest/spec" - "github.com/Microsoft/hcsshim/internal/guestpath" ) type createEnforcerFunc func(base64EncodedPolicy string, criMounts, criPrivilegedMounts []oci.Mount, maxErrorMessageLength int) (SecurityPolicyEnforcer, error) @@ -89,12 +82,15 @@ type SecurityPolicyEnforcer interface { GetUserInfo(containerID string, spec *oci.Process) (IDName, []IDName, string, error) } +//nolint type stringSet map[string]struct{} +//nolint func (s stringSet) add(item string) { s[item] = struct{}{} } +//nolint func (s stringSet) contains(item string) bool { _, contains := s[item] return contains @@ -856,18 +852,6 @@ func (c *securityPolicyContainer) matchMount(sandboxID string, m oci.Mount) (err return fmt.Errorf("mount is not allowed by policy: %+v", m) } -// substituteUVMPath substitutes mount prefix to an appropriate path inside -// UVM. At policy generation time, it's impossible to tell what the sandboxID -// will be, so the prefix substitution needs to happen during runtime. -func substituteUVMPath(sandboxID string, m mountInternal) mountInternal { - if strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) { - m.Source = specGuest.SandboxMountSource(sandboxID, m.Source) - } else if strings.HasPrefix(m.Source, guestpath.HugePagesMountPrefix) { - m.Source = specGuest.HugePagesMountSource(sandboxID, m.Source) - } - return m -} - func stringSlicesEqual(slice1, slice2 []string) bool { if len(slice1) != len(slice2) { return false diff --git a/test/gcs/container_test.go b/test/gcs/container_test.go index caf6ea2f77..4f3db015c0 100644 --- a/test/gcs/container_test.go +++ b/test/gcs/container_test.go @@ -12,7 +12,7 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "golang.org/x/sync/errgroup" - "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/bridgeutils/gcserr" "github.com/Microsoft/hcsshim/internal/guest/stdio" testoci "github.com/Microsoft/hcsshim/test/internal/oci"