diff --git a/.github/actions/build-alpha-release-notes/action.yml b/.github/actions/build-alpha-release-notes/action.yml index 0ab23c059..c9e68ad36 100644 --- a/.github/actions/build-alpha-release-notes/action.yml +++ b/.github/actions/build-alpha-release-notes/action.yml @@ -33,7 +33,7 @@ runs: sed 's/^/- /' } > docs/release.md - name: Upload release notes - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: release-notes path: docs/release.md diff --git a/.github/actions/build-mokapi-image/action.yml b/.github/actions/build-mokapi-image/action.yml index 0de29cdf9..eafbf7d80 100644 --- a/.github/actions/build-mokapi-image/action.yml +++ b/.github/actions/build-mokapi-image/action.yml @@ -30,7 +30,7 @@ runs: run: docker save -o /tmp/mokapi.tar ${{ inputs.image-name }} mokapi/mokapi:alpha shell: bash - name: Upload image - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ inputs.artifact-name }} path: /tmp/mokapi.tar \ No newline at end of file diff --git a/.github/actions/publish-docker-image/action.yml b/.github/actions/publish-docker-image/action.yml index 33f3a3ef2..d843b09e9 100644 --- a/.github/actions/publish-docker-image/action.yml +++ b/.github/actions/publish-docker-image/action.yml @@ -18,7 +18,7 @@ runs: steps: - name: Download image id: download - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ inputs.artifact-name }} path: /tmp diff --git a/.github/actions/publish-test-report/action.yml b/.github/actions/publish-test-report/action.yml index 7ddd1d774..9a386db9a 100644 --- a/.github/actions/publish-test-report/action.yml +++ b/.github/actions/publish-test-report/action.yml @@ -22,7 +22,7 @@ runs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" shell: bash - name: Download Report - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ inputs.artifact-test-report }} path: gh-pages/${{ inputs.test-report-path }} diff --git a/.github/actions/publish-website/action.yml b/.github/actions/publish-website/action.yml index 1d94c048f..b978145cb 100644 --- a/.github/actions/publish-website/action.yml +++ b/.github/actions/publish-website/action.yml @@ -19,7 +19,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 24 registry-url: 'https://registry.npmjs.org' @@ -31,7 +31,7 @@ runs: token: ${{ inputs.token }} - name: Download image id: download - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ env.image-artifact-name }} path: /tmp @@ -71,7 +71,7 @@ runs: shell: bash - name: Upload test results if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: website-test-report path: webui/playwright-report diff --git a/.github/actions/run-frontend-tests/action.yml b/.github/actions/run-frontend-tests/action.yml index a5354cb88..b98eaf46c 100644 --- a/.github/actions/run-frontend-tests/action.yml +++ b/.github/actions/run-frontend-tests/action.yml @@ -15,7 +15,7 @@ runs: steps: - name: Download image id: download - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: ${{ inputs.artifact-image-name }} path: /tmp @@ -23,9 +23,9 @@ runs: run: docker load --input /tmp/mokapi.tar shell: bash - name: Run mokapi image - run: docker run --name mokapi --rm -d -p 8090:8090 -p 8091:8091 --mount type=bind,source=$(pwd)/webui/e2e/mocks,target=/data --env MOKAPI_Log_Level=Debug --env MOKAPI_Api_Port=8091 --env MOKAPI_Api_Path=/mokapi --env MOKAPI_Providers_File_Directory=/data ${{ inputs.image-name }} + run: docker run --name mokapi --rm -d -p 8090:8090 -p 8091:8091 --mount type=bind,source=$(pwd)/webui,target=/data --env MOKAPI_Log_Level=Debug --env MOKAPI_Api_Port=8091 --env MOKAPI_Api_Path=/mokapi --env MOKAPI_Providers_File_Directory=/data/e2e/mocks ${{ inputs.image-name }} shell: bash - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 24 - name: Install and Build @@ -46,7 +46,7 @@ runs: shell: bash - name: Upload test results if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ inputs.artifact-test-report }} path: ./webui/playwright-report @@ -56,7 +56,7 @@ runs: shell: bash - name: Archive Mokapi logs if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: mokapi-test-logs path: /var/tmp/mokapi.log diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 054dfe519..aac4626f4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: go-version-file: 'go.mod' - name: Setup node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 26 - name: build webui diff --git a/.github/workflows/osv-scanner.yml b/.github/workflows/osv-scanner.yml index 6a9fbdb41..34f928233 100644 --- a/.github/workflows/osv-scanner.yml +++ b/.github/workflows/osv-scanner.yml @@ -26,7 +26,7 @@ permissions: jobs: scan-scheduled: if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} - uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc" # v2.6.0 + uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@b1d267aea5211fa648d9d442fa64b90dbdcbdd36" # v2.7.0 with: # Example of specifying custom arguments scan-args: |- diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8393323f5..806788deb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - uses: ./.github/actions/build-release-notes env: GH_TOKEN: ${{ github.token }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 24 registry-url: 'https://registry.npmjs.org' @@ -44,7 +44,7 @@ jobs: - name: Build run: task build-vue-app - name: Upload UI artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: dashboard path: webui/dist @@ -60,7 +60,7 @@ jobs: with: fetch-depth: 0 - name: Download dashboard - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: dashboard path: webui/dist @@ -77,7 +77,7 @@ jobs: with: go-version-file: 'go.mod' - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 + uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7 with: version: latest args: release --clean @@ -94,7 +94,7 @@ jobs: with: fetch-depth: 0 - name: Download dashboard - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: dashboard path: webui/dist @@ -127,7 +127,7 @@ jobs: -product-version="$($versionParts[0]).$($versionParts[1]).$($versionParts[2])" - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 + uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7 with: version: latest args: release -f .goreleaser.windows.yml --clean @@ -151,7 +151,7 @@ jobs: - uses: ./.github/actions/build-release-notes env: GH_TOKEN: ${{ github.token }} - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 24 registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64af30d73..5695d7ff3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: sparse-checkout: 'npm' - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7 with: node-version: 24 - name: Run TypeScript Compiler (tsc) diff --git a/acceptance/kafka_test.go b/acceptance/kafka_test.go index f30c601e8..8eb710e2f 100644 --- a/acceptance/kafka_test.go +++ b/acceptance/kafka_test.go @@ -1,7 +1,9 @@ package acceptance import ( + "encoding/json" "fmt" + "mokapi/api" "mokapi/config/static" "mokapi/kafka" "mokapi/kafka/fetch" @@ -51,7 +53,7 @@ func (suite *KafkaSuite) TestApi() { "name": "petstore.order-event", "metrics": map[string]any{ // skip timestamp check "kafka_message_timestamp" - "kafka_messages_total": float64(1), + "messages_total": float64(1), }, }}, } @@ -154,8 +156,20 @@ func (suite *KafkaSuite) TestKafkaEventAndMetrics() { try.BodyContains(`kafka_messages_total{service=\"A sample AsyncApi Kafka streaming api\",topic=\"petstore.order-event\"}","value":1}`), ) + var eventId string // test kafka events, header added by JavaScript event handler try.GetRequest(suite.T(), fmt.Sprintf("http://127.0.0.1:%d/api/events?namespace=kafka", suite.cfg.Api.Port), nil, + try.AssertBody(func(t *testing.T, body string) { + var data api.EventPage + err := json.Unmarshal([]byte(body), &data) + require.NoError(t, err) + require.Len(t, data.Items, 2) + // first event does not define a header specification therefor header only contains binary + eventId = data.Items[1].Id + }), + ) + + try.GetRequest(suite.T(), fmt.Sprintf("http://127.0.0.1:%d/api/events/"+eventId, suite.cfg.Api.Port), nil, try.BodyContains(`"headers":{"foo":{"value":"bar","binary":"YmFy"}`), try.BodyContains(`"messageId":"order"`), ) diff --git a/acceptance/ldap/ldap.yml b/acceptance/ldap/ldap.yml index 961619fad..a79dafd84 100644 --- a/acceptance/ldap/ldap.yml +++ b/acceptance/ldap/ldap.yml @@ -1,6 +1,6 @@ ldap: 1.0.0 info: - name: Futurama + title: Futurama server: address: "0.0.0.0:8389" rootDomainNamingContext: DC=mokapi,DC=io diff --git a/acceptance/ldap_test.go b/acceptance/ldap_test.go index e61a007cc..c0642e5e7 100644 --- a/acceptance/ldap_test.go +++ b/acceptance/ldap_test.go @@ -69,7 +69,7 @@ func (suite *LdapSuite) TestLog() { try.GetRequest(suite.T(), fmt.Sprintf("http://127.0.0.1:%v/api/events?namespace=ldap", suite.cfg.Api.Port), nil, try.HasStatusCode(http.StatusOK), - try.BodyContains(`:{"request":{"operation":"Search","baseDN":"","scope":"WholeSubtree","dereferencePolicy":0,"sizeLimit":0,"timeLimit":0,"typesOnly":false,"filter":"(objectClass=user)","attributes":["mail"],"controls":null}`), + try.BodyContains(`"criteria":"(objectClass=user)"`), ) } diff --git a/api/handler.go b/api/handler.go index 0b10f1be0..4dc0b2628 100644 --- a/api/handler.go +++ b/api/handler.go @@ -111,6 +111,7 @@ func New(app *runtime.App, config static.Api) Handler { h.setupMqtt() h.setupWebsocket() h.setupFaker() + h.setupEvents() return h } @@ -163,8 +164,6 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { h.getDashboard(w, r) case strings.HasPrefix(p, "/api/metrics"): h.getMetrics(w, r) - case strings.HasPrefix(p, "/api/events"): - h.getEvents(w, r) case p == "/api/schema/example": h.getExampleData(w, r) case p == "/api/schema/validate": diff --git a/api/handler_events.go b/api/handler_events.go index 9085e4d3f..7003cf147 100644 --- a/api/handler_events.go +++ b/api/handler_events.go @@ -1,32 +1,372 @@ package api import ( + "fmt" + "mokapi/providers/openapi" "mokapi/runtime/events" "net/http" + "net/netip" + "net/url" + "regexp" + "strconv" "strings" + "time" + + "github.com/gorilla/mux" ) +type httpFilter = func(d *openapi.HttpLog) bool + +type EventInfo struct { + Id string `json:"id"` + Traits events.Traits `json:"traits"` + Time time.Time `json:"time"` + Data any `json:"data,omitempty"` +} + +type EventPage struct { + Items []EventInfo `json:"items"` + Page int `json:"page"` + Size int `json:"size"` + Total int `json:"total"` +} + +type eventQuery struct { + traits events.Traits + http map[string][]string + page int + size int +} + +const maxEventPageSize = 1000 + +func (h *handler) setupEvents() { + r := h.router.PathPrefix("/api/events").Subrouter() + + r.HandleFunc("", h.getEvents).Methods(http.MethodGet) + r.HandleFunc("/{id}", h.getEvent).Methods(http.MethodGet) +} + func (h *handler) getEvents(w http.ResponseWriter, r *http.Request) { - var data interface{} + q, err := parseEventFilter(r.URL.Query()) + if err != nil { + writeError(w, err, http.StatusBadRequest) + return + } + + result := EventPage{Page: q.page, Size: q.size} + + data := h.app.Events.GetEvents(q.traits) + data, err = filterEvents(data, q) + if err != nil { + writeError(w, err, http.StatusBadRequest) + return + } + + result.Total = len(data) + start, end := paginate(q.page, q.size, len(data)) + + result.Items = make([]EventInfo, 0, min(q.size, len(data))) + for _, e := range data[start:end] { + result.Items = append(result.Items, eventInfo(e)) + } + + w.Header().Set("Content-Type", "application/json") + writeJsonBody(w, result) +} + +func (h *handler) getEvent(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + e := h.app.Events.GetEvent(vars["id"]) + if !e.IsValid() { + w.WriteHeader(404) + return + } + + w.Header().Set("Content-Type", "application/json") + writeJsonBody(w, e) +} + +func paginate(page int, size int, len int) (int, int) { + start := min((page-1)*size, len) + return start, min(start+size, len) +} + +func filterEvents(data []events.Event, q eventQuery) ([]events.Event, error) { + var result []events.Event + + filterHttp, err := matchHttpEvent(q) + if err != nil { + return nil, err + } + + for _, e := range data { + switch d := e.Data.(type) { + case *openapi.HttpLog: + if filterHttp == nil || filterHttp(d) { + result = append(result, e) + } + default: + result = append(result, e) + } + } + return result, nil +} + +func matchHttpEvent(q eventQuery) (httpFilter, error) { + if q.http == nil { + return nil, nil + } + + var filters []func(d *openapi.HttpLog) bool + for k, v := range q.http { + switch k { + case "method": + filters = append(filters, func(d *openapi.HttpLog) bool { + match := false + for _, m := range v { + if strings.EqualFold(d.Request.Method, m) { + match = true + } + } + return match + }) + case "url": + if v[0] != "" { + regex, err := regexp.Compile(v[0]) + if err != nil { + return nil, fmt.Errorf("invalid url regex: %w", err) + } + filters = append(filters, func(d *openapi.HttpLog) bool { + return regex.MatchString(d.Request.Url) + }) + } + case "requestHeader": + return func(d *openapi.HttpLog) bool { + for _, val := range v { + kv := strings.SplitN(val, ":", 2) + match := false + for _, p := range d.Request.Parameters { + if p.Type != string(openapi.ParameterHeader) { + continue + } + if p.Name == kv[0] { + if len(kv) > 1 { + if p.Raw != nil && kv[1] == *p.Raw { + match = true + break + } + } + } + } + if !match { + return false + } + } + return true + }, nil + case "responseHeader": + return func(d *openapi.HttpLog) bool { + for _, val := range v { + kv := strings.SplitN(val, ":", 2) + hv, ok := d.Response.Headers[kv[0]] + if len(kv) == 1 && !ok { + return false + } else if kv[1] != hv { + return false + } + } + return true + }, nil + case "statusCode": + var values []string + for _, val := range v { + values = append(values, strings.Split(val, ",")...) + } + var checks []httpFilter + for i, val := range values { + if val == "" { + continue + } + + not := false + match := false + if val[0] == '!' { + not = true + val = val[1:] + } + + if len(val) == 3 && strings.HasSuffix(val, "xx") { + val = fmt.Sprintf("%c00-%c99", val[0], val[0]) + } + if strings.Contains(val, "-") { + parts := strings.Split(val, "-") + if len(parts) != 2 { + continue + } + minVal, err := strconv.Atoi(parts[0]) + if err != nil { + return nil, fmt.Errorf("invalid min value in '%s'", values[i]) + } + maxValue, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("invalid max value in '%s'", values[i]) + } + checks = append(checks, func(d *openapi.HttpLog) bool { + match = d.Response.StatusCode >= minVal && d.Response.StatusCode <= maxValue + return not != match + }) + } else { + expect, err := strconv.Atoi(val) + if err != nil { + return nil, fmt.Errorf("invalid value in '%s'", values[i]) + } + checks = append(checks, func(d *openapi.HttpLog) bool { + match = d.Response.StatusCode == expect + return not != match + }) + } + + } + + return func(d *openapi.HttpLog) bool { + for _, check := range checks { + if !check(d) { + return false + } + } + return true + }, nil + case "clientIP": + var values []string + for _, val := range v { + values = append(values, strings.Split(val, ",")...) + } + var checks []httpFilter + for i, val := range values { + if val == "" { + continue + } + + not := false + match := false + if val[0] == '!' { + not = true + val = val[1:] + } - segments := strings.Split(r.URL.Path, "/") - if len(segments) == 3 { - traits := events.NewTraits() - for k := range r.URL.Query() { - traits.With(k, r.URL.Query().Get(k)) + if strings.Contains(val, "/") { + prefix, err := netip.ParsePrefix(val) + if err != nil { + return nil, fmt.Errorf("invalid prefix in '%s'", values[i]) + } + checks = append(checks, func(d *openapi.HttpLog) bool { + ip := netip.MustParseAddr(d.ClientIP) + return not != prefix.Contains(ip) + }) + } else { + checks = append(checks, func(d *openapi.HttpLog) bool { + match = d.ClientIP == val + return not != match + }) + } + + } + + return func(d *openapi.HttpLog) bool { + for _, check := range checks { + if !check(d) { + return false + } + } + return true + }, nil + } + } + if len(filters) == 0 { + return nil, nil + } + return func(d *openapi.HttpLog) bool { + for _, f := range filters { + if b := f(d); !b { + return false + } } + return true + }, nil +} + +func parseEventFilter(v url.Values) (eventQuery, error) { + q := eventQuery{ + page: 1, + size: 25, + traits: events.NewTraits(), + } + var err error - data = h.app.Events.GetEvents(traits) - } else { - e := h.app.Events.GetEvent(segments[3]) - if e.IsValid() { - data = e - } else { - w.WriteHeader(404) - return + s := v.Get("page") + if s != "" { + q.page, err = strconv.Atoi(s) + if err != nil || q.page < 1 { + return q, fmt.Errorf("invalid query parameter 'page'") } } - w.Header().Set("Content-Type", "application/json") - writeJsonBody(w, data) + s = v.Get("size") + if s != "" { + q.size, err = strconv.Atoi(s) + if err != nil || q.size < 1 || q.size > maxEventPageSize { + return q, fmt.Errorf("invalid query parameter 'size'") + } + } + + q.http = deepObject(v, "http") + + /*if h := deepObject(q, "http"); len(h["method"]) > 0 { + f.HTTPMethod = strings.ToUpper(h["method"][0]) + }*/ + for name, values := range deepObject(v, "traits") { + q.traits[name] = values[0] + } + return q, nil +} + +func deepObject(q url.Values, name string) map[string][]string { + prefix := name + "[" + result := make(map[string][]string) + + for key, values := range q { + if !strings.HasPrefix(key, prefix) || !strings.HasSuffix(key, "]") { + continue + } + prop := key[len(prefix) : len(key)-1] + if prop == "" || strings.ContainsAny(prop, "[]") { + continue + } + values = splitQueryValues(values) + result[prop] = append(result[prop], values...) + } + return result +} + +func splitQueryValues(values []string) []string { + var result []string + for _, v := range values { + for part := range strings.SplitSeq(v, ",") { + if part = strings.TrimSpace(part); part != "" { + result = append(result, part) + } + } + } + return result +} + +func eventInfo(e events.Event) EventInfo { + return EventInfo{ + Id: e.Id, + Traits: e.Traits, + Time: e.Time, + Data: e.Data.Info(), + } } diff --git a/api/handler_events_test.go b/api/handler_events_test.go index a45fb35eb..71b38f212 100644 --- a/api/handler_events_test.go +++ b/api/handler_events_test.go @@ -1,12 +1,16 @@ -package api +package api_test import ( "context" "encoding/json" "fmt" + "mokapi/api" "mokapi/config/dynamic/dynamictest" "mokapi/config/static" - "mokapi/providers/asyncapi3/kafka/store" + kafka "mokapi/providers/asyncapi3/kafka/store" + mqtt "mokapi/providers/asyncapi3/mqtt/store" + "mokapi/providers/asyncapi3/websocket" + "mokapi/providers/directory" "mokapi/providers/openapi" "mokapi/runtime" "mokapi/runtime/events" @@ -14,6 +18,7 @@ import ( "mokapi/try" "net/http" "net/http/httptest" + "slices" "testing" "time" @@ -21,6 +26,18 @@ import ( ) func TestHandler_Events(t *testing.T) { + + newHttpEvent := func(t *testing.T, r *http.Request, traits events.Traits, sm *events.StoreManager) events.Event { + ctx, err := openapi.NewLogEventContext(r, false, traits) + require.NoError(t, err) + l, _ := openapi.LogEventFromContext(ctx) + err = sm.Push(l, traits) + require.NoError(t, err) + evts := sm.GetEvents(traits) + // events are LIFO + return evts[0] + } + testcases := []struct { name string fn func(t *testing.T, h http.Handler, sm *events.StoreManager) @@ -36,27 +53,457 @@ func TestHandler_Events(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[]`)) + try.HasBody(`{"items":[],"page":1,"size":25,"total":0}`)) + }, + }, + { + name: "paging", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(5, events.NewTraits().WithNamespace("http")) + var ids []string + for i := 1; i <= 5; i++ { + r := httptest.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost/foo%v", i), nil) + evt := newHttpEvent(t, r, events.NewTraits().WithNamespace("http"), sm) + ids = append(ids, evt.Id) + } + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&page=2&size=2", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data api.EventPage + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Equal(t, 2, data.Page) + require.Equal(t, 2, data.Size) + require.Equal(t, 5, data.Total) + require.Len(t, data.Items, 2) + // ids are FIFO, events are LIFO + require.Equal(t, ids[2], data.Items[0].Id) + require.Equal(t, ids[1], data.Items[1].Id) + require.Equal(t, map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo3"}, data.Items[0].Data) + require.Equal(t, map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo2"}, data.Items[1].Data) + }), + ) }, }, { - name: "with http events", + name: "filter http events", fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { sm.SetStore(1, events.NewTraits().WithNamespace("http")) + sm.SetStore(1, events.NewTraits().WithNamespace("kafka")) + err := sm.Push(&eventstest.Event{Name: "foo"}, events.NewTraits().WithNamespace("http")) - event := sm.GetEvents(events.NewTraits())[0] require.NoError(t, err) + err = sm.Push(&eventstest.Event{Name: "foo"}, events.NewTraits().WithNamespace("kafka")) + require.NoError(t, err) + + evt := sm.GetEvents(events.NewTraits().WithNamespace("http"))[0] try.Handler(t, http.MethodGet, - "http://foo.api/api/events?namespace=http", + "http://foo.api/api/events?traits[namespace]=http", nil, "", h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(fmt.Sprintf(`[{"id":"%v","traits":{"namespace":"http"},"data":{"Name":"foo","api":""},"time":"%v"}]`, - event.Id, - event.Time.Format(time.RFC3339Nano)))) + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 1) + require.Equal(t, []any{ + map[string]any{ + "id": evt.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt.Time.Format(time.RFC3339Nano), + "data": map[string]any{ + "name": "foo", + "api": "", + }, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter http method", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + r1 := httptest.NewRequest(http.MethodGet, "http://localhost/foo", nil) + r2 := httptest.NewRequest(http.MethodPost, "http://localhost/foo", nil) + evt1 := newHttpEvent(t, r1, events.NewTraits().WithNamespace("http"), sm) + newHttpEvent(t, r2, events.NewTraits().WithNamespace("http"), sm) + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&http[method]=GET", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 1) + require.Equal(t, []any{ + map[string]any{ + "id": evt1.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt1.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo"}, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter http methods", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + r1 := httptest.NewRequest(http.MethodGet, "http://localhost/foo", nil) + r2 := httptest.NewRequest(http.MethodPost, "http://localhost/foo", nil) + evt1 := newHttpEvent(t, r1, events.NewTraits().WithNamespace("http"), sm) + evt2 := newHttpEvent(t, r2, events.NewTraits().WithNamespace("http"), sm) + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&http[method]=GET,POST", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 2) + require.Equal(t, []any{ + map[string]any{ + "id": evt2.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt2.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo"}, + }, + map[string]any{ + "id": evt1.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt1.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo"}, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter http url", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + r1 := httptest.NewRequest(http.MethodGet, "http://localhost/foo", nil) + r2 := httptest.NewRequest(http.MethodPost, "http://localhost/bar", nil) + newHttpEvent(t, r1, events.NewTraits().WithNamespace("http"), sm) + evt2 := newHttpEvent(t, r2, events.NewTraits().WithNamespace("http"), sm) + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&http[url]=bar", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 1) + require.Equal(t, []any{ + map[string]any{ + "id": evt2.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt2.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/bar"}, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter http request header", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + r1 := httptest.NewRequest(http.MethodGet, "http://localhost/foo", nil) + r2 := httptest.NewRequest(http.MethodPost, "http://localhost/bar", nil) + newHttpEvent(t, r1, events.NewTraits().WithNamespace("http"), sm) + evt2 := newHttpEvent(t, r2, events.NewTraits().WithNamespace("http"), sm) + log := evt2.Data.(*openapi.HttpLog) + log.Request.Parameters = append(log.Request.Parameters, openapi.HttpParameter{ + Name: "role", + Type: "header", + Value: "", + Raw: new("admin"), + }) + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&http[requestHeader]=role:admin", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 1) + require.Equal(t, []any{ + map[string]any{ + "id": evt2.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt2.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/bar"}, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter http status code", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + data := []struct { + url string + method string + status int + }{ + { + url: "http://localhost/foo", + method: http.MethodGet, + status: 200, + }, + { + url: "http://localhost/foo", + method: http.MethodGet, + status: 201, + }, + { + url: "http://localhost/foo", + method: http.MethodGet, + status: 301, + }, + } + + for _, d := range data { + r := httptest.NewRequest(d.method, d.url, nil) + evt := newHttpEvent(t, r, events.NewTraits().WithNamespace("http"), sm) + log := evt.Data.(*openapi.HttpLog) + log.Response.StatusCode = d.status + } + + getStatusCode := func(index int, items []any) int { + if index >= len(items) { + return -1 + } + m := items[index].(map[string]any) + d := m["data"].(map[string]any) + return int(d["statusCode"].(float64)) + } + + tests := []struct { + url string + check try.ResponseCondition + }{ + { + url: "http://foo.api/api/events?traits[namespace]=http&http[statusCode]=200", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 1) + require.Equal(t, 200, getStatusCode(0, items)) + }), + }, + { + url: "http://foo.api/api/events?traits[namespace]=http&http[statusCode]=200-201", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 2) + require.Equal(t, 201, getStatusCode(0, items)) + require.Equal(t, 200, getStatusCode(1, items)) + }), + }, + { + url: "http://foo.api/api/events?traits[namespace]=http&http[statusCode]=2xx", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 2) + require.Equal(t, 201, getStatusCode(0, items)) + require.Equal(t, 200, getStatusCode(1, items)) + }), + }, + { + url: "http://foo.api/api/events?traits[namespace]=http&http[statusCode]=!200-201", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 1) + require.Equal(t, 301, getStatusCode(0, items)) + }), + }, + } + + for _, d := range tests { + try.Handler(t, + http.MethodGet, + d.url, + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + d.check, + ) + } + + }, + }, + { + name: "filter http response header", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + r1 := httptest.NewRequest(http.MethodGet, "http://localhost/foo", nil) + r2 := httptest.NewRequest(http.MethodPost, "http://localhost/bar", nil) + newHttpEvent(t, r1, events.NewTraits().WithNamespace("http"), sm) + evt2 := newHttpEvent(t, r2, events.NewTraits().WithNamespace("http"), sm) + log := evt2.Data.(*openapi.HttpLog) + log.Response.Headers = map[string]string{"role": "admin"} + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events?traits[namespace]=http&http[responseHeader]=role:admin", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 1) + require.Equal(t, []any{ + map[string]any{ + "id": evt2.Id, + "traits": map[string]any{"namespace": "http"}, + "time": evt2.Time.Format(time.RFC3339Nano), + "data": map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/bar"}, + }, + }, data["items"]) + }), + ) + }, + }, + { + name: "filter clientIP", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits().WithNamespace("http")) + + data := []struct { + url string + method string + clientIP string + }{ + { + url: "http://localhost/foo1", + method: http.MethodGet, + clientIP: "127.0.0.1", + }, + { + url: "http://localhost/foo2", + method: http.MethodGet, + clientIP: "127.0.0.1", + }, + { + url: "http://localhost/foo3", + method: http.MethodGet, + clientIP: "192.0.1.24", + }, + } + + for _, d := range data { + r := httptest.NewRequest(d.method, d.url, nil) + evt := newHttpEvent(t, r, events.NewTraits().WithNamespace("http"), sm) + log := evt.Data.(*openapi.HttpLog) + log.ClientIP = d.clientIP + } + + getUrl := func(index int, items []any) string { + if index >= len(items) { + return "" + } + m := items[index].(map[string]any) + d := m["data"].(map[string]any) + return d["url"].(string) + } + + tests := []struct { + url string + check try.ResponseCondition + }{ + { + url: "http://foo.api/api/events?traits[namespace]=http&http[clientIP]=127.0.0.1", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 2) + require.Equal(t, "http://localhost/foo2", getUrl(0, items)) + require.Equal(t, "http://localhost/foo1", getUrl(1, items)) + }), + }, + { + url: "http://foo.api/api/events?traits[namespace]=http&http[clientIP]=192.0.1.1/24", + check: try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + items := data["items"].([]any) + require.Len(t, items, 1) + require.Equal(t, "http://localhost/foo3", getUrl(0, items)) + }), + }, + } + + for _, d := range tests { + try.Handler(t, + http.MethodGet, + d.url, + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + d.check, + ) + } + }, }, { @@ -64,19 +511,20 @@ func TestHandler_Events(t *testing.T) { fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { sm.SetStore(1, events.NewTraits().WithNamespace("http")) err := sm.Push(&eventstest.Event{Name: "foo"}, events.NewTraits().WithNamespace("http")) - event := sm.GetEvents(events.NewTraits())[0] require.NoError(t, err) + + evt := sm.GetEvents(events.NewTraits())[0] try.Handler(t, http.MethodGet, - "http://foo.api/api/events/"+event.Id, + "http://foo.api/api/events/"+evt.Id, nil, "", h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(fmt.Sprintf(`{"id":"%v","traits":{"namespace":"http"},"data":{"Name":"foo","api":""},"time":"%v"}`, - event.Id, - event.Time.Format(time.RFC3339Nano)))) + try.HasBody(fmt.Sprintf(`{"id":"%v","traits":{"namespace":"http"},"data":{"name":"foo","api":""},"time":"%v"}`, + evt.Id, + evt.Time.Format(time.RFC3339Nano)))) }, }, { @@ -101,26 +549,28 @@ func TestHandler_Events(t *testing.T) { l, _ := openapi.LogEventFromContext(ctx) err = sm.Push(l, traits) require.NoError(t, err) + + evt := sm.GetEvents(events.NewTraits())[0] try.Handler(t, http.MethodGet, - "http://foo.api/api/events?namespace=http", + "http://foo.api/api/events/"+evt.Id, nil, "", h, try.HasStatusCode(200), try.AssertBody(func(t *testing.T, body string) { - var m []map[string]any + var m map[string]any require.NoError(t, json.Unmarshal([]byte(body), &m)) - require.Equal(t, map[string]interface{}{ - "actions": interface{}(nil), + require.Equal(t, map[string]any{ + "actions": any(nil), "api": "", "deprecated": false, "duration": float64(0), "path": "", - "request": map[string]interface{}{ + "request": map[string]any{ "method": "get", - "parameters": []interface{}{ - map[string]interface{}{ + "parameters": []any{ + map[string]any{ "name": "Foo", "raw": "bar", "type": "header", @@ -129,10 +579,10 @@ func TestHandler_Events(t *testing.T) { }, "url": "http://localhost/foo", }, - "response": map[string]interface{}{"body": "", "size": float64(0), "statusCode": float64(0)}, + "response": map[string]any{"body": "", "size": float64(0), "statusCode": float64(0)}, "clientIP": "192.0.2.1", }, - m[0]["data"]) + m["data"]) })) }, }, @@ -158,26 +608,28 @@ func TestHandler_Events(t *testing.T) { l, _ := openapi.LogEventFromContext(ctx) err = sm.Push(l, traits) require.NoError(t, err) + + evt := sm.GetEvents(events.NewTraits())[0] try.Handler(t, http.MethodGet, - "http://foo.api/api/events?namespace=http", + "http://foo.api/api/events/"+evt.Id, nil, "", h, try.HasStatusCode(200), try.AssertBody(func(t *testing.T, body string) { - var m []map[string]any + var m map[string]any require.NoError(t, json.Unmarshal([]byte(body), &m)) - require.Equal(t, map[string]interface{}{ - "actions": interface{}(nil), + require.Equal(t, map[string]any{ + "actions": any(nil), "api": "", "deprecated": false, "duration": float64(0), "path": "", - "request": map[string]interface{}{ + "request": map[string]any{ "method": "get", - "parameters": []interface{}{ - map[string]interface{}{ + "parameters": []any{ + map[string]any{ "name": "Foo", "raw": "bar", "type": "header", @@ -186,15 +638,15 @@ func TestHandler_Events(t *testing.T) { }, "url": "http://localhost/foo", }, - "response": map[string]interface{}{"body": "", "size": float64(0), "statusCode": float64(0)}, + "response": map[string]any{"body": "", "size": float64(0), "statusCode": float64(0)}, "clientIP": "192.0.2.1", }, - m[0]["data"]) + m["data"]) })) }, }, { - name: "http with request parameter", + name: "event not found", fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { try.Handler(t, http.MethodGet, @@ -205,15 +657,71 @@ func TestHandler_Events(t *testing.T) { try.HasStatusCode(404)) }, }, + { + name: "test all events", + fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { + sm.SetStore(2, events.NewTraits()) + + newHttpEvent(t, httptest.NewRequest("get", "http://localhost/foo", nil), events.NewTraits().WithNamespace("http"), sm) + err := sm.Push(&kafka.KafkaMessageLog{ + Topic: "topic", + Key: &kafka.LogValue{Value: "foo"}, + Message: &kafka.LogValue{Value: "bar"}, + }, events.NewTraits().WithNamespace("kafka")) + require.NoError(t, err) + err = sm.Push(&mqtt.LogMessage{Topic: "topic", Message: &mqtt.LogValue{Value: "foo"}}, events.NewTraits().WithNamespace("kafka")) + require.NoError(t, err) + err = sm.Push(&websocket.MessageLog{Channel: "channel", Message: &websocket.LogValue{Value: "foo"}}, events.NewTraits().WithNamespace("kafka")) + require.NoError(t, err) + err = sm.Push(&directory.SearchLog{ + Request: &directory.SearchRequest{Operation: "Search", BaseDN: "foo", Filter: "bar"}, + Response: &directory.SearchResponse{Status: "ok"}, + Duration: int64(10), + }, events.NewTraits().WithNamespace("kafka")) + require.NoError(t, err) + + getItemData := func(items []any, index int) map[string]any { + if index >= len(items) { + t.FailNow() + } + m := items[index].(map[string]any) + d, ok := m["data"] + require.True(t, ok, "data field not found for index %d", index) + return d.(map[string]any) + } + + try.Handler(t, + http.MethodGet, + "http://foo.api/api/events", + nil, + "", + h, + try.HasStatusCode(200), + try.HasHeader("Content-Type", "application/json"), + try.AssertBody(func(t *testing.T, body string) { + var data map[string]any + require.NoError(t, json.Unmarshal([]byte(body), &data)) + require.Len(t, data["items"], 5) + items := data["items"].([]any) + // get the inserted order not LIFO + slices.Reverse(items) + require.Equal(t, map[string]any{"deprecated": false, "duration": float64(0), "statusCode": float64(0), "url": "http://localhost/foo"}, getItemData(items, 0)) + require.Equal(t, map[string]any{"topic": "topic", "deleted": false, "key": map[string]any{"binary": nil, "value": "foo"}, "message": map[string]any{"binary": nil, "value": "bar"}}, getItemData(items, 1)) + require.Equal(t, map[string]any{"topic": "topic", "message": map[string]any{"binary": nil, "value": "foo"}}, getItemData(items, 2)) + require.Equal(t, map[string]any{"channel": "channel", "client": map[string]any{"address": "", "header": nil, "id": "", "query": nil, "server": ""}, "direction": "", "message": map[string]any{"binary": nil, "value": "foo"}}, getItemData(items, 3)) + require.Equal(t, map[string]any{"criteria": "bar", "dn": "foo", "duration": float64(10), "operation": "Search", "status": "ok"}, getItemData(items, 4)) + }), + ) + }, + }, } for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - h := New(app, static.Api{}) + h := api.New(app, static.Api{}) tc.fn(t, h, app.Events) }) } @@ -235,7 +743,7 @@ func TestHandler_KafkaEvents(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[]`)) + try.HasBody(`{"items":[],"page":1,"size":25,"total":0}`)) }, }, { @@ -253,7 +761,7 @@ func TestHandler_KafkaEvents(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(fmt.Sprintf(`[{"id":"%v","traits":{"namespace":"kafka"},"data":{"Name":"foo","api":""},"time":"%v"}]`, + try.HasBody(fmt.Sprintf(`{"items":[{"id":"%v","traits":{"namespace":"kafka"},"time":"%v","data":{"name":"foo","api":""}}],"page":1,"size":25,"total":1}`, event.Id, event.Time.Format(time.RFC3339Nano)))) }, @@ -263,8 +771,9 @@ func TestHandler_KafkaEvents(t *testing.T) { fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { sm.SetStore(1, events.NewTraits().WithNamespace("kafka")) err := sm.Push(&eventstest.Event{Name: "foo"}, events.NewTraits().WithNamespace("kafka")) - event := sm.GetEvents(events.NewTraits())[0] require.NoError(t, err) + + event := sm.GetEvents(events.NewTraits())[0] try.Handler(t, http.MethodGet, "http://foo.api/api/events/"+event.Id, @@ -273,7 +782,7 @@ func TestHandler_KafkaEvents(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(fmt.Sprintf(`{"id":"%v","traits":{"namespace":"kafka"},"data":{"Name":"foo","api":""},"time":"%v"}`, + try.HasBody(fmt.Sprintf(`{"id":"%v","traits":{"namespace":"kafka"},"data":{"name":"foo","api":""},"time":"%v"}`, event.Id, event.Time.Format(time.RFC3339Nano)))) }, @@ -283,10 +792,11 @@ func TestHandler_KafkaEvents(t *testing.T) { fn: func(t *testing.T, h http.Handler, sm *events.StoreManager) { sm.SetStore(1, events.NewTraits().WithNamespace("kafka")) - err := sm.Push(&store.KafkaMessageLog{ + err := sm.Push(&kafka.KafkaMessageLog{ + Topic: "foo", Offset: 123, - Key: store.LogValue{}, - Message: store.LogValue{}, + Key: &kafka.LogValue{}, + Message: &kafka.LogValue{}, MessageId: "foo-1", Partition: 1, ProducerId: 3, @@ -297,26 +807,28 @@ func TestHandler_KafkaEvents(t *testing.T) { }, events.NewTraits().WithNamespace("kafka")) require.NoError(t, err) + event := sm.GetEvents(events.NewTraits())[0] try.Handler(t, http.MethodGet, - "http://foo.api/api/events?namespace=kafka", + "http://foo.api/api/events/"+event.Id, nil, "", h, try.HasStatusCode(200), try.AssertBody(func(t *testing.T, body string) { - var m []map[string]any + var m map[string]any require.NoError(t, json.Unmarshal([]byte(body), &m)) require.Equal(t, map[string]any{ + "topic": "foo", "api": "", "deleted": false, - "headers": interface{}(nil), - "key": map[string]interface{}{ - "binary": interface{}(nil), + "headers": any(nil), + "key": map[string]any{ + "binary": any(nil), "value": "", }, - "message": map[string]interface{}{ - "binary": interface{}(nil), + "message": map[string]any{ + "binary": any(nil), "value": "", }, "messageId": "foo-1", @@ -330,19 +842,18 @@ func TestHandler_KafkaEvents(t *testing.T) { "script": "123-123", "actions": nil, }, - m[0]["data"]) + m["data"]) })) }, }, } for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { cfg := &static.Config{} app := runtime.New(cfg, &dynamictest.Reader{}) - h := New(app, static.Api{}) + h := api.New(app, static.Api{}) tc.fn(t, h, app.Events) }) } diff --git a/api/handler_http.go b/api/handler_http.go index 61ef7ee48..1087c5e29 100644 --- a/api/handler_http.go +++ b/api/handler_http.go @@ -116,8 +116,8 @@ type server struct { type securityRequirement map[string]securityScheme type securityScheme struct { - Scopes []string `json:"scopes"` - Configs interface{} `json:"configs"` + Scopes []string `json:"scopes"` + Configs any `json:"configs"` } type tag struct { @@ -129,9 +129,9 @@ type tag struct { } type httpMetrics struct { - Requests float64 `json:"http_requests_total"` - RequestErrors float64 `json:"http_requests_errors_total"` - LastRequest float64 `json:"http_request_timestamp"` + Requests float64 `json:"requests_total"` + RequestErrors float64 `json:"requests_errors_total"` + LastRequest float64 `json:"request_timestamp"` } type openapiSchema struct { diff --git a/api/handler_http_test.go b/api/handler_http_test.go index 8ad0cf1f8..f08d828b2 100644 --- a/api/handler_http_test.go +++ b/api/handler_http_test.go @@ -40,7 +40,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"http","status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "summary takes precedence over description", @@ -53,7 +53,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"summary","version":"1.0","type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}`, + responseBody: `[{"name":"foo","description":"summary","version":"1.0","type":"http","status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}`, }, { name: "get http services with contact", @@ -65,7 +65,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"type":"http","status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"name":"foo","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"type":"http","status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "get specific http service", @@ -178,7 +178,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"parameters":[{"name":"bar","type":"path","required":true,"deprecated":false,"explode":false,"allowReserved":false,"schema":{"type":"string"}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"parameters":[{"name":"bar","type":"path","required":true,"deprecated":false,"explode":false,"allowReserved":false,"schema":{"type":"string"}}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "get http service with requestBody", @@ -196,7 +196,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"requestBody":{"description":"foo","contents":[{"type":"application/json","schema":{"type":"string"}}],"required":true},"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"requestBody":{"description":"foo","contents":[{"type":"application/json","schema":{"type":"string"}}],"required":true},"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "get http service with security", @@ -218,7 +218,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo", - responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "get http service with global security", @@ -239,7 +239,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo", - responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo","deprecated":false,"security":[{"foo":{"scopes":[],"configs":{"type":"apiKey","in":"header","name":"X-API-Key"}}}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "get http service with response", @@ -261,7 +261,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "reference override summary/description", @@ -299,7 +299,7 @@ func TestHandler_Http(t *testing.T) { return runtimetest.NewHttpApp(c) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"Description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"Description","contents":[{"type":"application/json","schema":{"type":"string"}}],"headers":[{"name":"foo","description":"bar","schema":{"type":"string"}}]}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "schema with string or number", @@ -321,7 +321,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":["string","number"]}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":["string","number"]}}]}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "schema with default", @@ -342,7 +342,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo/operations?method=get&path=/foo/{bar}", - responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string","default":"foobar"}}]}],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]`, + responseBody: `[{"method":"get","path":"/foo/{bar}","deprecated":false,"responses":[{"statusCode":"200","description":"foo description","contents":[{"type":"application/json","schema":{"type":"string","default":"foobar"}}]}],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]`, }, { name: "tags", @@ -360,7 +360,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"tags":["foo"],"status":"valid","metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]}],"tags":[{"name":"foo","summary":"sum","description":"desc","parent":"","kind":""}]`, + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"tags":["foo"],"status":"valid","metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]}],"tags":[{"name":"foo","summary":"sum","description":"desc","parent":"","kind":""}]`, }, { name: "operation with errors", @@ -379,7 +379,7 @@ func TestHandler_Http(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"invalid","errors":[{"message":"error"}],"metrics":{"http_requests_total":0,"http_requests_errors_total":0,"http_request_timestamp":0}}]}]`, + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"invalid","errors":[{"message":"error"}],"metrics":{"requests_total":0,"requests_errors_total":0,"request_timestamp":0}}]}]`, }, { name: "export to OpenAPI.json", @@ -657,7 +657,7 @@ func TestHandler_Http_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewHttpApp(openapitest.NewConfig("3.0.0", openapitest.WithInfo("foo", "", ""))), requestUrl: "http://foo.api/api/services", - responseBody: `{"name":"foo","type":"http","status":"valid","metrics":{"http_requests_total":1,"http_requests_errors_total":3,"http_request_timestamp":123456}}`, + responseBody: `{"name":"foo","type":"http","status":"valid","metrics":{"requests_total":1,"requests_errors_total":3,"request_timestamp":123456}}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Http.RequestCounter.WithLabel("foo", "bar", http.MethodGet).Add(1) monitor.Http.RequestErrorCounter.WithLabel("foo", "bar", http.MethodGet).Add(3) @@ -673,7 +673,7 @@ func TestHandler_Http_Metrics(t *testing.T) { ), ), requestUrl: "http://foo.api/api/services/http/foo", - responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"valid","metrics":{"http_requests_total":1,"http_requests_errors_total":3,"http_request_timestamp":123456}}]}],`, + responseBody: `{"name":"foo","servers":[{"url":"/","description":""}],"paths":[{"path":"/foo/{bar}","status":"valid","operations":[{"method":"get","deprecated":false,"status":"valid","metrics":{"requests_total":1,"requests_errors_total":3,"request_timestamp":123456}}]}],`, addMetrics: func(monitor *monitor.Monitor) { monitor.Http.RequestCounter.WithLabel("foo", "/foo/{bar}", http.MethodGet).Add(1) monitor.Http.RequestErrorCounter.WithLabel("foo", "/foo/{bar}", http.MethodGet).Add(3) diff --git a/api/handler_kafka.go b/api/handler_kafka.go index 6053db894..706dd0d39 100644 --- a/api/handler_kafka.go +++ b/api/handler_kafka.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "maps" "mokapi/media" "mokapi/providers/asyncapi3" "mokapi/providers/asyncapi3/kafka/store" @@ -25,15 +26,15 @@ import ( ) type kafkaInfo struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Version string `json:"version"` - Contact *contact `json:"contact,omitempty"` - Servers []kafkaServer `json:"servers,omitempty"` - Topics []kafkaTopicInfo `json:"topics,omitempty"` - Groups []kafkaGroupInfo `json:"groups,omitempty"` - Configs []configInfo `json:"configs,omitempty"` - Clients []kafkaClientInfo `json:"clients,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Version string `json:"version"` + Contact *contact `json:"contact,omitempty"` + Servers []kafkaServer `json:"servers,omitempty"` + Topics []kafkaChannelInfo `json:"topics,omitempty"` + Groups []kafkaGroupInfo `json:"groups,omitempty"` + Configs []configInfo `json:"configs,omitempty"` + Clients []kafkaClientInfo `json:"clients,omitempty"` } type kafkaServer struct { @@ -98,25 +99,44 @@ type kafkaMember struct { Partitions map[string][]int `json:"partitions"` } -type kafkaTopicInfo struct { - Name string `json:"name"` - Summary string `json:"summary,omitempty"` - Tags []kafkaTag `json:"tags,omitempty"` - Metrics kafkaTopicMetric `json:"metrics"` +type kafkaChannelInfo struct { + Name string `json:"name"` + Summary string `json:"summary,omitempty"` + Tags []kafkaTag `json:"tags,omitempty"` + Parameters []string `json:"parameters,omitempty"` + Instances []kafkaTopicInstance `json:"instances,omitempty"` + Metrics kafkaTopicMetric `json:"metrics"` } -type kafkaTopic struct { +type kafkaChannel struct { Name string `json:"name"` Title string `json:"title,omitempty"` Summary string `json:"summary,omitempty"` Description string `json:"description,omitempty"` - Partitions []kafkaPartition `json:"partitions"` Messages map[string]messageConfig `json:"messages,omitempty"` - Bindings kafkaBindings `json:"bindings,omitempty"` + Parameters []string `json:"parameters,omitempty"` + Instances []kafkaTopicInstance `json:"instances,omitempty"` Tags []kafkaTag `json:"tags,omitempty"` - Groups []kafkaGroupInfo `json:"groups,omitempty"` + Metrics kafkaTopicMetric `json:"metrics"` } +type kafkaTopic struct { + Name string `json:"name"` + Parameters kafkaTopicInstanceParameters `json:"parameters,omitempty"` + Bindings kafkaBindings `json:"bindings"` + Partitions []kafkaPartition `json:"partitions"` + Groups []kafkaGroupInfo `json:"groups,omitempty"` + Metrics kafkaTopicMetric `json:"metrics"` +} + +type kafkaTopicInstance struct { + Name string `json:"name"` + Parameters kafkaTopicInstanceParameters `json:"parameters,omitempty"` + Metrics mqttTopicMetrics `json:"metrics"` +} + +type kafkaTopicInstanceParameters map[string]string + type kafkaPartition struct { Id int `json:"id"` StartOffset int64 `json:"startOffset"` @@ -160,19 +180,19 @@ type KafkaRecordResult struct { } type kafkaTopicMetric struct { - NumMessages float64 `json:"kafka_messages_total"` - LastMessageTime float64 `json:"kafka_message_timestamp"` + NumMessages float64 `json:"messages_total"` + LastMessageTime float64 `json:"message_timestamp"` } type kafkaGroupMetrics struct { - LastRebalancing float64 `json:"kafka_rebalance_timestamp"` + LastRebalancing float64 `json:"rebalance_timestamp"` Topics map[string][]kafkaTopicGroupMetrics `json:"topics,omitempty"` } type kafkaTopicGroupMetrics struct { Partition int `json:"partition"` - Lags float64 `json:"kafka_consumer_group_lag"` - Commit float64 `json:"kafka_consumer_group_commit"` + Lags float64 `json:"consumer_group_lag"` + Commit float64 `json:"consumer_group_commit"` } func getKafkaServices(store *runtime.KafkaStore, m *monitor.Monitor) []service { @@ -211,6 +231,7 @@ func (h *handler) setupKafka() { r.HandleFunc("", h.getKafkaClusters).Methods(http.MethodGet) r.HandleFunc("/{cluster}", h.getKafkaInfo).Methods(http.MethodGet) r.HandleFunc("/{cluster}/asyncapi.{ext}", h.exportAsyncApi).Methods(http.MethodGet) + r.HandleFunc("/{cluster}/channels/{channel}", h.getKafkaChannel).Methods(http.MethodGet) r.HandleFunc("/{cluster}/topics", h.getKafkaTopics).Methods(http.MethodGet) r.HandleFunc("/{cluster}/topics/{topic}", h.getKafkaTopic).Methods(http.MethodGet) r.HandleFunc("/{cluster}/topics/{topic}", h.produceKafkaMessage).Methods(http.MethodPost) @@ -286,7 +307,7 @@ func (h *handler) getKafkaInfo(w http.ResponseWriter, r *http.Request) { return strings.Compare(k.Servers[i].Name, k.Servers[j].Name) < 0 }) - k.Topics = getTopicInfos(ki, h.app.Monitor.Kafka) + k.Topics = getKafkaChannels(slices.Collect(maps.Values(ki.Channels)), ki, h.app.Monitor.Kafka) k.Groups = getGroupInfos(ki, "", h.app.Monitor.Kafka) k.Configs = getConfigInfos(ki.Configs()) @@ -303,6 +324,70 @@ func (h *handler) getKafkaInfo(w http.ResponseWriter, r *http.Request) { write(w, k) } +func (h *handler) getKafkaChannel(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + ki := h.app.Kafka.Get(vars["cluster"]) + if ki == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + var ch *asyncapi3.Channel + for name, item := range ki.Channels { + if name == vars["channel"] { + ch = item.Value + break + } + if item.Value != nil && item.Value.Address == vars["channel"] { + ch = item.Value + break + } + } + + if ch == nil || !ch.IsChannelAvailable("kafka") { + w.WriteHeader(http.StatusNotFound) + return + } + + addr := ch.ResolveAddress() + result := kafkaChannel{ + Name: addr, + Title: ch.Title, + Summary: ch.Summary, + Description: ch.Description, + Messages: getMessageConfigs(ch, ki.Config), + Tags: getKafkaTags(ch), + } + + if len(ch.Parameters) > 0 { + result.Parameters = slices.SortedFunc(maps.Keys(ch.Parameters), func(a, b string) int { + return strings.Compare(a, b) + }) + for _, t := range ki.Topics() { + if err := ch.IsNameValid(t.Name); err == nil { + params, _ := ch.ExtractParams(t.Name) + inst := kafkaTopicInstance{ + Name: t.Name, + Parameters: params, + } + inst.Metrics = mqttTopicMetrics{ + NumMessages: h.app.Monitor.Kafka.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("instance", t.Name))), + LastMessageTime: h.app.Monitor.Kafka.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("instance", t.Name))), + } + result.Instances = append(result.Instances, inst) + } + } + } + + result.Metrics = kafkaTopicMetric{ + NumMessages: h.app.Monitor.Kafka.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("topic", addr))), + LastMessageTime: h.app.Monitor.Kafka.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("topic", addr))), + } + + write(w, result) +} + func (h *handler) getKafkaTopics(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -312,7 +397,12 @@ func (h *handler) getKafkaTopics(w http.ResponseWriter, r *http.Request) { return } - topics := getTopicInfos(ki, h.app.Monitor.Kafka) + var channels []*asyncapi3.ChannelRef + for _, topic := range ki.Topics() { + channels = append(channels, &asyncapi3.ChannelRef{Value: topic.Config}) + } + + topics := getKafkaChannels(channels, ki, h.app.Monitor.Kafka) write(w, topics) } @@ -322,23 +412,8 @@ func (h *handler) getKafkaTopic(w http.ResponseWriter, r *http.Request) { return } - for n, ch := range ki.Channels { - if ch.Value == nil { - continue - } - addr := ch.Value.Address - if addr == "" { - addr = n - } - if addr == t.Name { - r := newTopic(t, ki, ch.Value, h.app.Monitor.Kafka) - write(w, r) - return - } - - } - w.WriteHeader(http.StatusNotFound) - _, _ = w.Write([]byte("kafka topic not found")) + ti := newTopic(t, ki, t.Config, h.app.Monitor.Kafka) + write(w, ti) } func (h *handler) getKafkaPartitions(w http.ResponseWriter, r *http.Request) { @@ -572,26 +647,39 @@ func (h *handler) produceKafkaMessage(w http.ResponseWriter, r *http.Request) { write(w, res) } -func getTopicInfos(ki *runtime.KafkaInfo, m *monitor.Kafka) []kafkaTopicInfo { - topics := make([]kafkaTopicInfo, 0, len(ki.Config.Channels)) - for name, ch := range ki.Config.Channels { +func getKafkaChannels(channels []*asyncapi3.ChannelRef, ki *runtime.KafkaInfo, m *monitor.Kafka) []kafkaChannelInfo { + topics := make([]kafkaChannelInfo, 0, len(ki.Config.Channels)) + for _, ch := range channels { if ch.Value == nil { continue } if !ch.Value.IsChannelAvailable("kafka") { continue } - addr := ch.Value.Address - if addr == "" { - addr = name - } + addr := ch.Value.ResolveAddress() - ti := kafkaTopicInfo{ + ti := kafkaChannelInfo{ Name: addr, Summary: ch.Value.Summary, Tags: getKafkaTags(ch.Value), } + if len(ch.Value.Parameters) > 0 { + ti.Parameters = slices.SortedFunc(maps.Keys(ch.Value.Parameters), func(a, b string) int { + return strings.Compare(a, b) + }) + for _, t := range ki.Topics() { + if err := ch.Value.IsNameValid(t.Name); err == nil { + params, _ := ch.Value.ExtractParams(t.Name) + inst := kafkaTopicInstance{ + Name: t.Name, + Parameters: params, + } + ti.Instances = append(ti.Instances, inst) + } + } + } + ti.Metrics = kafkaTopicMetric{ NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("topic", addr))), LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", ki.Info.Name), metrics.ByLabel("topic", addr))), @@ -620,11 +708,8 @@ func newTopic(t *store.Topic, ki *runtime.KafkaInfo, ch *asyncapi3.Channel, m *m } result := kafkaTopic{ - Name: t.Name, - Title: ch.Title, - Summary: ch.Summary, - Description: ch.Description, - Partitions: partitions, + Name: t.Name, + Partitions: partitions, Bindings: kafkaBindings{ Partitions: bindings.Partitions, RetentionBytes: bindings.RetentionBytes, @@ -634,11 +719,27 @@ func newTopic(t *store.Topic, ki *runtime.KafkaInfo, ch *asyncapi3.Channel, m *m ValueSchemaValidation: bindings.ValueSchemaValidation, KeySchemaValidation: bindings.KeySchemaValidation, }, - Tags: getKafkaTags(ch), Groups: getGroupInfos(ki, t.Name, m), } - result.Messages = getMessageConfigs(ch, ki.Config) + if len(ch.Parameters) > 0 { + params, _ := ch.ExtractParams(t.Name) + result.Parameters = params + } + + labels := []metrics.QueryOptions{ + metrics.ByLabel("service", ki.Info.Name), + } + if len(t.Config.Parameters) > 0 { + labels = append(labels, metrics.ByLabel("instance", t.Name)) + } else { + labels = append(labels, metrics.ByLabel("topic", t.Name)) + } + + result.Metrics = kafkaTopicMetric{ + NumMessages: m.Messages.Sum(metrics.NewQuery(labels...)), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(labels...)), + } return result } @@ -919,3 +1020,22 @@ func (h *handler) exportAsyncApi(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", ct) _, _ = w.Write(b) } + +func (p kafkaTopicInstanceParameters) MarshalJSON() ([]byte, error) { + keys := slices.SortedFunc(maps.Keys(p), func(a, b string) int { + return strings.Compare(a, b) + }) + if len(keys) == 0 { + return []byte("{}"), nil + } + var b bytes.Buffer + b.WriteString("{") + for i, k := range keys { + if i > 0 { + b.WriteString(",") + } + b.WriteString(fmt.Sprintf(`"%s": "%s"`, k, p[k])) + } + b.WriteString("}") + return b.Bytes(), nil +} diff --git a/api/handler_kafka_test.go b/api/handler_kafka_test.go index 824a8de26..828d94dd6 100644 --- a/api/handler_kafka_test.go +++ b/api/handler_kafka_test.go @@ -2,6 +2,7 @@ package api_test import ( "encoding/json" + "fmt" "io" "mokapi/api" "mokapi/config/dynamic" @@ -32,14 +33,6 @@ import ( func TestHandler_Kafka(t *testing.T) { logrus.SetOutput(io.Discard) - mustTime := func(s string) time.Time { - t, err := time.Parse(time.RFC3339, s) - if err != nil { - panic(err) - } - return t - } - testcases := []struct { name string app func() *runtime.App @@ -56,7 +49,7 @@ func TestHandler_Kafka(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"kafka","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get kafka services", @@ -78,7 +71,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka", - responseBody: `[{"name":"foo","description":"bar","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"kafka","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get kafka services with contact", @@ -90,7 +83,7 @@ func TestHandler_Kafka(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"test","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"version":"1.0","type":"kafka","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`, + responseBody: `[{"name":"test","contact":{"name":"foo","url":"https://foo.bar","email":"foo@bar.com"},"version":"1.0","type":"kafka","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get cluster info", @@ -102,7 +95,7 @@ func TestHandler_Kafka(t *testing.T) { asyncapi3test.WithInfo("foo", "bar", "1.0"), ), } - cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + cfg.Info.Time = try.MustTime("2023-12-27T13:01:30+00:00") _, _ = app.Kafka.Add(cfg, enginetest.NewEngine()) return app @@ -183,7 +176,7 @@ func TestHandler_Kafka(t *testing.T) { })) }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","summary":"bar","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"foo","summary":"bar","metrics":{"messages_total":0,"message_timestamp":0}}]}`, }, { name: "get cluster info with topic and tag", @@ -210,7 +203,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"bar-1","metrics":{"kafka_messages_total":10,"kafka_message_timestamp":11}},{"name":"foo","tags":[{"name":"env:test","description":"bar"}],"metrics":{"kafka_messages_total":1,"kafka_message_timestamp":2}}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","topics":[{"name":"bar-1","metrics":{"messages_total":10,"message_timestamp":11}},{"name":"foo","tags":[{"name":"env:test","description":"bar"}],"metrics":{"messages_total":1,"message_timestamp":2}}]}`, }, { name: "get topic and multi schema format", @@ -232,8 +225,8 @@ func TestHandler_Kafka(t *testing.T) { Store: s, })) }, - requestUrl: "http://foo.api/api/services/kafka/foo/topics/foo", - responseBody: `{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true,"keySchemaValidation":true}}`, + requestUrl: "http://foo.api/api/services/kafka/foo/channels/foo", + responseBody: `{"name":"foo","description":"bar","messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"metrics":{"messages_total":0,"message_timestamp":0}}`, }, { name: "get cluster info with group", @@ -258,7 +251,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka"}],"groups":[{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":0,"metrics":{"kafka_rebalance_timestamp":0}}]}`, + responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"foo.bar","protocol":"kafka"}],"groups":[{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":0,"metrics":{"rebalance_timestamp":0}}]}`, }, { name: "get group", @@ -283,7 +276,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", - responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, + responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"rebalance_timestamp":0}}`, }, { name: "get group without generation", @@ -302,7 +295,7 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", - responseBody: `{"name":"foo","generation":-1,"state":"PreparingRebalance","protocol":"","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, + responseBody: `{"name":"foo","generation":-1,"state":"PreparingRebalance","protocol":"","members":null,"leader":"","topics":null,"metrics":{"rebalance_timestamp":0}}`, }, { name: "get specific with group containing members", @@ -358,10 +351,10 @@ func TestHandler_Kafka(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/kafka/foo/groups/foo", - responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":[{"name":"m1","clientId":"client1","addr":"192.168.0.100","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:05+07:00","partitions":{"topic":[1,2,5]}},{"name":"m2","clientId":"client2","addr":"192.168.0.200","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:10+07:00","partitions":{"topic":[3,4,6]}}],"leader":"m1","topics":null,"metrics":{"kafka_rebalance_timestamp":0}}`, + responseBody: `{"name":"foo","generation":3,"state":"PreparingRebalance","protocol":"range","members":[{"name":"m1","clientId":"client1","addr":"192.168.0.100","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:05+07:00","partitions":{"topic":[1,2,5]}},{"name":"m2","clientId":"client2","addr":"192.168.0.200","software":"mokapi 1.0","heartbeat":"2024-04-22T15:04:10+07:00","partitions":{"topic":[3,4,6]}}],"leader":"m1","topics":null,"metrics":{"rebalance_timestamp":0}}`, }, { - name: "get topic with openapi schema", + name: "get channel with openapi schema", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) app.Kafka.Set("foo", getKafkaInfo(asyncapi3test.NewConfig( @@ -376,8 +369,8 @@ func TestHandler_Kafka(t *testing.T) { ))) return app }, - requestUrl: "http://foo.api/api/services/kafka/foo/topics/foo", - responseBody: `{"name":"foo","description":"bar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true,"keySchemaValidation":true}}`, + requestUrl: "http://foo.api/api/services/kafka/foo/channels/foo", + responseBody: `{"name":"foo","description":"bar","messages":{"foo":{"name":"foo","payload":{"format":"foo","schema":{"type":"string"}},"contentType":"application/json"}},"metrics":{"messages_total":0,"message_timestamp":0}}`, }, } @@ -459,12 +452,12 @@ func TestHandler_KafkaAPI(t *testing.T) { h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`[{"name":"topic-1","metrics":{"kafka_messages_total":0,"kafka_message_timestamp":0}}]`), + try.HasBody(`[{"name":"topic-1","metrics":{"messages_total":0,"message_timestamp":0}}]`), ) }, }, { - name: "get specific kafka topic", + name: "get specific kafka channel", app: func() *runtime.App { app := runtime.New(&static.Config{}, &dynamictest.Reader{}) _, _ = app.Kafka.Add(&dynamic.Config{ @@ -483,13 +476,13 @@ func TestHandler_KafkaAPI(t *testing.T) { test: func(t *testing.T, app *runtime.App, h http.Handler) { try.Handler(t, http.MethodGet, - "http://foo.api/api/services/kafka/foo/topics/topic-1", + "http://foo.api/api/services/kafka/foo/channels/topic-1", nil, "", h, try.HasStatusCode(200), try.HasHeader("Content-Type", "application/json"), - try.HasBody(`{"name":"topic-1","description":"foobar","partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"messages":{"foo":{"name":"foo","payload":null,"contentType":"application/json"}},"bindings":{"partitions":1,"valueSchemaValidation":true,"keySchemaValidation":true}}`), + try.HasBody(`{"name":"topic-1","description":"foobar","messages":{"foo":{"name":"foo","payload":null,"contentType":"application/json"}},"metrics":{"messages_total":0,"message_timestamp":0}}`), ) }, }, @@ -985,7 +978,7 @@ func TestHandler_KafkaAPI(t *testing.T) { res, err := c.Write("topic-1", []store.Record{ { Key: "foo", - Value: map[string]interface{}{"value": "bar"}, + Value: map[string]any{"value": "bar"}, }, }, ct) if err != nil { @@ -1043,7 +1036,7 @@ func TestHandler_KafkaAPI(t *testing.T) { _, _ = c.Write("topic-1", []store.Record{ { Key: "foo", - Value: map[string]interface{}{"value": "bar"}, + Value: map[string]any{"value": "bar"}, }, }, ct) @@ -1144,7 +1137,7 @@ func TestHandler_Kafka_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewApp(runtimetest.WithKafkaInfo("foo", getKafkaInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","version":"1.0","type":"kafka","metrics":{"kafka_messages_total":1,"kafka_message_timestamp":12345678}}]`, + responseBody: `[{"name":"foo","version":"1.0","type":"kafka","metrics":{"messages_total":1,"message_timestamp":12345678}}]`, addMetrics: func(monitor *monitor.Monitor) { monitor.Kafka.Messages.WithLabel("foo", "topic").Add(1) monitor.Kafka.LastMessage.WithLabel("foo", "topic").Set(12345678) @@ -1159,7 +1152,7 @@ func TestHandler_Kafka_Metrics(t *testing.T) { ), ))), requestUrl: "http://foo.api/api/services/kafka/foo", - responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"foo","metrics":{"kafka_messages_total":1,"kafka_message_timestamp":12345678}}]}`, + responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"foo","metrics":{"messages_total":1,"message_timestamp":12345678}}]}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Kafka.Messages.WithLabel("foo", "foo").Add(1) monitor.Kafka.LastMessage.WithLabel("foo", "foo").Set(12345678) @@ -1188,7 +1181,7 @@ func TestHandler_Kafka_Metrics(t *testing.T) { return app }(), requestUrl: "http://foo.api/api/services/kafka/foo/groups/group-1", - responseBody: `{"name":"group-1","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"kafka_rebalance_timestamp":12345678,"topics":{"topic-foo":[{"partition":0,"kafka_consumer_group_lag":123,"kafka_consumer_group_commit":11}]}}}`, + responseBody: `{"name":"group-1","generation":3,"state":"PreparingRebalance","protocol":"range","members":null,"leader":"","topics":null,"metrics":{"rebalance_timestamp":12345678,"topics":{"topic-foo":[{"partition":0,"consumer_group_lag":123,"consumer_group_commit":11}]}}}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Kafka.Lags.WithLabel("foo", "group-1", "topic-foo", "0").Set(123) monitor.Kafka.Commits.WithLabel("foo", "group-1", "topic-foo", "0").Set(11) @@ -1219,6 +1212,100 @@ func TestHandler_Kafka_Metrics(t *testing.T) { } } +func TestHandler_Kafka_Topic_Parameter(t *testing.T) { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + addr := fmt.Sprintf(":%v", try.GetFreePort()) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "kafka", addr), + asyncapi3test.WithChannel("orders.{region}.{eventType}", + asyncapi3test.WithParameter("region", &asyncapi3.Parameter{}), + asyncapi3test.WithParameter("eventType", &asyncapi3.Parameter{}), + ), + ), + } + cfg.Info.Time = try.MustTime("2023-12-27T13:01:30+00:00") + + mi, err := app.Kafka.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + _, err = mi.NewTopic("orders.eu.created", mi.Brokers()[0]) + require.NoError(t, err) + + app.Monitor.Kafka.Messages.WithLabel("foo", "orders.{region}.{eventType}", "orders.eu.created").Add(1) + app.Monitor.Kafka.LastMessage.WithLabel("foo", "orders.{region}.{eventType}", "orders.eu.created").Set(1234) + + h := api.New(app, static.Api{}) + + testcases := []struct { + name string + test func(t *testing.T) + }{ + { + name: "get service", + test: func(t *testing.T) { + try.Handler(t, + http.MethodGet, + "http://foo.api/api/services/kafka/foo", + nil, + "", + h, + try.HasStatusCode(http.StatusOK), + try.HasBody(fmt.Sprintf(`{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"foo","host":"%v","protocol":"kafka"}],"topics":[{"name":"orders.{region}.{eventType}","parameters":["eventType","region"],"instances":[{"name":"orders.eu.created","parameters":{"eventType":"created","region":"eu"},"metrics":{"messages_total":0,"message_timestamp":0}}],"metrics":{"messages_total":1,"message_timestamp":1234}}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, addr)), + ) + }, + }, + { + name: "get channel", + test: func(t *testing.T) { + try.Handler(t, + http.MethodGet, + "http://foo.api/api/services/kafka/foo/channels/orders.{region}.{eventType}", + nil, + "", + h, + try.HasStatusCode(http.StatusOK), + try.HasBody(`{"name":"orders.{region}.{eventType}","parameters":["eventType","region"],"instances":[{"name":"orders.eu.created","parameters":{"eventType":"created","region":"eu"},"metrics":{"messages_total":1,"message_timestamp":1234}}],"metrics":{"messages_total":1,"message_timestamp":1234}}`), + ) + }, + }, + { + name: "get topic using template name", + test: func(t *testing.T) { + try.Handler(t, + http.MethodGet, + "http://foo.api/api/services/kafka/foo/topics/orders.{region}.{eventType}", + nil, + "", + h, + try.HasStatusCode(http.StatusNotFound), + ) + }, + }, + { + name: "get topic", + test: func(t *testing.T) { + try.Handler(t, + http.MethodGet, + "http://foo.api/api/services/kafka/foo/topics/orders.eu.created", + nil, + "", + h, + try.HasStatusCode(http.StatusOK), + try.HasBody(`{"name":"orders.eu.created","parameters":{"eventType":"created","region":"eu"},"bindings":{"partitions":1,"valueSchemaValidation":true,"keySchemaValidation":true},"partitions":[{"id":0,"startOffset":0,"offset":0,"segments":0}],"metrics":{"messages_total":1,"message_timestamp":1234}}`), + ) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + tc.test(t) + }) + } +} + func Test_IsAsyncApiConfig(t *testing.T) { v2 := &asyncapi3.Config{Info: asyncapi3.Info{Name: "foo"}} _, ok := runtime.IsAsyncApiConfig(&dynamic.Config{Data: v2}) diff --git a/api/handler_ldap.go b/api/handler_ldap.go index c57dcb1b1..2fa8ca400 100644 --- a/api/handler_ldap.go +++ b/api/handler_ldap.go @@ -17,8 +17,8 @@ type ldapInfo struct { } type ldapMetrics struct { - NumRequests float64 `json:"ldap_requests_total"` - LastRequest float64 `json:"ldap_request_timestamp"` + NumRequests float64 `json:"requests_total"` + LastRequest float64 `json:"request_timestamp"` } func getLdapServices(store *runtime.LdapStore, m *monitor.Monitor) []service { diff --git a/api/handler_ldap_test.go b/api/handler_ldap_test.go index df69dedb2..454a03c8b 100644 --- a/api/handler_ldap_test.go +++ b/api/handler_ldap_test.go @@ -40,7 +40,7 @@ func TestHandler_Ldap(t *testing.T) { }, requestUrl: "http://foo.api/api/services", contentType: "application/json", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"ldap","metrics":{"ldap_requests_total":0,"ldap_request_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"ldap","metrics":{"requests_total":0,"request_timestamp":0}}]`, }, { name: "get ldap service", diff --git a/api/handler_mail.go b/api/handler_mail.go index 46915b98e..b4cb63174 100644 --- a/api/handler_mail.go +++ b/api/handler_mail.go @@ -117,8 +117,8 @@ type attachment struct { } type mailMetrics struct { - NumMails float64 `json:"mail_mails_total"` - LastMail float64 `json:"mail_mail_timestamp"` + NumMails float64 `json:"mails_total"` + LastMail float64 `json:"mail_timestamp"` } func getMailServices(store *runtime.MailStore, m *monitor.Monitor) []service { diff --git a/api/handler_mail_test.go b/api/handler_mail_test.go index 13bf3c1bc..766bb6496 100644 --- a/api/handler_mail_test.go +++ b/api/handler_mail_test.go @@ -43,7 +43,7 @@ func TestHandler_Smtp(t *testing.T) { }, requestUrl: "http://foo.api/api/services", contentType: "application/json", - responseBody: `[{"name":"foo","description":"bar","version":"2.1","type":"mail","metrics":{"mail_mails_total":0,"mail_mail_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"2.1","type":"mail","metrics":{"mails_total":0,"mail_timestamp":0}}]`, }, { name: "get /api/services/mail", diff --git a/api/handler_metrics.go b/api/handler_metrics.go index 2426ed594..af500c93f 100644 --- a/api/handler_metrics.go +++ b/api/handler_metrics.go @@ -20,8 +20,8 @@ func (h *handler) getMetrics(w http.ResponseWriter, r *http.Request) { if !r.URL.Query().Has("names") { result = h.app.Monitor.FindAll(query...) } else { - names := strings.Split(r.URL.Query().Get("names"), ",") - for _, name := range names { + names := strings.SplitSeq(r.URL.Query().Get("names"), ",") + for name := range names { result = append( result, h.app.Monitor.FindAll(append(query, metrics.ByFQName(name))...)..., diff --git a/api/handler_metrics_test.go b/api/handler_metrics_test.go index bfac703be..e1fa56c2b 100644 --- a/api/handler_metrics_test.go +++ b/api/handler_metrics_test.go @@ -17,7 +17,7 @@ func TestHandler_Metrics(t *testing.T) { fn func(t *testing.T, h http.Handler, app *runtime.App) }{ { - name: "/api/metrics", + name: "get metrics", app: &runtime.App{ Monitor: monitor.New(), }, @@ -34,7 +34,7 @@ func TestHandler_Metrics(t *testing.T) { }, }, { - name: "/api/metrics?names=app_start_timestamp,app_memory_usage_bytes", + name: "using query names", app: &runtime.App{ Monitor: monitor.New(), }, @@ -51,7 +51,7 @@ func TestHandler_Metrics(t *testing.T) { }, }, { - name: "/api/metrics/kafka", + name: "get kafka", app: &runtime.App{ Monitor: monitor.New(), }, @@ -68,7 +68,7 @@ func TestHandler_Metrics(t *testing.T) { }, }, { - name: "/api/metrics/kafka with metric", + name: "get kafka with metric", app: &runtime.App{ Monitor: monitor.New(), }, diff --git a/api/handler_mqtt.go b/api/handler_mqtt.go index b9d4d84a9..3e6d2e556 100644 --- a/api/handler_mqtt.go +++ b/api/handler_mqtt.go @@ -1,6 +1,7 @@ package api import ( + "maps" "mokapi/providers/asyncapi3" "mokapi/runtime" "mokapi/runtime/metrics" @@ -42,13 +43,15 @@ type mqttTopic struct { Description string `json:"description,omitempty"` Messages map[string]messageConfig `json:"messages,omitempty"` Tags []kafkaTag `json:"tags,omitempty"` + Parameters []string `json:"parameters,omitempty"` Instances []mqttTopicInstance `json:"instances,omitempty"` - Metrics mqttTopicMetrics `json:"metrics,omitempty"` + Metrics mqttTopicMetrics `json:"metrics"` } type mqttTopicInstance struct { Name string `json:"name"` Parameters map[string]string `json:"parameters,omitempty"` + Metrics mqttTopicMetrics `json:"metrics"` } type mqttClient struct { @@ -59,8 +62,8 @@ type mqttClient struct { } type mqttTopicMetrics struct { - NumMessages float64 `json:"mqtt_messages_total"` - LastMessageTime float64 `json:"mqtt_message_timestamp"` + NumMessages float64 `json:"messages_total"` + LastMessageTime float64 `json:"message_timestamp"` } func getMqttServices(store *runtime.MqttStore, m *monitor.Monitor) []service { @@ -193,34 +196,39 @@ func (h *handler) getMqttTopics(w http.ResponseWriter, r *http.Request) { func getMqttTopics(mi *runtime.MqttInfo, m *monitor.Mqtt) []mqttTopic { topics := make([]mqttTopic, 0, len(mi.Config.Channels)) - for name, ch := range mi.Config.Channels { + for _, ch := range mi.Config.Channels { if ch.Value == nil { continue } - if !ch.Value.IsChannelAvailable("kafka") { + if !ch.Value.IsChannelAvailable("mqtt") { continue } - addr := name - if ch.Value.Address != "" { - addr = ch.Value.Address - } + addr := ch.Value.ResolveAddress() data := newMqttTopic(addr, ch.Value, mi.Config) if len(ch.Value.Parameters) > 0 { + data.Parameters = slices.SortedFunc(maps.Keys(ch.Value.Parameters), func(a, b string) int { + return strings.Compare(a, b) + }) for _, t := range mi.Topics { if err := ch.Value.IsNameValid(t.Name); err == nil { params, _ := ch.Value.ExtractParams(t.Name) - data.Instances = append(data.Instances, mqttTopicInstance{ + inst := mqttTopicInstance{ Name: t.Name, Parameters: params, - }) + } + inst.Metrics = mqttTopicMetrics{ + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name), metrics.ByLabel("instance", t.Name))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name), metrics.ByLabel("instance", t.Name))), + } + data.Instances = append(data.Instances, inst) } } } data.Metrics = mqttTopicMetrics{ - NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), - LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name))), + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name), metrics.ByLabel("topic", addr))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", mi.Info.Name), metrics.ByLabel("topic", addr))), } topics = append(topics, data) diff --git a/api/handler_mqtt_test.go b/api/handler_mqtt_test.go index d1c9c0458..1ce98ae1d 100644 --- a/api/handler_mqtt_test.go +++ b/api/handler_mqtt_test.go @@ -53,7 +53,7 @@ func TestHandler_Mqtt(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"mqtt","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get MQTT services", @@ -76,7 +76,7 @@ func TestHandler_Mqtt(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/mqtt", - responseBody: `[{"name":"foo","description":"mqtt","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"mqtt","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"mqtt","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get specific", @@ -96,6 +96,70 @@ func TestHandler_Mqtt(t *testing.T) { requestUrl: "http://foo.api/api/services/mqtt/foo", responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"mokapi","host":":1883","protocol":"mqtt","title":"Mokapi Default Broker","summary":"Automatically added broker because no servers are defined in the AsyncAPI spec","description":""}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, }, + { + name: "channels with metrics", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("bar", "mqtt", "foo.bar"), + asyncapi3test.WithChannel("channel1"), + asyncapi3test.WithChannel("channel2"), + ), + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + + app.Monitor.Mqtt.Messages.WithLabel("foo", "channel1").Add(1) + app.Monitor.Mqtt.Messages.WithLabel("foo", "channel2").Add(2) + app.Monitor.Mqtt.LastMessage.WithLabel("foo", "channel1").Set(1234) + app.Monitor.Mqtt.LastMessage.WithLabel("foo", "channel2").Set(7890) + + _, err := app.Mqtt.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + + return app + }, + requestUrl: "http://foo.api/api/services/mqtt/foo/topics", + responseBody: `[{"name":"channel1","metrics":{"messages_total":1,"message_timestamp":1234}},{"name":"channel2","metrics":{"messages_total":2,"message_timestamp":7890}}]`, + }, + { + name: "channels with assigned server", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + addr := fmt.Sprintf(":%v", try.GetFreePort()) + + mqtt := asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "mqtt", addr), + asyncapi3test.WithServer("bar", "kafka", "foo.bar"), + asyncapi3test.WithChannel("channel1", + asyncapi3test.AssignToServer("#/servers/foo"), + ), + asyncapi3test.WithChannel("channel2", + asyncapi3test.AssignToServer("#/servers/bar"), + ), + ) + + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: mqtt, + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + err := mqtt.Parse(cfg, &dynamictest.Reader{}) + require.NoError(t, err) + + mi, err := app.Mqtt.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + mi.Topics["sensors/1234z/data"] = &store.Topic{Name: "sensors/1234z/data"} + + return app + }, + requestUrl: "http://foo.api/api/services/mqtt/foo/topics", + responseBody: `[{"name":"channel1","metrics":{"messages_total":0,"message_timestamp":0}}]`, + }, { name: "topic with parameter", app: func() *runtime.App { @@ -117,10 +181,13 @@ func TestHandler_Mqtt(t *testing.T) { require.NoError(t, err) mi.Topics["sensors/1234z/data"] = &store.Topic{Name: "sensors/1234z/data"} + app.Monitor.Mqtt.Messages.WithLabel("foo", "sensors/{sensorId}/data", "sensors/1234z/data").Add(1) + app.Monitor.Mqtt.LastMessage.WithLabel("foo", "sensors/{sensorId}/data", "sensors/1234z/data").Set(1234) + return app }, requestUrl: "http://foo.api/api/services/mqtt/foo/topics", - responseBody: `[{"name":"sensors/{sensorId}/data","instances":[{"name":"sensors/1234z/data","parameters":{"sensorId":"1234z"}}],"metrics":{"mqtt_messages_total":0,"mqtt_message_timestamp":0}}]`, + responseBody: `[{"name":"sensors/{sensorId}/data","parameters":["sensorId"],"instances":[{"name":"sensors/1234z/data","parameters":{"sensorId":"1234z"},"metrics":{"messages_total":1,"message_timestamp":1234}}],"metrics":{"messages_total":1,"message_timestamp":1234}}]`, }, } @@ -157,7 +224,7 @@ func TestHandler_Mqtt_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewApp(runtimetest.WithMqttInfo("foo", getMqttInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","version":"1.0","type":"mqtt","metrics":{"mqtt_messages_total":1,"mqtt_message_timestamp":12345678}}]`, + responseBody: `[{"name":"foo","version":"1.0","type":"mqtt","metrics":{"messages_total":1,"message_timestamp":12345678}}]`, addMetrics: func(monitor *monitor.Monitor) { monitor.Mqtt.Messages.WithLabel("foo", "topic").Add(1) monitor.Mqtt.LastMessage.WithLabel("foo", "topic").Set(12345678) @@ -168,14 +235,14 @@ func TestHandler_Mqtt_Metrics(t *testing.T) { app: runtimetest.NewApp( runtimetest.WithMqttInfo("foo", getMqttInfo( asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"), - asyncapi3test.WithChannel("foo"), + asyncapi3test.WithChannel("channel-1"), ), ))), requestUrl: "http://foo.api/api/services/mqtt/foo", - responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"foo","metrics":{"mqtt_messages_total":1,"mqtt_message_timestamp":12345678}}]}`, + responseBody: `{"name":"foo","version":"1.0","topics":[{"name":"channel-1","metrics":{"messages_total":1,"message_timestamp":12345678}}]}`, addMetrics: func(monitor *monitor.Monitor) { - monitor.Mqtt.Messages.WithLabel("foo", "topic").Add(1) - monitor.Mqtt.LastMessage.WithLabel("foo", "topic").Set(12345678) + monitor.Mqtt.Messages.WithLabel("foo", "channel-1").Add(1) + monitor.Mqtt.LastMessage.WithLabel("foo", "channel-1").Set(12345678) }, }, } diff --git a/api/handler_websocket.go b/api/handler_websocket.go index 70a922ef5..be03b6efa 100644 --- a/api/handler_websocket.go +++ b/api/handler_websocket.go @@ -1,6 +1,7 @@ package api import ( + "maps" "mokapi/providers/asyncapi3" "mokapi/runtime" "mokapi/runtime/metrics" @@ -41,13 +42,15 @@ type websocketChannel struct { Description string `json:"description,omitempty"` Messages map[string]messageConfig `json:"messages,omitempty"` Tags []kafkaTag `json:"tags,omitempty"` + Parameters []string `json:"parameters,omitempty"` Instances []websocketChannelInstance `json:"instances,omitempty"` Metrics websocketChannelMetrics `json:"metrics,omitempty"` } type websocketChannelInstance struct { - Name string `json:"name"` - Parameters map[string]string `json:"parameters,omitempty"` + Name string `json:"name"` + Parameters map[string]string `json:"parameters,omitempty"` + Metrics websocketChannelMetrics `json:"metrics,omitempty"` } type websocketClient struct { @@ -57,8 +60,8 @@ type websocketClient struct { } type websocketChannelMetrics struct { - NumMessages float64 `json:"websocket_messages_total"` - LastMessageTime float64 `json:"websocket_message_timestamp"` + NumMessages float64 `json:"messages_total"` + LastMessageTime float64 `json:"message_timestamp"` } func getWebsocketServices(store *runtime.WebsocketStore, m *monitor.Monitor) []service { @@ -190,34 +193,39 @@ func (h *handler) getWebsocketChannels(w http.ResponseWriter, r *http.Request) { func getWebsocketChannels(wi *runtime.WebsocketInfo, m *monitor.Websocket) []websocketChannel { channels := make([]websocketChannel, 0, len(wi.Config.Channels)) - for name, ch := range wi.Config.Channels { + for _, ch := range wi.Config.Channels { if ch.Value == nil { continue } if !ch.Value.IsChannelAvailable("ws") { continue } - addr := name - if ch.Value.Address != "" { - addr = ch.Value.Address - } + addr := ch.Value.ResolveAddress() data := newWebsocketChannel(addr, ch.Value, wi.Config) if len(ch.Value.Parameters) > 0 { + data.Parameters = slices.SortedFunc(maps.Keys(ch.Value.Parameters), func(a, b string) int { + return strings.Compare(a, b) + }) for _, c := range wi.Store.Channels { if err := ch.Value.IsNameValid(c.Name); err == nil { params, _ := ch.Value.ExtractParams(c.Name) - data.Instances = append(data.Instances, websocketChannelInstance{ + inst := websocketChannelInstance{ Name: c.Name, Parameters: params, - }) + } + inst.Metrics = websocketChannelMetrics{ + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name), metrics.ByLabel("instance", c.Name))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name), metrics.ByLabel("instance", c.Name))), + } + data.Instances = append(data.Instances, inst) } } } data.Metrics = websocketChannelMetrics{ - NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name))), - LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name))), + NumMessages: m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name), metrics.ByLabel("channel", addr))), + LastMessageTime: m.LastMessage.Max(metrics.NewQuery(metrics.ByLabel("service", wi.Info.Name), metrics.ByLabel("channel", addr))), } channels = append(channels, data) diff --git a/api/handler_websocket_test.go b/api/handler_websocket_test.go index 943a6154b..40645dfa3 100644 --- a/api/handler_websocket_test.go +++ b/api/handler_websocket_test.go @@ -53,7 +53,7 @@ func TestHandler_Websocket(t *testing.T) { ) }, requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"websocket","metrics":{"websocket_messages_total":0,"websocket_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"bar","version":"1.0","type":"websocket","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get Websocket services", @@ -76,7 +76,7 @@ func TestHandler_Websocket(t *testing.T) { return app }, requestUrl: "http://foo.api/api/services/websocket", - responseBody: `[{"name":"foo","description":"websocket","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"websocket","metrics":{"websocket_messages_total":0,"websocket_message_timestamp":0}}]`, + responseBody: `[{"name":"foo","description":"websocket","contact":{"name":"mokapi","url":"https://mokapi.io","email":"info@mokapi.io"},"version":"1.0","type":"websocket","metrics":{"messages_total":0,"message_timestamp":0}}]`, }, { name: "get specific", @@ -96,6 +96,70 @@ func TestHandler_Websocket(t *testing.T) { requestUrl: "http://foo.api/api/services/websocket/foo", responseBody: `{"name":"foo","description":"bar","version":"1.0","servers":[{"name":"mokapi","host":":80","protocol":"ws","title":"Mokapi Default Server","summary":"Automatically added server because no servers are defined in the AsyncAPI spec","description":""}],"configs":[{"id":"64613435-3062-6462-3033-316532633233","url":"file://foo.yml","provider":"test","time":"2023-12-27T13:01:30Z"}]}`, }, + { + name: "channels with metrics", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("bar", "mqtt", "foo.bar"), + asyncapi3test.WithChannel("channel1"), + asyncapi3test.WithChannel("channel2"), + ), + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + + app.Monitor.Websocket.Messages.WithLabel("foo", "channel1").Add(1) + app.Monitor.Websocket.Messages.WithLabel("foo", "channel2").Add(2) + app.Monitor.Websocket.LastMessage.WithLabel("foo", "channel1").Set(1234) + app.Monitor.Websocket.LastMessage.WithLabel("foo", "channel2").Set(7890) + + _, err := app.Websocket.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + + return app + }, + requestUrl: "http://foo.api/api/services/websocket/foo/channels", + responseBody: `[{"name":"channel1","metrics":{"messages_total":1,"message_timestamp":1234}},{"name":"channel2","metrics":{"messages_total":2,"message_timestamp":7890}}]`, + }, + { + name: "channels with assigned server", + app: func() *runtime.App { + app := runtime.New(&static.Config{}, &dynamictest.Reader{}) + addr := fmt.Sprintf(":%v", try.GetFreePort()) + ws := asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "bar", "1.0"), + asyncapi3test.WithServer("foo", "ws", addr), + asyncapi3test.WithServer("bar", "mqtt", "foo.bar"), + asyncapi3test.WithChannel("channel1", + asyncapi3test.AssignToServer("#/servers/foo"), + ), + asyncapi3test.WithChannel("channel2", + asyncapi3test.AssignToServer("#/servers/bar"), + ), + ) + cfg := &dynamic.Config{ + Info: dynamictest.NewConfigInfo(), + Data: ws, + } + cfg.Info.Time = mustTime("2023-12-27T13:01:30+00:00") + err := ws.Parse(cfg, &dynamictest.Reader{}) + require.NoError(t, err) + + mi, err := app.Websocket.Add(cfg, enginetest.NewEngine()) + require.NoError(t, err) + mi.Store.Channels = map[string]*websocket.Channel{ + "chats/1234z": {Name: "chats/1234z"}, + } + + return app + }, + requestUrl: "http://foo.api/api/services/websocket/foo/channels", + responseBody: `[{"name":"channel1","metrics":{"messages_total":0,"message_timestamp":0}}]`, + }, { name: "channel with parameter", app: func() *runtime.App { @@ -119,10 +183,13 @@ func TestHandler_Websocket(t *testing.T) { "chats/1234z": {Name: "chats/1234z"}, } + app.Monitor.Websocket.Messages.WithLabel("foo", "chats/{chatId}", "chats/1234z").Add(1) + app.Monitor.Websocket.LastMessage.WithLabel("foo", "chats/{chatId}", "chats/1234z").Set(1234) + return app }, requestUrl: "http://foo.api/api/services/websocket/foo/channels", - responseBody: `[{"name":"chats/{chatId}","instances":[{"name":"chats/1234z","parameters":{"chatId":"1234z"}}],"metrics":{"websocket_messages_total":0,"websocket_message_timestamp":0}}]`, + responseBody: `[{"name":"chats/{chatId}","parameters":["chatId"],"instances":[{"name":"chats/1234z","parameters":{"chatId":"1234z"},"metrics":{"messages_total":1,"message_timestamp":1234}}],"metrics":{"messages_total":1,"message_timestamp":1234}}]`, }, } @@ -159,7 +226,7 @@ func TestHandler_Websocket_Metrics(t *testing.T) { name: "service list with metric", app: runtimetest.NewApp(runtimetest.WithWebsocketInfo("foo", getWebsocketInfo(asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"))))), requestUrl: "http://foo.api/api/services", - responseBody: `[{"name":"foo","version":"1.0","type":"websocket","metrics":{"websocket_messages_total":1,"websocket_message_timestamp":12345678}}]`, + responseBody: `[{"name":"foo","version":"1.0","type":"websocket","metrics":{"messages_total":1,"message_timestamp":12345678}}]`, addMetrics: func(monitor *monitor.Monitor) { monitor.Websocket.Messages.WithLabel("foo", "channel-1").Add(1) monitor.Websocket.LastMessage.WithLabel("foo", "channel-1").Set(12345678) @@ -170,11 +237,11 @@ func TestHandler_Websocket_Metrics(t *testing.T) { app: runtimetest.NewApp( runtimetest.WithWebsocketInfo("foo", getWebsocketInfo( asyncapi3test.NewConfig(asyncapi3test.WithTitle("foo"), - asyncapi3test.WithChannel("foo"), + asyncapi3test.WithChannel("channel-1"), ), ))), requestUrl: "http://foo.api/api/services/websocket/foo", - responseBody: `{"name":"foo","version":"1.0","channels":[{"name":"foo","metrics":{"websocket_messages_total":1,"websocket_message_timestamp":12345678}}]}`, + responseBody: `{"name":"foo","version":"1.0","channels":[{"name":"channel-1","metrics":{"messages_total":1,"message_timestamp":12345678}}]}`, addMetrics: func(monitor *monitor.Monitor) { monitor.Websocket.Messages.WithLabel("foo", "channel-1").Add(1) monitor.Websocket.LastMessage.WithLabel("foo", "channel-1").Set(12345678) diff --git a/docs/ldap/quick-start.md b/docs/ldap/quick-start.md index 471d47184..b7ee60295 100644 --- a/docs/ldap/quick-start.md +++ b/docs/ldap/quick-start.md @@ -25,7 +25,9 @@ than interacting with actual LDAP entries. You can later extend this configurati advanced server settings as needed. ```yaml tab=ldap.yaml -ldap: 1.0 # file configuration version not LDAP protocol version +ldap: 1.0 # file configuration version not LDAP protocol version +info: + title: Basic LDAP Server host: :389 ``` @@ -81,6 +83,8 @@ In the LDAP configuration, you would reference this LDIF file as follows: ```yaml tab=ldap.yaml ldap: 1.0 # file configuration version not LDAP protocol version +info: + title: LDAP Server using LDIF host: :389 files: - ./example.ldif diff --git a/engine/common/host.go b/engine/common/host.go index 2d67ce656..ab1cd0077 100644 --- a/engine/common/host.go +++ b/engine/common/host.go @@ -100,10 +100,10 @@ type Host interface { } type Logger interface { - Info(args ...interface{}) - Warn(args ...interface{}) - Error(args ...interface{}) - Debug(args ...interface{}) + Info(args ...any) + Warn(args ...any) + Error(args ...any) + Debug(args ...any) IsLevelEnabled(level string) bool } @@ -112,22 +112,28 @@ type KafkaClient interface { } type KafkaProduceArgs struct { - Cluster string - Topic string - Messages []KafkaMessage - Retry RetryArgs - ClientId string - ScriptFile string + Cluster string + Topic string + Messages []KafkaMessage + AllowAutoTopicCreation bool + Retry RetryArgs + ClientId string + ScriptFile string } type KafkaMessage struct { Key any - Value []byte - Data any + Value Optional[[]byte] + Data Optional[any] Headers map[string]string Partition int } +type Optional[T any] struct { + IsSet bool + Value T +} + type RetryArgs struct { MaxRetryTime time.Duration InitialRetryTime time.Duration @@ -243,13 +249,17 @@ func (e *JobExecution) Title() string { return e.Tags["name"] } +func (e *JobExecution) Info() any { + return e +} + func (e *JobExecution) Domain() string { return "Job Execution" } type FakerNode interface { Name() string - Fake(r *generator.Request) (interface{}, error) + Fake(r *generator.Request) (any, error) } type Store interface { diff --git a/engine/common/http.go b/engine/common/http.go index 9e48163bb..1eb19b11d 100644 --- a/engine/common/http.go +++ b/engine/common/http.go @@ -2,6 +2,7 @@ package common import ( "fmt" + "mokapi/lib/jsonpreview" "reflect" "strings" ) @@ -53,6 +54,21 @@ func (r *HttpEventRequest) String() string { return s } +func (r *HttpEventRequest) Snapshot() any { + if r == nil { + return r + } + + type alias HttpEventRequest + return struct { + *alias + Body jsonpreview.Truncated `json:"body"` + }{ + alias: (*alias)(r), + Body: jsonpreview.Truncated{V: r.Body, Limit: 256 << 10}, + } +} + func (u Url) String() string { sb := strings.Builder{} sb.WriteString(u.Scheme) @@ -71,6 +87,24 @@ func (r *HttpEventResponse) HasBody() bool { return len(r.Body) > 0 || r.Data != nil } +func (r *HttpEventResponse) Snapshot() any { + if r == nil { + return r + } + + limit := 256 << 10 + type alias HttpEventResponse + return struct { + *alias + Body jsonpreview.Truncated `json:"body"` + Data jsonpreview.Truncated `json:"data"` + }{ + alias: (*alias)(r), + Body: jsonpreview.Truncated{V: r.Body, Limit: limit}, + Data: jsonpreview.Truncated{V: r.Data, Limit: limit}, + } +} + func HttpEventHandler(req *HttpEventRequest, res *HttpEventResponse, resources any) (bool, error) { resource := getResource(req.Url, resources) if resource == nil { diff --git a/engine/event.go b/engine/event.go index 00fb89ddd..a996e8e63 100644 --- a/engine/event.go +++ b/engine/event.go @@ -78,12 +78,12 @@ func (e *EventHandler) Has(key string) bool { return false } -func addDefaultTags(args *common.EventArgs, sh *scriptHost) { +func addDefaultTags(args *common.EventArgs, sh *scriptHost, event string) { defaultTags := map[string]string{ "name": sh.name, "file": sh.name, "fileKey": sh.file.Info.Key(), - "event": "http", + "event": event, } if args.Tags == nil { args.Tags = defaultTags diff --git a/engine/event_http.go b/engine/event_http.go index 4dbbe6bca..f368a711c 100644 --- a/engine/event_http.go +++ b/engine/event_http.go @@ -44,7 +44,7 @@ func (e *HttpEventDispatcher) Register(key string, handler *HttpEventHandler) { } func (sh *scriptHost) OnHttp(filter common.HttpFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "http") h := &HttpEventHandler{ Filter: filter, Execute: do, diff --git a/engine/event_kafka.go b/engine/event_kafka.go index 828de4947..e698934dc 100644 --- a/engine/event_kafka.go +++ b/engine/event_kafka.go @@ -44,7 +44,7 @@ func (e *KafkaEventDispatcher) Register(key string, handler *KafkaEventHandler) } func (sh *scriptHost) OnKafka(filter common.KafkaFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "kafka") h := &KafkaEventHandler{ Filter: filter, Execute: do, diff --git a/engine/event_ldap.go b/engine/event_ldap.go index 0119a200d..c8a9b3f55 100644 --- a/engine/event_ldap.go +++ b/engine/event_ldap.go @@ -45,7 +45,7 @@ func (e *LdapEventDispatcher) Register(key string, handler *LdapEventHandler) { } func (sh *scriptHost) OnLdap(filter common.LdapFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "ldap") h := &LdapEventHandler{ Filter: filter, Execute: do, diff --git a/engine/event_mail.go b/engine/event_mail.go index f8a4dc882..ecd2082d0 100644 --- a/engine/event_mail.go +++ b/engine/event_mail.go @@ -45,7 +45,7 @@ func (e *MailEventDispatcher) Register(key string, handler *MailEventHandler) { } func (sh *scriptHost) OnMail(filter common.MailFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "smtp") h := &MailEventHandler{ Filter: filter, Execute: do, diff --git a/engine/event_mqtt.go b/engine/event_mqtt.go index 1e7c1eecf..fb0ccd0f3 100644 --- a/engine/event_mqtt.go +++ b/engine/event_mqtt.go @@ -44,7 +44,7 @@ func (e *MqttEventDispatcher) Register(key string, handler *MqttEventHandler) { } func (sh *scriptHost) OnMqtt(filter common.MqttFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "mqtt") h := &MqttEventHandler{ Filter: filter, Execute: do, diff --git a/engine/event_websocket.go b/engine/event_websocket.go index 62fd26bc9..a4c1af2d0 100644 --- a/engine/event_websocket.go +++ b/engine/event_websocket.go @@ -86,7 +86,7 @@ func (e *WebsocketEventDispatcher) Register(key string, handler *WebsocketEventH } func (sh *scriptHost) OnWebsocket(filter common.WebsocketFilter, do common.EventHandler, args common.EventArgs) { - addDefaultTags(&args, sh) + addDefaultTags(&args, sh, "websocket") h := &WebsocketEventHandler{ Filter: filter, Execute: do, diff --git a/engine/eventhandler_test.go b/engine/eventhandler_test.go index d310fb8b6..988956c92 100644 --- a/engine/eventhandler_test.go +++ b/engine/eventhandler_test.go @@ -134,7 +134,7 @@ export default () => { require.NoError(t, err) require.Len(t, actions, 1) require.Len(t, actions[0].Parameters, 2) - require.Equal(t, `{"method":"","url":{"scheme":"","host":"","port":0,"path":"","query":""},"body":null,"path":null,"query":null,"header":null,"cookie":null,"querystring":null,"api":"","key":"","operationId":"","operation":null}`, actions[0].Parameters[0]) + require.Equal(t, `{"method":"","url":{"scheme":"","host":"","port":0,"path":"","query":""},"path":null,"query":null,"header":null,"cookie":null,"querystring":null,"api":"","key":"","operationId":"","operation":null,"body":null}`, actions[0].Parameters[0]) }, }, { diff --git a/engine/host.go b/engine/host.go index 23cf389fb..8b48a2810 100644 --- a/engine/host.go +++ b/engine/host.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "maps" "mokapi/config/dynamic" "mokapi/config/dynamic/provider/file" "mokapi/engine/common" @@ -19,6 +20,10 @@ import ( log "github.com/sirupsen/logrus" ) +type Snapshotter interface { + Snapshot() any +} + type eventHandler struct { handler common.EventHandler tags map[string]string @@ -112,9 +117,7 @@ func (sh *scriptHost) newJobFunc(handler func(), opt common.JobOptions, schedule "file": sh.name, "fileKey": sh.file.Info.Key(), } - for k, v := range opt.Tags { - tags[k] = v - } + maps.Copy(tags, opt.Tags) t := events.NewTraits().WithNamespace("job").WithName(tags["name"]) if len(sh.sm.GetStores(t)) == 1 { @@ -189,9 +192,7 @@ func (sh *scriptHost) On(event string, handler common.EventHandler, args common. priority: args.Priority, } - for k, v := range args.Tags { - h.tags[k] = v - } + maps.Copy(h.tags, args.Tags) sh.events[event] = append(sh.events[event], h) } @@ -222,22 +223,22 @@ func (sh *scriptHost) AddCleanupFunc(f func()) { sh.cleanupFuncs = append(sh.cleanupFuncs, f) } -func (sh *scriptHost) Info(args ...interface{}) { +func (sh *scriptHost) Info(args ...any) { sh.engine.logger.Info(args...) sh.logEvent("info", "log", args...) } -func (sh *scriptHost) Warn(args ...interface{}) { +func (sh *scriptHost) Warn(args ...any) { sh.engine.logger.Warn(args...) sh.logEvent("warn", "warn", args...) } -func (sh *scriptHost) Error(args ...interface{}) { +func (sh *scriptHost) Error(args ...any) { sh.engine.logger.Error(args...) sh.logEvent("error", "error", args...) } -func (sh *scriptHost) Debug(args ...interface{}) { +func (sh *scriptHost) Debug(args ...any) { sh.engine.logger.Debug(args...) sh.logEvent("debug", "debug", args...) } @@ -248,7 +249,7 @@ func (sh *scriptHost) SetEventLogger(logger func(level, message string)) { sh.eventLogger = logger } -func (sh *scriptHost) logEvent(configuredLevel, eventLevel string, args ...interface{}) { +func (sh *scriptHost) logEvent(configuredLevel, eventLevel string, args ...any) { if !sh.IsLevelEnabled(configuredLevel) { return } @@ -385,6 +386,9 @@ func (sh *scriptHost) endEventHandler() { func getDeepCopy(args []any) []string { result := make([]string, len(args)) for i, arg := range args { + if s, ok := arg.(Snapshotter); ok { + arg = s.Snapshot() + } b, _ := json.Marshal(arg) result[i] = string(b) } diff --git a/engine/kafka.go b/engine/kafka.go index f1d7e7fa5..cfa720782 100644 --- a/engine/kafka.go +++ b/engine/kafka.go @@ -29,47 +29,50 @@ func NewKafkaClient(app *runtime.App) *KafkaClient { } func (c *KafkaClient) Produce(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { - k, t, err := c.tryGet(args.Cluster, args.Topic, args.Retry) + k, ch, err := c.tryGet(args.Cluster, args.Topic, args.Retry) if err != nil { return nil, err } + if args.Topic == "" { + args.Topic = ch.ResolveAddress() + } client := store.NewClient(k.Store, c.app.Monitor.Kafka) + client.AllowAutoTopicCreation = args.AllowAutoTopicCreation client.ClientId = args.ClientId client.ScriptFile = args.ScriptFile var produced []common.KafkaMessageResult for _, m := range args.Messages { - value := m.Data - if m.Value != nil { - value = m.Value - } - keySchema := &asyncapi3.SchemaRef{ Value: &schema.Schema{Type: schema.Types{"string"}, Pattern: "[a-z]{9}"}, } - // if m.Value is not used then select Kafka message config by the data which must be valid - if m.Value == nil && (value == nil || m.Key == nil) { - var payload *asyncapi3.SchemaRef - var msg *asyncapi3.Message + var value any + if m.Value.IsSet { + value = m.Value.Value + } else if m.Data.IsSet { + value = m.Data.Value + } - msg, err = t.Config.GetMessageByPayload(value) - if err != nil { - return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %w", t.Name, err) - } - if msg != nil { - if msg.Bindings != nil && msg.Bindings.Kafka.Key != nil { - keySchema = msg.Bindings.Kafka.Key - } - payload = msg.Payload + // returns the first valid message or the first message if value is nil + var msg *asyncapi3.Message + msg, err = ch.GetMessageByPayload(value) + if !m.Value.IsSet && err != nil { + return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %w", args.Topic, err) + } + var payload *asyncapi3.SchemaRef + if msg != nil { + if msg.Bindings != nil && msg.Bindings.Kafka.Key != nil { + keySchema = msg.Bindings.Kafka.Key } + payload = msg.Payload + } - if value == nil { - value, err = createValue(payload) - if err != nil { - return nil, fmt.Errorf("unable to generate kafka value: %v", err) - } + if !m.Value.IsSet && !m.Data.IsSet { + value, err = createValue(payload) + if err != nil { + return nil, fmt.Errorf("unable to generate kafka value: %v", err) } } @@ -85,19 +88,19 @@ func (c *KafkaClient) Produce(args *common.KafkaProduceArgs) (*common.KafkaProdu headers = append(headers, store.RecordHeader{Name: hk, Value: hv}) } - rec, err := client.Write(t.Name, []store.Record{{ + rec, err := client.Write(args.Topic, []store.Record{{ Key: m.Key, Value: value, Headers: headers, Partition: m.Partition, - SkipValidation: m.Value != nil, + SkipValidation: m.Value.IsSet, }}, media.ContentType{}) if err != nil { - return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %w", t.Name, err) + return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %w", args.Topic, err) } if rec[0].Error != "" { - return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %s", t.Name, rec[0].Error) + return nil, fmt.Errorf("failed to produce message to Kafka topic '%v': %s", args.Topic, rec[0].Error) } produced = append(produced, common.KafkaMessageResult{ @@ -111,13 +114,13 @@ func (c *KafkaClient) Produce(args *common.KafkaProduceArgs) (*common.KafkaProdu return &common.KafkaProduceResult{ Api: k.Info.Name, - Topic: t.Name, + Topic: args.Topic, Messages: produced, }, nil } -func (c *KafkaClient) tryGet(cluster string, topic string, retry common.RetryArgs) (k *runtime.KafkaInfo, t *store.Topic, err error) { +func (c *KafkaClient) tryGet(cluster string, topic string, retry common.RetryArgs) (k *runtime.KafkaInfo, t *asyncapi3.Channel, err error) { count := 0 backoff := retry.InitialRetryTime for { @@ -136,7 +139,7 @@ func (c *KafkaClient) tryGet(cluster string, topic string, retry common.RetryArg } } -func (c *KafkaClient) get(cluster string, topic string) (k *runtime.KafkaInfo, t *store.Topic, err error) { +func (c *KafkaClient) get(cluster string, topic string) (k *runtime.KafkaInfo, ch *asyncapi3.Channel, err error) { if len(cluster) == 0 { if len(topic) == 0 { clusters := c.app.Kafka.List() @@ -149,34 +152,47 @@ func (c *KafkaClient) get(cluster string, topic string) (k *runtime.KafkaInfo, t err = newAmbiguousError("ambiguous topic %v. Specify the cluster", topic) return } - return clusters[0], topics[0], nil + return clusters[0], topics[0].Config, nil } - var topics []*store.Topic + var topics []*asyncapi3.Channel for _, v := range c.app.Kafka.List() { if t := v.Topic(topic); t != nil { k = v if len(cluster) == 0 { cluster = v.Info.Name } - topics = append(topics, t) + topics = append(topics, t.Config) + } else if topic != "" { + for _, ref := range v.Channels { + if ref.Value == nil { + continue + } + if err = ref.Value.IsNameValid(topic); err == nil { + k = v + topics = append(topics, ref.Value) + } + } } } if len(topics) > 1 { err = newAmbiguousError("ambiguous topic %v. Specify the cluster", topic) return } else if len(topics) == 1 { - t = topics[0] + ch = topics[0] } } else { if k = c.app.Kafka.Get(cluster); k != nil { - t = k.Topic(topic) + t := k.Topic(topic) + if t != nil { + ch = t.Config + } } else { return nil, nil, fmt.Errorf("kafka cluster '%v' not found", cluster) } } - if t == nil { + if ch == nil { err = fmt.Errorf("kafka topic '%v' not found", topic) return } @@ -195,7 +211,7 @@ func (c *KafkaClient) getPartition(t *store.Topic, partition int) (*store.Partit return t.Partition(partition), nil } -func createValue(r *asyncapi3.SchemaRef) (value interface{}, err error) { +func createValue(r *asyncapi3.SchemaRef) (value any, err error) { var s asyncapi3.Schema if r != nil && r.Value != nil { s, err = r.GetSchema() diff --git a/engine/kafka_test.go b/engine/kafka_test.go index 4829a3804..a2a20bd84 100644 --- a/engine/kafka_test.go +++ b/engine/kafka_test.go @@ -41,7 +41,7 @@ func TestKafkaClient(t *testing.T) { return asyncapi3test.NewConfig( asyncapi3test.WithInfo("foo", "", ""), - asyncapi3test.AddChannel("foo", ch), + asyncapi3test.AddChannel(topic, ch), asyncapi3test.WithComponentMessage("foo", msg), asyncapi3test.WithOperation("sendAction", asyncapi3test.WithOperationAction("send"), @@ -353,7 +353,7 @@ func TestKafkaClient(t *testing.T) { return createCfg("foo", msg) }, test: func(t *testing.T, e *engine.Engine, app *runtime.App) { - for i := 0; i < 3; i++ { + for i := range 3 { _, _ = app.Kafka.Add(getConfig( asyncapi3test.NewConfig(asyncapi3test.WithInfo(fmt.Sprintf("x%v", i), "", ""))), e) } @@ -785,6 +785,86 @@ func TestKafkaClient(t *testing.T) { require.Equal(t, `bar`, kafka.BytesToString(b.Records[0].Value)) }, }, + { + name: "using data=nil", + cfg: func() *asyncapi3.Config { + msg := asyncapi3test.NewMessage( + asyncapi3test.WithPayloadOpenAPI( + opSchematest.NewAny( + opSchematest.New("object", + opSchematest.WithProperty( + "id", + opSchematest.New( + "string", + opSchematest.WithXml(&opSchema.Xml{Attribute: true}), + ), + ), + opSchematest.WithRequired("id"), + ), + opSchematest.New("null"), + ), + ), + ) + return createCfg("foo", msg) + }, + test: func(t *testing.T, e *engine.Engine, app *runtime.App) { + err := e.AddScript(newScript("test.js", ` + import { produce } from 'mokapi/kafka' + export default function() { + produce({ messages: [{ data: null }] }) + } + `)) + + require.NoError(t, err) + b, errCode := app.Kafka.Get("foo").Store.Topic("foo").Partition(0).Read(0, 1000) + require.Equal(t, kafka.None, errCode) + require.NotNil(t, b) + require.Equal(t, "gbrmarxhk", kafka.BytesToString(b.Records[0].Key)) + require.Equal(t, `null`, kafka.BytesToString(b.Records[0].Value)) + }, + }, + { + name: "using auto create topic", + cfg: func() *asyncapi3.Config { + msg := asyncapi3test.NewMessage( + asyncapi3test.WithPayloadOpenAPI( + opSchematest.New("string"), + ), + ) + + ch := asyncapi3test.NewChannel( + asyncapi3test.WithParameter("param", &asyncapi3.Parameter{}), + asyncapi3test.UseMessage("foo", + &asyncapi3.MessageRef{Value: msg}, + ), + ) + + return asyncapi3test.NewConfig( + asyncapi3test.WithInfo("foo", "", ""), + asyncapi3test.AddChannel("foo.{param}", ch), + asyncapi3test.WithComponentMessage("foo", msg), + asyncapi3test.WithOperation("sendAction", + asyncapi3test.WithOperationAction("send"), + asyncapi3test.WithOperationChannel(ch), + asyncapi3test.UseOperationMessage(msg), + ), + ) + }, + test: func(t *testing.T, e *engine.Engine, app *runtime.App) { + err := e.AddScript(newScript("test.js", ` + import { produce } from 'mokapi/kafka' + export default function() { + produce({ topic: 'foo.bar', messages: [{ data: 'hello' }], allowAutoTopicCreation: true, retry: { retries: 0 } }) + } + `)) + + require.NoError(t, err) + b, errCode := app.Kafka.Get("foo").Topic("foo.bar").Partition(0).Read(0, 1000) + require.Equal(t, kafka.None, errCode) + require.NotNil(t, b) + require.Equal(t, `"hello"`, kafka.BytesToString(b.Records[0].Value)) + }, + }, } for _, tc := range testcases { diff --git a/engine/mokapi_on_test.go b/engine/mokapi_on_test.go index 2c2ac5aa3..69ffa9867 100644 --- a/engine/mokapi_on_test.go +++ b/engine/mokapi_on_test.go @@ -63,7 +63,7 @@ export default () => { run: func(evt common.EventEmitter) []*common.Action { res := &common.HttpEventResponse{} actions := evt.EmitHttp(&common.HttpEventRequest{}, res) - require.Equal(t, &map[string]interface{}{"foo": "bar"}, res.Data) + require.Equal(t, &map[string]any{"foo": "bar"}, res.Data) return actions }, test: func(t *testing.T, actions []*common.Action, _ *test.Hook, err error) { @@ -72,7 +72,8 @@ export default () => { var res *common.HttpEventResponse err = json.Unmarshal([]byte(actions[0].Parameters[1]), &res) - require.Equal(t, map[string]interface{}{"foo": "bar"}, res.Data) + require.NoError(t, err) + require.Equal(t, map[string]any{"foo": "bar"}, res.Data) }, }, { diff --git a/engine/scheduler.go b/engine/scheduler.go index d0bcc0011..3be551202 100644 --- a/engine/scheduler.go +++ b/engine/scheduler.go @@ -47,6 +47,10 @@ func (s *DefaultScheduler) Every(every string, handler func(), opt common.JobOpt var jobOpts []gocron.JobOption if opt.Times > 0 { jobOpts = append(jobOpts, gocron.WithLimitedRuns(uint(opt.Times))) + // Ensure WithLimitedRuns works correctly if job execution outlasts the schedule interval. + // Without SingletonMode, gocron might trigger concurrent runs before the previous run + // finishes and decrements the remaining execution counter, leading to too many runs. + jobOpts = append(jobOpts, gocron.WithSingletonMode(gocron.LimitModeReschedule)) } if !opt.SkipImmediateFirstRun { jobOpts = append(jobOpts, gocron.WithStartAt(gocron.WithStartImmediately())) diff --git a/engine/scheduler_test.go b/engine/scheduler_test.go index 07e61c0c5..00e243469 100644 --- a/engine/scheduler_test.go +++ b/engine/scheduler_test.go @@ -1,10 +1,11 @@ package engine import ( - "github.com/stretchr/testify/require" "mokapi/engine/common" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestScheduler(t *testing.T) { @@ -41,6 +42,7 @@ func TestScheduler(t *testing.T) { test: func(t *testing.T, s Scheduler) { count := 0 _, err := s.Every("100ms", func() { + time.Sleep(200 * time.Millisecond) count++ }, common.JobOptions{Times: 1}) require.NoError(t, err) diff --git a/go.mod b/go.mod index b0dd00d91..ac8303fd4 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/blevesearch/bleve/v2 v2.6.1 github.com/blevesearch/bleve_index_api v1.4.1 github.com/bradleyfalzon/ghinstallation/v2 v2.19.0 - github.com/brianvoe/gofakeit/v7 v7.17.1 + github.com/brianvoe/gofakeit/v7 v7.18.0 github.com/cbroglie/mustache v1.4.2 github.com/coder/websocket v1.8.15 - github.com/dop251/goja v0.0.0-20261004200024-481fdb442bb4 + github.com/dop251/goja v0.0.0-20261014135309-2cb8274eed06 github.com/evanw/esbuild v0.28.2 github.com/fsnotify/fsnotify v1.10.1 github.com/go-co-op/gocron/v2 v2.22.0 diff --git a/go.sum b/go.sum index 4ef752d4e..c2afd4ccf 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/brianvoe/gofakeit/v7 v7.17.0 h1:usvHa73vmAjC1JmX4dpZAm1eQkR0riXaxvilS github.com/brianvoe/gofakeit/v7 v7.17.0/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/brianvoe/gofakeit/v7 v7.17.1 h1:50FLBhTGVJQaj6ysRUu0it8wCdYO2uGM9VfuxI+csEc= github.com/brianvoe/gofakeit/v7 v7.17.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= +github.com/brianvoe/gofakeit/v7 v7.18.0 h1:vn3tb6OKRUazCkzhotd8xEwz/mhkLFbDoR8OUREFm80= +github.com/brianvoe/gofakeit/v7 v7.18.0/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/cbroglie/mustache v1.4.2 h1:yHvAjVmSyYwCmEIYq7kBaZ4A+Q3kSYjJheLdB2H2r9U= github.com/cbroglie/mustache v1.4.2/go.mod h1:Q5dS171cNzDjfoeB6S1/GBl8bUJgIa3t8i3eyL80vzc= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -92,6 +94,8 @@ github.com/dop251/goja v0.0.0-20261001102346-43ac577c6fa3 h1:kI6ni9uYdzgZNqk+TUA github.com/dop251/goja v0.0.0-20261001102346-43ac577c6fa3/go.mod h1:u8yZRUavu+N4EnFFy6J5fVtjE7lEcZ2YyV2GcBXY9c8= github.com/dop251/goja v0.0.0-20261004200024-481fdb442bb4 h1:MRZ0SrzpwIYx0fE7FFz6jeUSC95M/tivyy90sHyt+BA= github.com/dop251/goja v0.0.0-20261004200024-481fdb442bb4/go.mod h1:GrCNKUrVplXZj0juFue0Cj2Xt/eL2Nuu2+y3UTanEo8= +github.com/dop251/goja v0.0.0-20261014135309-2cb8274eed06 h1:r8PdcvqyLRXlJ3JBirFPOdKKNoDpXvXTIoU/Oos9BTY= +github.com/dop251/goja v0.0.0-20261014135309-2cb8274eed06/go.mod h1:Cp9NYOLrEKg/ypIRkUWYXWLuKAyyFGkDmGeJ95Sw0zI= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= diff --git a/images/alpha.Dockerfile b/images/alpha.Dockerfile index 6183f1a28..0fec55658 100644 --- a/images/alpha.Dockerfile +++ b/images/alpha.Dockerfile @@ -1,6 +1,6 @@ ARG VERSION -FROM node:26.11.1@sha256:32fa97f3363975684b08bf4e8a68a47c7905175cc20275b50b974bbd02aba731 as webui +FROM node:26.12.0@sha256:1b84b89ada025e3b14543cbb837ffaa1c1b5273b841c5b94dd1a8f9c1d8cf7fe as webui COPY ./webui ./webui @@ -11,7 +11,7 @@ COPY ./docs ./src/assets/docs RUN npm install RUN npm run build-dashboard -FROM golang:1.27.2-alpine@sha256:85dc1069ac644ea3c527b177303a406eb3358192816cd7f9e5848eb658851673 AS gobuild +FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS gobuild ARG VERSION=dev diff --git a/js/faker/node_test.go b/js/faker/node_test.go index 1c6bd20b5..e804a7664 100644 --- a/js/faker/node_test.go +++ b/js/faker/node_test.go @@ -328,7 +328,7 @@ func TestNode(t *testing.T) { v, err := street.Fake(req) r.NoError(t, err) r.Equal(t, "bar", v) - r.Equal(t, generator.Values{"foo": "bar"}, req.Context.Values) + r.Equal(t, map[string]any{"foo": "bar"}, req.Context.Values) }, }, } diff --git a/js/faker/tree.go b/js/faker/tree.go index d4538b00b..918d0d050 100644 --- a/js/faker/tree.go +++ b/js/faker/tree.go @@ -33,6 +33,8 @@ func convertToNode(v goja.Value, m *Module) *generator.Node { case "fake": fake, _ := goja.AssertFunction(obj.Get(k)) n.Fake = func(r *generator.Request) (any, error) { + b := r.Context.Values == nil + _ = b v, err := m.loop.RunSync(func(vm *goja.Runtime) (goja.Value, error) { param, err := ToJsValue(m.vm, r) if err != nil { @@ -46,7 +48,9 @@ func convertToNode(v goja.Value, m *Module) *generator.Node { if err != nil { panic(m.vm.ToValue(err)) } - r.Context.Values = r2.Context.Values + if r2.Context.Values != nil { + r.Context.Values = r2.Context.Values + } return val, nil }) if err != nil { diff --git a/js/kafka/kafka.go b/js/kafka/kafka.go index 2237ae3f1..4f95e4ff5 100644 --- a/js/kafka/kafka.go +++ b/js/kafka/kafka.go @@ -20,8 +20,8 @@ type kafkaResult struct { *common.KafkaProduceResult // deprecated fields - Key interface{} - Value interface{} + Key any + Value any Offset int64 Headers map[string]string Partition int @@ -41,7 +41,7 @@ func Require(vm *goja.Runtime, module *goja.Object) { _ = obj.Set("produceAsync", f.ProduceAsync) } -func (m *Module) Produce(v goja.Value) interface{} { +func (m *Module) Produce(v goja.Value) any { defer func() { r := recover() if r != nil { @@ -70,7 +70,7 @@ func (m *Module) Produce(v goja.Value) interface{} { return result } -func (m *Module) ProduceAsync(v goja.Value) interface{} { +func (m *Module) ProduceAsync(v goja.Value) any { p, resolve, reject := m.rt.NewPromise() go func() { defer func() { @@ -147,7 +147,7 @@ func (m *Module) mapParams(args goja.Value) (*common.KafkaProduceArgs, error) { if goja.IsUndefined(value) || goja.IsNull(value) { continue } - message.Data = value.Export() + message.Data = common.Optional[any]{Value: value.Export(), IsSet: true} case "headers": if message == nil { message = &common.KafkaMessage{Partition: -1} @@ -165,9 +165,9 @@ func (m *Module) mapParams(args goja.Value) (*common.KafkaProduceArgs, error) { } } case "messages": - records := params.Get(k).Export().([]interface{}) + records := params.Get(k).Export().([]any) for _, item := range records { - rec, ok := item.(map[string]interface{}) + rec, ok := item.(map[string]any) if !ok { return nil, fmt.Errorf("invalid type in messages: expected Object but got %v", util.JsType(item)) } @@ -178,9 +178,9 @@ func (m *Module) mapParams(args goja.Value) (*common.KafkaProduceArgs, error) { if v, ok := rec["value"]; ok { switch val := v.(type) { case []byte: - r.Value = val + r.Value = common.Optional[[]byte]{Value: val, IsSet: true} case string: - r.Value = []byte(val) + r.Value = common.Optional[[]byte]{Value: []byte(val), IsSet: true} case []any: b := make([]byte, len(val)) for i, item := range val { @@ -190,13 +190,13 @@ func (m *Module) mapParams(args goja.Value) (*common.KafkaProduceArgs, error) { return nil, fmt.Errorf("invalid type in messages: byte numbers: %v", item) } } - r.Value = b + r.Value = common.Optional[[]byte]{Value: b, IsSet: true} default: - r.Value = []byte(fmt.Sprintf("%v", v)) + r.Value = common.Optional[[]byte]{Value: []byte(fmt.Sprintf("%v", v)), IsSet: true} } } if d, ok := rec["data"]; ok { - r.Data = d + r.Data = common.Optional[any]{Value: d, IsSet: true} } if h, ok := rec["headers"]; ok { r.Headers = map[string]string{} @@ -220,6 +220,8 @@ func (m *Module) mapParams(args goja.Value) (*common.KafkaProduceArgs, error) { if err != nil { return nil, err } + case "allowAutoTopicCreation": + opt.AllowAutoTopicCreation = params.Get(k).Export().(bool) } } if message != nil { diff --git a/js/kafka/kafka_test.go b/js/kafka/kafka_test.go index 44245d685..b3503a9a2 100644 --- a/js/kafka/kafka_test.go +++ b/js/kafka/kafka_test.go @@ -79,6 +79,22 @@ func TestKafka(t *testing.T) { r.EqualError(t, err, "TEST at mokapi/js/kafka.(*Module).Produce-fm (native)") }, }, + + { + name: "auto create topic", + test: func(t *testing.T, vm *goja.Runtime, host *enginetest.Host) { + host.KafkaClientTest = &enginetest.KafkaClient{ProduceFunc: func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { + r.Equal(t, true, args.AllowAutoTopicCreation) + return &common.KafkaProduceResult{}, nil + }} + + _, err := vm.RunString(` + const kafka = require("mokapi/kafka") + kafka.produce({ allowAutoTopicCreation: true }) + `) + r.NoError(t, err) + }, + }, } for _, tc := range testcases { diff --git a/js/mokapi/app_http.go b/js/mokapi/app_http.go index ddffe1abd..7fba99ba1 100644 --- a/js/mokapi/app_http.go +++ b/js/mokapi/app_http.go @@ -25,11 +25,19 @@ func (h *Http) Route(path string) goja.Value { return newRouteObject(r) } -func (h *Http) Operation(operationId string) goja.Value { +func (h *Http) Operation(operationId string, do goja.Value, vArgs goja.Value) goja.Value { filter := h.filter filter.OperationId = operationId - r := &HttpRoute{filter: filter, m: h.m} - return newRouteObject(r) + + args, err := getOnArgs(h.m.vm, vArgs) + if err != nil { + panic(err) + } + f := getHandler(do, args, h.m.vm, h.m.loop) + + h.m.host.OnHttp(filter, f, common.EventArgs{Tags: args.tags, Priority: args.priority}) + + return newHttpObject(h) } func (h *Http) Method(method string, path string, do goja.Value, vArgs goja.Value) goja.Value { @@ -110,6 +118,8 @@ func (r *HttpObject) Get(key string) goja.Value { return r.vm.ToValue(r.http.Use) case "route": return r.vm.ToValue(r.http.Route) + case "operation": + return r.vm.ToValue(r.http.Operation) case "api": return r.vm.ToValue(r.http.Api) } diff --git a/js/mokapi/app_http_test.go b/js/mokapi/app_http_test.go index 04ed57302..443e6de92 100644 --- a/js/mokapi/app_http_test.go +++ b/js/mokapi/app_http_test.go @@ -168,6 +168,17 @@ m.app.http().route('/pets').use((req, res) => {}).get((req, res) => {}) r.Equal(t, "/pets", handlers[1].Filter.Path) }, }, + { + name: "operationId", + script: ` +const m = require('mokapi') +m.app.http().operation('foo', (req, res) => {}) +`, + test: func(t *testing.T, handlers []handler, err error) { + r.Len(t, handlers, 1) + r.Equal(t, "foo", handlers[0].Filter.OperationId) + }, + }, } for _, tc := range testcases { diff --git a/js/mokapi/app_kafka.go b/js/mokapi/app_kafka.go index 84f69682a..c2c4cbd8f 100644 --- a/js/mokapi/app_kafka.go +++ b/js/mokapi/app_kafka.go @@ -213,22 +213,22 @@ func mapProduceMessage(vMsg *goja.Object) (common.KafkaMessage, error) { case "key": msg.Key = vMsg.Get(propName).Export() case "data": - msg.Data = vMsg.Get(propName).Export() + msg.Data = common.Optional[any]{Value: vMsg.Get(propName).Export(), IsSet: true} case "value": v := vMsg.Get(propName).Export() switch val := v.(type) { case goja.ArrayBuffer: - msg.Value = val.Bytes() + msg.Value = common.Optional[[]byte]{Value: val.Bytes(), IsSet: true} case string, int64, float64, bool: - msg.Value = []byte(fmt.Sprintf("%v", val)) + msg.Value = common.Optional[[]byte]{Value: []byte(fmt.Sprintf("%v", val)), IsSet: true} case nil: - msg.Value = nil + msg.Value = common.Optional[[]byte]{Value: nil, IsSet: true} default: b, err := json.Marshal(val) if err != nil { return msg, fmt.Errorf("kafka produce: value cannot be serialized: %v", err) } - msg.Value = b + msg.Value = common.Optional[[]byte]{Value: b, IsSet: true} } case "headers": diff --git a/js/mokapi/app_kafka_test.go b/js/mokapi/app_kafka_test.go index 09d805759..77aaca877 100644 --- a/js/mokapi/app_kafka_test.go +++ b/js/mokapi/app_kafka_test.go @@ -78,7 +78,7 @@ m.app.kafka().produce('foo', { key: 'key', value: 'val' }) r.Equal(t, common.RetryArgs{MaxRetryTime: 180000000000, InitialRetryTime: 500000000, Factor: 2, Retries: 10}, produces[0].Retry) r.NotEmpty(t, produces[0].ScriptFile) r.Equal(t, "key", produces[0].Messages[0].Key) - r.Equal(t, "val", string(produces[0].Messages[0].Value)) + r.Equal(t, "val", string(produces[0].Messages[0].Value.Value)) r.Equal(t, -1, produces[0].Messages[0].Partition) r.Nil(t, produces[0].Messages[0].Headers) @@ -119,11 +119,11 @@ m.app.kafka().produce('foo', ['bar', 123, 12.3, true, null].map(x => { return { r.Len(t, produces, 1) r.Equal(t, "foo", produces[0].Topic) r.Len(t, produces[0].Messages, 5) - r.Equal(t, "bar", string(produces[0].Messages[0].Value)) - r.Equal(t, "123", string(produces[0].Messages[1].Value)) - r.Equal(t, "12.3", string(produces[0].Messages[2].Value)) - r.Equal(t, "true", string(produces[0].Messages[3].Value)) - r.Equal(t, "", string(produces[0].Messages[4].Value)) + r.Equal(t, "bar", string(produces[0].Messages[0].Value.Value)) + r.Equal(t, "123", string(produces[0].Messages[1].Value.Value)) + r.Equal(t, "12.3", string(produces[0].Messages[2].Value.Value)) + r.Equal(t, "true", string(produces[0].Messages[3].Value.Value)) + r.Equal(t, "", string(produces[0].Messages[4].Value.Value)) }, }, { @@ -138,7 +138,7 @@ uint8View[2] = 64; m.app.kafka().produce('foo', { value: buffer }) `, test: func(t *testing.T, handlers []handler, produces []*common.KafkaProduceArgs, vm *goja.Runtime, err error) { - r.Equal(t, []byte{255, 128, 64, 0, 0, 0, 0, 0}, produces[0].Messages[0].Value) + r.Equal(t, []byte{255, 128, 64, 0, 0, 0, 0, 0}, produces[0].Messages[0].Value.Value) }, }, { @@ -148,7 +148,7 @@ const m = require('mokapi') m.app.kafka().produce('foo', { value: { foo: 'bar' } }) `, test: func(t *testing.T, handlers []handler, produces []*common.KafkaProduceArgs, vm *goja.Runtime, err error) { - r.Equal(t, `{"foo":"bar"}`, string(produces[0].Messages[0].Value)) + r.Equal(t, `{"foo":"bar"}`, string(produces[0].Messages[0].Value.Value)) }, }, { @@ -158,7 +158,7 @@ const m = require('mokapi') m.app.kafka().produce('foo', { data: { foo: 'bar' } }) `, test: func(t *testing.T, handlers []handler, produces []*common.KafkaProduceArgs, vm *goja.Runtime, err error) { - r.Equal(t, map[string]any{"foo": "bar"}, produces[0].Messages[0].Data) + r.Equal(t, map[string]any{"foo": "bar"}, produces[0].Messages[0].Data.Value) }, }, { diff --git a/js/mokapi/app_websocket.go b/js/mokapi/app_websocket.go new file mode 100644 index 000000000..fffdedfb2 --- /dev/null +++ b/js/mokapi/app_websocket.go @@ -0,0 +1 @@ +package mokapi diff --git a/js/mokapi/on.go b/js/mokapi/on.go index 4c39e98aa..14a12d548 100644 --- a/js/mokapi/on.go +++ b/js/mokapi/on.go @@ -2,6 +2,7 @@ package mokapi import ( "bytes" + "crypto/sha256" "encoding/json" "fmt" "mokapi/engine/common" @@ -49,7 +50,7 @@ func (m *Module) On(event string, do goja.Value, vArgs goja.Value) { func getHandler(do goja.Value, args onArgs, vm *goja.Runtime, loop *eventloop.EventLoop) common.EventHandler { return func(ctx *common.EventContext) (bool, error) { - origin, err := getHashes(ctx.Args...) + origin, err := hashOf(ctx.Args) if err != nil { return false, err } @@ -81,7 +82,7 @@ func getHandler(do goja.Value, args onArgs, vm *goja.Runtime, loop *eventloop.Ev return args.track(params...) } - newHashes, err := getHashes(ctx.Args...) + newHashes, err := hashOf(ctx.Args) if err != nil { return false, err } @@ -153,26 +154,18 @@ func getOnArgs(vm *goja.Runtime, args goja.Value) (onArgs, error) { return onArgs{}, nil } -func getHashes(args ...any) ([][]byte, error) { - var result [][]byte - for _, arg := range args { - b, err := json.Marshal(arg) - if err != nil { - return nil, fmt.Errorf("failed to marshal arg: %v", err) - } - result = append(result, b) +func hashOf(v any) ([32]byte, error) { + h := sha256.New() + var sum [32]byte + if err := json.NewEncoder(h).Encode(v); err != nil { + return sum, fmt.Errorf("failed to marshal arg: %v", err) } - return result, nil + copy(sum[:], h.Sum(nil)) + return sum, nil } -func haveChanges(origin [][]byte, new [][]byte) bool { - for i, o := range origin { - n := new[i] - if !bytes.Equal(o, n) { - return true - } - } - return false +func haveChanges(origin, new [32]byte) bool { + return !bytes.Equal(origin[:], new[:]) } func ArgToJs(arg any, vm *goja.Runtime) goja.Value { diff --git a/js/script_kafka_test.go b/js/script_kafka_test.go index 54bf1b91a..9f758e942 100644 --- a/js/script_kafka_test.go +++ b/js/script_kafka_test.go @@ -81,7 +81,7 @@ func TestScript_Kafka_Produce(t *testing.T) { host.KafkaClientTest.ProduceFunc = func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { msg := args.Messages[0] r.Equal(t, "key", msg.Key) - r.Equal(t, "value", msg.Data) + r.Equal(t, "value", msg.Data.Value) r.Equal(t, 2, msg.Partition) return &common.KafkaProduceResult{}, nil } @@ -103,7 +103,7 @@ func TestScript_Kafka_Produce(t *testing.T) { host.KafkaClientTest.ProduceFunc = func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { msg := args.Messages[0] r.Equal(t, "key", msg.Key) - r.Equal(t, "value", msg.Data) + r.Equal(t, "value", msg.Data.Value) r.Equal(t, 2, msg.Partition) return &common.KafkaProduceResult{}, nil } @@ -145,7 +145,7 @@ func TestScript_Kafka_Produce(t *testing.T) { host.KafkaClientTest.ProduceFunc = func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { msg := args.Messages[0] r.Equal(t, "key1", msg.Key) - r.Equal(t, []byte("hello world"), msg.Value) + r.Equal(t, []byte("hello world"), msg.Value.Value) r.Equal(t, map[string]string{"system-id": "foo"}, msg.Headers) r.Equal(t, msg.Partition, 12) return &common.KafkaProduceResult{}, nil @@ -167,7 +167,7 @@ func TestScript_Kafka_Produce(t *testing.T) { test: func(t *testing.T, host *enginetest.Host) { host.KafkaClientTest.ProduceFunc = func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { msg := args.Messages[0] - r.Equal(t, "hello world", msg.Data) + r.Equal(t, "hello world", msg.Data.Value) return &common.KafkaProduceResult{}, nil } diff --git a/kafka/client_context.go b/kafka/client_context.go index 98322dc16..6151b3d92 100644 --- a/kafka/client_context.go +++ b/kafka/client_context.go @@ -20,15 +20,14 @@ func SetUUIDGenerator(g UUIDGenerator) { } type ClientContext struct { - Addr string - ClientId string - ClientSoftwareName string - ClientSoftwareVersion string - Heartbeat time.Time - Member map[string]string - Close func() - AllowAutoTopicCreation bool - ServerAddress string + Addr string + ClientId string + ClientSoftwareName string + ClientSoftwareVersion string + Heartbeat time.Time + Member map[string]string + Close func() + ServerAddress string } func (c *ClientContext) AddGroup(groupName, memberId string) { @@ -56,5 +55,5 @@ func ClientFromContext(ctx context.Context) *ClientContext { } func NewClientContext(ctx context.Context, addr, serverAddress string) context.Context { - return context.WithValue(ctx, clientKey, &ClientContext{Addr: addr, ServerAddress: serverAddress, AllowAutoTopicCreation: true, Heartbeat: time.Now()}) + return context.WithValue(ctx, clientKey, &ClientContext{Addr: addr, ServerAddress: serverAddress, Heartbeat: time.Now()}) } diff --git a/kafka/errors.go b/kafka/errors.go index d8dcc0db7..f324da4e6 100644 --- a/kafka/errors.go +++ b/kafka/errors.go @@ -26,6 +26,7 @@ const ( UnsupportedVersion ErrorCode = 35 TopicAlreadyExists ErrorCode = 36 UnsupportedForMessageFormat ErrorCode = 43 + PolicyViolation ErrorCode = 44 OutOfOrderSequenceNumber ErrorCode = 45 DuplicateSequenceNumber ErrorCode = 46 InvalidProducerEpoch ErrorCode = 47 diff --git a/lib/jsonpreview/truncated.go b/lib/jsonpreview/truncated.go new file mode 100644 index 000000000..c5de6bf63 --- /dev/null +++ b/lib/jsonpreview/truncated.go @@ -0,0 +1,40 @@ +package jsonpreview + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "unicode/utf8" +) + +type Truncated struct { + V any + Limit int +} + +func (t Truncated) MarshalJSON() ([]byte, error) { + data, err := json.Marshal(t.V) + if err != nil { + return nil, err + } + if len(data) <= t.Limit { + return data, nil + } + cut := t.Limit + // do not cut in the middle of a UTF-8 char + for cut > 0 && !utf8.RuneStart(data[cut]) { + cut-- + } + sum := sha256.Sum256(data) + return json.Marshal(struct { + Truncated bool `json:"truncated"` + Size int `json:"size"` + Hash string `json:"hash"` + Preview string `json:"preview"` + }{ + Truncated: true, + Size: len(data), + Hash: hex.EncodeToString(sum[:]), + Preview: string(data[:cut]), + }) +} diff --git a/lua/kafka/kafka.go b/lua/kafka/kafka.go index 828c98470..16a98df1f 100644 --- a/lua/kafka/kafka.go +++ b/lua/kafka/kafka.go @@ -48,7 +48,7 @@ func (m *Module) Produce(state *lua.LState) int { args := &common.KafkaProduceArgs{Cluster: opts.Cluster, Topic: opts.Topic} args.Messages = append(args.Messages, common.KafkaMessage{ Key: opts.Key, - Data: opts.Value, + Data: common.Optional[any]{Value: opts.Value, IsSet: true}, Headers: opts.Headers, Partition: opts.Partition, }) diff --git a/lua/kafka/kafka_test.go b/lua/kafka/kafka_test.go index 415f71863..31dedc3b8 100644 --- a/lua/kafka/kafka_test.go +++ b/lua/kafka/kafka_test.go @@ -2,11 +2,12 @@ package kafka import ( "fmt" - "github.com/stretchr/testify/require" - lua "github.com/yuin/gopher-lua" "mokapi/engine/common" "mokapi/sortedmap" "testing" + + "github.com/stretchr/testify/require" + lua "github.com/yuin/gopher-lua" ) type client struct { @@ -92,8 +93,8 @@ func TestModule_Produce(t *testing.T) { "value", func(t *testing.T, l *lua.LState, m *Module, c *client) { c.produce = func(args *common.KafkaProduceArgs) (*common.KafkaProduceResult, error) { - require.IsType(t, &sortedmap.LinkedHashMap[string, interface{}]{}, args.Messages[0].Data) - foo, _ := args.Messages[0].Data.(*sortedmap.LinkedHashMap[string, interface{}]).Get("foo") + require.IsType(t, &sortedmap.LinkedHashMap[string, any]{}, args.Messages[0].Data.Value) + foo, _ := args.Messages[0].Data.Value.(*sortedmap.LinkedHashMap[string, any]).Get("foo") require.Equal(t, "bar", foo) return nil, nil } diff --git a/mcp/run_events_test.go b/mcp/run_events_test.go index 782d22f5f..b3ae03853 100644 --- a/mcp/run_events_test.go +++ b/mcp/run_events_test.go @@ -73,8 +73,8 @@ func TestEvents(t *testing.T) { app: runtimetest.NewApp( runtimetest.WithEvent(events.NewTraits().WithNamespace("kafka").With("topic", "foo"), &store.KafkaMessageLog{ Offset: 1234, - Key: store.LogValue{Value: "key"}, - Message: store.LogValue{Value: "message"}, + Key: &store.LogValue{Value: "key"}, + Message: &store.LogValue{Value: "message"}, Partition: 8, Api: "cluster", }), diff --git a/npm/go-mokapi/types/http.d.ts b/npm/go-mokapi/types/http.d.ts index b6d17a940..5d7ee22ae 100644 --- a/npm/go-mokapi/types/http.d.ts +++ b/npm/go-mokapi/types/http.d.ts @@ -1,5 +1,3 @@ -import { JSONValue } from "."; - /** * Make GET request. * https://mokapi.io/docs/javascript-api/mokapi-http/get @@ -23,7 +21,7 @@ export function get(url: string, args?: Args): Response; * headers: { 'Content-Type': "application/json" } * }) */ -export function post(url: string, body?: JSONValue, args?: Args): Response; +export function post(url: string, body?: any, args?: Args): Response; /** * Make PUT request. @@ -37,7 +35,7 @@ export function post(url: string, body?: JSONValue, args?: Args): Response; * headers: { 'Content-Type': "application/json" } * }) */ -export function put(url: string, body?: JSONValue, args?: Args): Response; +export function put(url: string, body?: any, args?: Args): Response; /** * Make HEAD request. @@ -60,7 +58,7 @@ export function head(url: string, args?: Args): Response; * @example * patch("https://foo.bar/foo") */ -export function patch(url: string, body?: JSONValue, args?: Args): Response; +export function patch(url: string, body?: any, args?: Args): Response; /** * Make PATCH request. @@ -72,7 +70,7 @@ export function patch(url: string, body?: JSONValue, args?: Args): Response; * @example * del("https://foo.bar/foo") */ -export function del(url: string, body?: JSONValue, args?: Args): Response; +export function del(url: string, body?: any, args?: Args): Response; /** * Make OPTIONS request. @@ -84,7 +82,7 @@ export function del(url: string, body?: JSONValue, args?: Args): Response; * @example * options("https://foo.bar/foo") */ -export function options(url: string, body?: JSONValue, args?: Args): Response; +export function options(url: string, body?: any, args?: Args): Response; /** * Sends an HTTP request and returns a Promise resolving to the response. @@ -224,5 +222,5 @@ export interface Response { * const res = get(url) * res.json() */ - json(): JSONValue; + json(): any; } diff --git a/npm/go-mokapi/types/index.d.ts b/npm/go-mokapi/types/index.d.ts index 63713af37..785af3e23 100644 --- a/npm/go-mokapi/types/index.d.ts +++ b/npm/go-mokapi/types/index.d.ts @@ -9,8 +9,6 @@ * https://mokapi.io/docs/javascript-api/overview */ -import {ProduceRetry} from "./mqtt"; - /** * Attaches an event handler for the given event. * @@ -946,7 +944,7 @@ export enum LdapResultStatus { SizeLimitExceeded = 4, } -export type SmtpEventHandler = (record: SmtpEventMessage) => void | Promise; +export type SmtpEventHandler = (message: SmtpEventMessage) => void | Promise; export interface SmtpEventMessage { server: string; @@ -1218,18 +1216,6 @@ export interface ScheduledEventArgs { runFirstTimeImmediately?: boolean; } -/** - * JavaScript value representable with JSON. - */ -export type JSONValue = null | undefined | boolean | number | string | JSONValue[] | JSONObject; - -/** - * Object representable with JSON. - */ -export interface JSONObject { - [key: string]: JSONValue; -} - /** * Specifies the date-time format defined in [RFC 3339](https://datatracker.ietf.org/doc/html/rfc3339). * This constant can be used when defining or validating datetime strings. @@ -1756,24 +1742,39 @@ export interface HttpRouter { route(path: string): HttpRoute /** - * Returns an `HttpRoute` matched by the given `operationId` as defined - * in the OpenAPI specification. + * Registers a handler directly for the operation identified by the given + * `operationId` as defined in the OpenAPI specification. + * + * Since an `operationId` already uniquely identifies both the path and the + * HTTP method of an operation, no further method scoping is needed. + * This makes it a concise alternative to `route(path).get(handler)` — + * especially useful when paths are long or contain many parameters. * - * Use this as an alternative to `route()` when you prefer to reference - * operations by name rather than by path template. + * Behaves identically to the method shorthand functions such as `get()` and + * `post()` on `HttpRouter`: all registered handlers run in order, and + * `res.stopPropagation()` can be used to prevent subsequent handlers + * from running. * - * @param operationId The `operationId` defined in the OpenAPI operation. - * @returns An `HttpRoute` for the matching operation. + * @param operationId The `operationId` as defined in the OpenAPI operation. + * @param handler The event handler to invoke. + * @param opts Optional handler configuration such as priority and tracking. + * @returns This `HttpRouter` instance for chaining. * * @example + * // Equivalent to: app.api('Petstore').http().route('/pets').get(handler) * import { app } from 'mokapi' * export default function() { * app.api('Petstore').http() - * .operation('listPets') - * .get((req, res) => { res.data = [] }) + * .operation('listPets', (req, res) => { + * res.data = [{ id: 1, name: 'Bello' }] + * }) + * .operation('createPet', (req, res) => { + * res.statusCode = 201 + * res.data = { id: 2, name: req.body.name } + * }) * } */ - operation(operationId: string): HttpRoute + operation(operationId: string, handler: HttpEventHandler, opts?: HttpEventArgs): HttpRouter /** * Narrows this router to a specific API by title. diff --git a/npm/go-mokapi/types/kafka.d.ts b/npm/go-mokapi/types/kafka.d.ts index 4f1285f75..20466bade 100644 --- a/npm/go-mokapi/types/kafka.d.ts +++ b/npm/go-mokapi/types/kafka.d.ts @@ -1,43 +1,39 @@ -import { JSONValue } from "."; - /** - * Sends messages to a Kafka topic. + * Sends one or more messages to a Kafka topic. * https://mokapi.io/docs/javascript-api/mokapi-kafka/produce - * @param args - ProduceArgs object contains Kafka produce arguments. - * @returns The produce result. + * @param args - Produce arguments. If omitted, a random message is written to a random topic. + * @returns Information about the written messages. * @example + * import { produce } from 'mokapi/kafka' + * * export default function() { * const res = produce({ * topic: 'foo', * messages: [ - * { - * key: 'foo-1', - * data: { foo: 'bar' } - * } + * { key: 'foo-1', data: { foo: 'bar' } } * ] - * }); - * console.log(`new kafka message written with offset: ${res.offset}`) + * }) + * console.log(`new kafka message written with offset: ${res.messages[0].offset}`) * } */ export function produce(args?: ProduceArgs): ProduceResult; /** - * Sends messages to a Kafka topic asynchronously. + * Sends one or more messages to a Kafka topic asynchronously. * https://mokapi.io/docs/javascript-api/mokapi-kafka/produce - * @param args - ProduceArgs object contains Kafka produce arguments. - * @returns The produce result. + * @param args - Produce arguments. If omitted, a random message is written to a random topic. + * @returns A promise resolving to information about the written messages. * @example - * export default function() { - * const res = produce({ + * import { produceAsync } from 'mokapi/kafka' + * + * export default async function() { + * const res = await produceAsync({ * topic: 'foo', * messages: [ - * { - * key: 'foo-1', - * data: { foo: 'bar' } - * } + * { key: 'foo-1', data: { foo: 'bar' } } * ] - * }); - * console.log(`new kafka message written with offset: ${res.offset}`) + * }) + * console.log(`new kafka message written with offset: ${res.messages[0].offset}`) * } */ export function produceAsync(args?: ProduceArgs): Promise; @@ -47,204 +43,241 @@ export function produceAsync(args?: ProduceArgs): Promise; * https://mokapi.io/docs/javascript-api/mokapi-kafka/produceargs * @example * export default function() { - * const res = produce({ + * produce({ * topic: 'foo', * messages: [ - * { - * key: 'foo-1', - * data: { foo: 'bar-1' } - * }, - * { - * key: 'foo-2', - * data: { foo: 'bar-2' } - * } + * { key: 'foo-1', data: { foo: 'bar-1' } }, + * { key: 'foo-2', data: { foo: 'bar-2' } } * ] - * }); + * }) * } */ export interface ProduceArgs { - /** Kafka cluster name. Used when topic name is not unique. */ + /** + * Kafka cluster name. Only required if the topic name exists in more than one cluster. + */ cluster?: string; - /** Kafka topic name. If not specified, message will be written to a random topic. */ + /** + * Kafka topic name. If not specified, the message is written to a random topic. + * + * The name can also be a concrete instance of a parameterized channel address, + * e.g. `shipments.eu.updates` for the address `shipments.{region}.updates`. + * If that topic does not exist yet, set {@link allowAutoTopicCreation} to create it. + */ topic?: string; - /** Kafka partition index. If not specified, the message will be written to any partition */ + /** + * Kafka partition index. If not specified, the partition is chosen based on the key. + * Shorthand for a single message; prefer {@link messages}. + */ partition?: number; - /** Kafka message key. If not specified, a random key will be generated based on the topic configuration. */ - key?: JSONValue; + /** + * Kafka message key. If not specified, a random key is generated based on the topic configuration. + * Shorthand for a single message; prefer {@link messages}. + */ + key?: any; - /** Kafka message value. If not specified, a random value will be generated based on the topic configuration. */ - value?: JSONValue; + /** + * Kafka message value. If not specified, a random value is generated based on the topic configuration. + * Shorthand for a single message; prefer {@link messages}. + */ + value?: any; - /** Kafka message headers. */ - headers?: { [name: string]: JSONValue }; + /** + * Kafka message headers. + * Shorthand for a single message; prefer {@link messages}. + */ + headers?: { [name: string]: any }; /** - * An array of Kafka messages + * Messages to write. Each message can define its own partition, key, value and headers. * @see Message * @example * export default function() { - * const res = produce({ + * produce({ * topic: 'foo', * messages: [ - * { - * key: 'foo-1', - * data: { foo: 'bar-1' } - * }, - * { - * key: 'foo-2', - * data: { foo: 'bar-2' } - * } + * { key: 'foo-1', data: { foo: 'bar-1' } }, + * { key: 'foo-2', data: { foo: 'bar-2' } } * ] - * }); + * }) * } */ messages?: Message[]; /** - * The retry option is used if script is executed before Kafka topic is set up. + * Creates the topic if it does not exist yet. + * + * The topic is only created if its name matches a channel address defined in the + * AsyncAPI specification, including parameterized addresses such as + * `shipments.{region}.updates`, and the channel is available on the target server. + * Parameter values must satisfy the parameter's `enum`, if defined. + * @default false + * @example + * export default function() { + * produce({ + * topic: 'shipments.eu.updates', + * allowAutoTopicCreation: true, + * messages: [ + * { key: 'c9bf9e57-1685-4c89-bafb-ff5af830be8a', data: { status: 'IN_TRANSIT' } } + * ] + * }) + * } + */ + allowAutoTopicCreation?: boolean; + + /** + * Retries producing if the script runs before the Kafka topic is set up. + * @see ProduceRetry */ retry?: ProduceRetry; } /** - * Represents a Kafka message + * Represents a Kafka message to write. */ export interface Message { - /** Kafka partition index. If not specified, the message will be written to any partition */ + /** Kafka partition index. If not specified, the partition is chosen based on the key. */ partition?: number; - /** Kafka message key. If not specified, a random key will be generated based on the topic configuration. */ - key?: JSONValue; + /** Kafka message key. If not specified, a random key is generated based on the topic configuration. */ + key?: any; - /** Kafka message value. If data and value are not specified, a random value will be generated based on the topic configuration. */ - data?: JSONValue; + /** + * Kafka message value, validated against the message schema of the topic. + * If neither `data` nor `value` is specified, a random value is generated based on the topic configuration. + */ + data?: any; - /** Kafka message value not validating against schema. If data and value are not specified, a random value will be generated based on the topic configuration. */ + /** + * Raw Kafka message value, written without schema validation. + * Use it to simulate invalid messages. If neither `data` nor `value` is specified, + * a random value is generated based on the topic configuration. + */ value?: string | number | boolean | null; /** Kafka message headers. */ - headers?: { [name: string]: JSONValue }; + headers?: { [name: string]: any }; } /** - * Contains information of the written Kafka message. + * Contains information about the written Kafka messages. * https://mokapi.io/docs/javascript-api/mokapi-kafka/produceresult * @example * export default function() { * const res = produce({ topic: 'foo' }) - * console.log(`new kafka message written with offset: ${res.offset}`) + * for (const msg of res.messages) { + * console.log(`written to partition ${msg.partition} with offset ${msg.offset}`) + * } * } */ export interface ProduceResult { - /** Name of the Kafka cluster where the message was written. */ + /** Name of the Kafka cluster where the messages were written. */ readonly cluster: string; - /** Kafka topic name where the message was written. */ + /** Name of the Kafka topic where the messages were written. */ readonly topic: string; - // ** List of produced Kafka messages */ - messages: MessageResult[]; + /** The written messages, in the same order as in {@link ProduceArgs.messages}. */ + readonly messages: readonly MessageResult[]; - /** Kafka partition where the message was written. */ + /** + * Partition of the first written message. + * @deprecated Use `messages[i].partition` instead. + */ readonly partition: number; - /** The offset of the written message. */ + /** + * Offset of the first written message. + * @deprecated Use `messages[i].offset` instead. + */ readonly offset: number; - /** The key of the written message, */ + /** + * Key of the first written message. + * @deprecated Use `messages[i].key` instead. + */ readonly key: string; - /** The value of the written message. */ + /** + * Value of the first written message. + * @deprecated Use `messages[i].value` instead. + */ readonly value: string; - /** The headers of the written message */ + /** + * Headers of the first written message. + * @deprecated Use `messages[i].headers` instead. + */ readonly headers: { [name: string]: string }; } /** - * Represents the outcome of producing a Kafka message in Mokapi. - * - * Provides details about the target topic location, message payloads, - * and schema validation outcomes. - * + * Represents a single written Kafka message. * https://mokapi.io/docs/javascript-api/mokapi-kafka/kafkamessageresult */ export interface MessageResult { - /** - * The zero-based index of the Kafka partition where the message was written. - */ + /** Zero-based index of the partition where the message was written. */ readonly partition: number; - /** - * The unique sequential identifier (offset) assigned to the message within its partition. - */ + /** Offset assigned to the message within its partition. */ readonly offset: number; - /** - * The key of the written Kafka message, typically used for partitioning. - */ + /** Key of the written message. */ readonly key: string; - /** - * The main payload data of the written Kafka message. - */ + /** Value of the written message. */ readonly value: string; - /** - * Key-value pairs representing the metadata headers attached to the Kafka message. - */ + /** Headers of the written message. */ readonly headers: { [name: string]: string }; } /** - * The retry option can be used to customize the configuration of the retry mechanism. + * A duration, either as a number of milliseconds or as a string with unit suffix, + * such as `"300ms"`, `"1.5s"` or `"2m30s"`. + * Valid units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. + */ +export type Duration = string | number; + +/** + * Configures the retry mechanism used when the topic is not available yet. + * The wait time grows exponentially: `initialRetryTime * factor^(n-1)`, + * capped at {@link maxRetryTime}. + * @example + * export default function() { + * produce({ + * topic: 'foo', + * messages: [{ value: 'value-1' }], + * retry: { initialRetryTime: '500ms', retries: 10 } + * }) + * } */ export interface ProduceRetry { /** - * Maximum wait time for a retry - * MaxRetryTime number express the wait time in milliseconds. - * MaxRetryTime string is a possibly signed sequence of - * decimal numbers, each with optional fraction and a unit suffix, - * such as "300ms", "-1.5h" or "2h45m". - * Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - * @default 30000ms - * @example - * export default function() { - * produce({ topic: 'foo', messages: [{ value: 'value-1' }], retry: { maxRetryTime: '30s' } }) - * } + * Maximum wait time for a single retry. + * @default '30s' */ - maxRetryTime: string | number; + maxRetryTime?: Duration; /** - * Initial value used to calculate the wait time - * InitialRetryTime number express the wait time in milliseconds. - * InitialRetryTime string is a possibly signed sequence of - * decimal numbers, each with optional fraction and a unit suffix, - * such as "300ms", "-1.5h" or "2h45m". - * Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - * @default 200ms - * @example - * export default function() { - * produce({ topic: 'foo', messages: [{ value: 'value-1' }], retry: { initialRetryTime: '2s' } }) - * } + * Wait time before the first retry. + * @default '200ms' */ - initialRetryTime: string | number; + initialRetryTime?: Duration; /** - * Factor for increasing the wait time for next retry. - * 1st retry: 200ms - * 2nd retry: 4 * 200ms = 800ms - * 3th retry: 4 * 800ms = 3200ms + * Factor by which the wait time increases with each retry. + * With the defaults: 1st retry 200ms, 2nd 800ms, 3rd 3200ms. * @default 4 */ - factor: number; + factor?: number; /** - * Max number of retries + * Maximum number of retries. * @default 5 */ - retries: number; + retries?: number; } \ No newline at end of file diff --git a/npm/go-mokapi/types/mustache.d.ts b/npm/go-mokapi/types/mustache.d.ts index d7be4be1b..47f6fe3dc 100644 --- a/npm/go-mokapi/types/mustache.d.ts +++ b/npm/go-mokapi/types/mustache.d.ts @@ -1,5 +1,3 @@ -import { JSONValue } from "."; - /** * Renders the given mustache template with the given data. * https://mokapi.io/docs/javascript-api/mokapi-mustache/render @@ -22,7 +20,7 @@ import { JSONValue } from "."; */ export function render(template: string, data: Scope): string; -export type Scope = null | Scope[] | ScopeObject | JSONValue; +export type Scope = null | Scope[] | ScopeObject | any; /** * Object representable with JSON. diff --git a/npm/go-mokapi/types/yaml.d.ts b/npm/go-mokapi/types/yaml.d.ts index 267a2e7b1..3a4151156 100644 --- a/npm/go-mokapi/types/yaml.d.ts +++ b/npm/go-mokapi/types/yaml.d.ts @@ -1,5 +1,3 @@ -import { JSONValue } from "."; - /** * Parses a YAML string, constructing the JavaScript value or object described by the string. * https://mokapi.io/docs/javascript-api/mokapi-yaml/parse @@ -35,4 +33,4 @@ export function parse(text: string): any; * }) * } */ -export function stringify(value: JSONValue): string; +export function stringify(value: any): string; diff --git a/pkg/cmd/mokapi/flags/api_test.go b/pkg/cmd/mokapi/flags/api_test.go index 7d5c7a90d..9398585cd 100644 --- a/pkg/cmd/mokapi/flags/api_test.go +++ b/pkg/cmd/mokapi/flags/api_test.go @@ -37,6 +37,17 @@ func TestRoot_Api(t *testing.T) { require.Equal(t, true, cfg.Api.Search.InMemory) }, }, + { + name: "search disabled", + cmd: func(t *testing.T) *cli.Command { + cmd := mokapi.NewCmdMokapi() + cmd.SetArgs([]string{"--api-search-enabled", "false"}) + return cmd + }, + test: func(t *testing.T, cfg *static.Config, flags *cli.FlagSet) { + require.Equal(t, false, cfg.Api.Search.Enabled) + }, + }, } for _, tc := range testcases { diff --git a/providers/asyncapi3/channel.go b/providers/asyncapi3/channel.go index fd57f0ccf..1603a22ac 100644 --- a/providers/asyncapi3/channel.go +++ b/providers/asyncapi3/channel.go @@ -5,6 +5,7 @@ import ( "fmt" "mokapi/config/dynamic" "regexp" + "strings" "gopkg.in/yaml.v3" ) @@ -140,19 +141,34 @@ func (c *Channel) IsNameValid(topic string) error { match := re.FindStringSubmatch(topic) if match == nil { - return fmt.Errorf("channel name does not match channel address expression") + return fmt.Errorf("does not match channel address expression '%s'", address) } return nil } +var paramRe = regexp.MustCompile(`\{([^}]+)\}`) + func (c *Channel) ExtractParams(topicName string) (map[string]string, error) { - // Find all {param} names - re := regexp.MustCompile(`\{([^}]+)\}`) - names := re.FindAllStringSubmatch(c.ResolveAddress(), -1) + address := c.ResolveAddress() - // Replace {param} with regex group - pattern := "^" + re.ReplaceAllString(c.ResolveAddress(), `([^/]+)`) + "$" - re = regexp.MustCompile(pattern) + var b strings.Builder + var names []string + b.WriteString("^") + last := 0 + for _, m := range paramRe.FindAllStringSubmatchIndex(address, -1) { + // m[0]:m[1] = "{name}", m[2]:m[3] = "name" + b.WriteString(regexp.QuoteMeta(address[last:m[0]])) + b.WriteString(`([^/]+?)`) + names = append(names, address[m[2]:m[3]]) + last = m[1] + } + b.WriteString(regexp.QuoteMeta(address[last:])) + b.WriteString("$") + + re, err := regexp.Compile(b.String()) + if err != nil { + return nil, err + } match := re.FindStringSubmatch(topicName) if match == nil { @@ -163,7 +179,7 @@ func (c *Channel) ExtractParams(topicName string) (map[string]string, error) { // Build result map params := map[string]string{} for i, name := range names { - params[name[1]] = match[i+1] + params[name] = match[i+1] } return params, nil diff --git a/providers/asyncapi3/kafka/store/client.go b/providers/asyncapi3/kafka/store/client.go index 5309c0ba8..e9cbb308d 100644 --- a/providers/asyncapi3/kafka/store/client.go +++ b/providers/asyncapi3/kafka/store/client.go @@ -43,8 +43,9 @@ type RecordResult struct { } type Client struct { - ClientId string - ScriptFile string + ClientId string + ScriptFile string + AllowAutoTopicCreation bool store *Store monitor *monitor.Kafka @@ -60,7 +61,15 @@ func NewClient(s *Store, m *monitor.Kafka) *Client { func (c *Client) Write(topic string, records []Record, ct media.ContentType) ([]RecordResult, error) { t := c.store.Topic(topic) if t == nil { - return nil, TopicNotFound + if c.AllowAutoTopicCreation { + var err error + t, err = c.store.NewTopic(topic, nil) + if err != nil { + return nil, err + } + } else { + return nil, TopicNotFound + } } var result []RecordResult diff --git a/providers/asyncapi3/kafka/store/create_topics.go b/providers/asyncapi3/kafka/store/create_topics.go index ecd7ca219..fb6097291 100644 --- a/providers/asyncapi3/kafka/store/create_topics.go +++ b/providers/asyncapi3/kafka/store/create_topics.go @@ -3,37 +3,46 @@ package store import ( "mokapi/kafka" "mokapi/kafka/createTopics" - "mokapi/providers/asyncapi3" ) -func (s *Store) createtopics(rw kafka.ResponseWriter, req *kafka.Request) error { +func (s *Store) createTopics(rw kafka.ResponseWriter, req *kafka.Request) error { r := req.Message.(*createTopics.Request) res := &createTopics.Response{} + b := s.getBrokerByPort(req.Host) - for _, t := range r.Topics { - channel := &asyncapi3.Channel{ - Title: t.Name, - Bindings: &asyncapi3.ChannelBindings{}, - } - channel.Bindings.Kafka.Partitions = int(t.NumPartitions) - ops := []*asyncapi3.Operation{ - { - Action: "send", - Channel: &asyncapi3.ChannelRef{Value: channel}, - }, - { - Action: "receive", - Channel: &asyncapi3.ChannelRef{Value: channel}, - }, + for _, rt := range r.Topics { + t := s.Topic(rt.Name) + if t != nil { + appendCreateTopicResponse(res, createTopics.TopicResponse{ + Name: rt.Name, + ErrorCode: kafka.TopicAlreadyExists, + }) + continue } - _, err := s.NewTopic(t.Name, channel, ops) - errCode := kafka.None + t, err := s.NewTopic(rt.Name, b) if err != nil { - errCode = kafka.TopicAlreadyExists + appendCreateTopicResponse(res, createTopics.TopicResponse{ + Name: rt.Name, + ErrorCode: kafka.PolicyViolation, + ErrorMessage: err.Error(), + }) + continue + } + + if rt.NumPartitions >= 0 { + t.updatePartitions(int(rt.NumPartitions)) } - res.Topics = append(res.Topics, createTopics.TopicResponse{Name: t.Name, ErrorCode: errCode}) + + appendCreateTopicResponse(res, createTopics.TopicResponse{ + Name: rt.Name, + NumPartitions: int32(len(t.Partitions)), + }) } return rw.Write(res) } + +func appendCreateTopicResponse(res *createTopics.Response, topic createTopics.TopicResponse) { + res.Topics = append(res.Topics, topic) +} diff --git a/providers/asyncapi3/kafka/store/create_topics_test.go b/providers/asyncapi3/kafka/store/create_topics_test.go index 8f6f2692e..aa09b2d35 100644 --- a/providers/asyncapi3/kafka/store/create_topics_test.go +++ b/providers/asyncapi3/kafka/store/create_topics_test.go @@ -16,43 +16,209 @@ import ( ) func TestCreateTopic(t *testing.T) { - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewKafka()) - defer s.Close() + defaultConfig := asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), + asyncapi3test.WithServer("bar", "kafka", "127.0.0.1:9093"), + asyncapi3test.WithChannel("already.exists"), + asyncapi3test.WithChannel("orders.{region}.{eventType}", + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + asyncapi3test.WithParameter( + "eventType", &asyncapi3.Parameter{ + Enum: []string{"created", "processed"}, + }, + ), + ), + asyncapi3test.WithChannel("products.{region}", + asyncapi3test.AssignToServer("foo"), + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + ), + ) - rr := kafkatest.NewRecorder() - s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ - Topics: []createTopics.Topic{ - { - Name: "test", - NumPartitions: 2, - ReplicationFactor: 1, + testcases := []struct { + name string + cfg *asyncapi3.Config + test func(t *testing.T, s *store.Store) + }{ + { + name: "valid topic name and num partitions", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "orders.eu.created", + NumPartitions: 3, + }, + }, + })) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "orders.eu.created", res.Topics[0].Name) + require.Equal(t, kafka.None, res.Topics[0].ErrorCode) + require.Equal(t, int32(3), res.Topics[0].NumPartitions) }, }, - })) - - res, ok := rr.Message.(*createTopics.Response) - require.True(t, ok) - require.Equal(t, "test", res.Topics[0].Name) - require.Equal(t, kafka.None, res.Topics[0].ErrorCode) -} - -func TestCreateTopic_AlreadyExists(t *testing.T) { - s := store.New(asyncapi3test.NewConfig(asyncapi3test.AddChannel("test", &asyncapi3.Channel{})), enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewKafka()) - defer s.Close() - - rr := kafkatest.NewRecorder() - s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ - Topics: []createTopics.Topic{ - { - Name: "test", - NumPartitions: 2, - ReplicationFactor: 1, + { + name: "valid topic name no num partitions should use default number", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "orders.us.created", + NumPartitions: -1, + }, + }, + })) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "orders.us.created", res.Topics[0].Name) + require.Equal(t, kafka.None, res.Topics[0].ErrorCode) + require.Equal(t, int32(1), res.Topics[0].NumPartitions) + }, + }, + { + name: "invalid topic name", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "test", + NumPartitions: -1, + }, + }, + })) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "test", res.Topics[0].Name) + require.Equal(t, kafka.PolicyViolation, res.Topics[0].ErrorCode) + require.Equal(t, `topic 'test' does not match any channel available on server 'foo': +- does not match channel address expression 'orders.{region}.{eventType}' +- does not match channel address expression 'products.{region}'`, res.Topics[0].ErrorMessage) + }, + }, + { + name: "create twice", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "orders.us.created", + }, + { + Name: "orders.us.created", + }, + }, + })) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 2) + require.Equal(t, "orders.us.created", res.Topics[0].Name) + require.Equal(t, kafka.None, res.Topics[0].ErrorCode) + require.Equal(t, int32(0), res.Topics[0].NumPartitions) + require.Equal(t, "orders.us.created", res.Topics[1].Name) + require.Equal(t, kafka.TopicAlreadyExists, res.Topics[1].ErrorCode) + require.Equal(t, "", res.Topics[1].ErrorMessage) }, }, - })) + { + name: "already exists", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + s.ServeMessage(rr, kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "already.exists", + }, + }, + })) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "already.exists", res.Topics[0].Name) + require.Equal(t, kafka.TopicAlreadyExists, res.Topics[0].ErrorCode) + require.Equal(t, "", res.Topics[0].ErrorMessage) + }, + }, + { + name: "channel with parameter is not enabled for broker", + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "products.us", + }, + }, + }) + r.Host = "127.0.0.1:9093" + s.ServeMessage(rr, r) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "products.us", res.Topics[0].Name) + require.Equal(t, kafka.PolicyViolation, res.Topics[0].ErrorCode) + require.Equal(t, `topic 'products.us' does not match any channel available on server 'bar': +- channel 'products.{region}' is not available for Kafka protocol +- does not match channel address expression 'orders.{region}.{eventType}'`, res.Topics[0].ErrorMessage) + }, + }, + { + name: "channel is not for Kafka available", + cfg: asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "mqtt", "127.0.0.1:1883"), + asyncapi3test.WithChannel("products.{region}", + asyncapi3test.AssignToServer("foo"), + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + ), + ), + test: func(t *testing.T, s *store.Store) { + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 3, &createTopics.Request{ + Topics: []createTopics.Topic{ + { + Name: "products.eu", + }, + }, + }) + s.ServeMessage(rr, r) + res, ok := rr.Message.(*createTopics.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "products.eu", res.Topics[0].Name) + require.Equal(t, kafka.PolicyViolation, res.Topics[0].ErrorCode) + require.Equal(t, `channel 'products.{region}' is not available for Kafka protocol`, res.Topics[0].ErrorMessage) + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + if cfg == nil { + cfg = defaultConfig + } + s := store.New(cfg, enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewKafka()) + defer s.Close() - res, ok := rr.Message.(*createTopics.Response) - require.True(t, ok) - require.Equal(t, "test", res.Topics[0].Name) - require.Equal(t, kafka.TopicAlreadyExists, res.Topics[0].ErrorCode) + tc.test(t, s) + }) + } } diff --git a/providers/asyncapi3/kafka/store/group.go b/providers/asyncapi3/kafka/store/group.go index 9f3a5d0fd..2bcf94245 100644 --- a/providers/asyncapi3/kafka/store/group.go +++ b/providers/asyncapi3/kafka/store/group.go @@ -39,7 +39,7 @@ func (s *Store) newGroup(name string, coordinator *Broker) *Group { g := &Group{ Name: name, } - g.balancer = newGroupBalancer(g, coordinator.kafkaConfig, &groupMonitor{cluster: s.cluster, monitor: s.monitor}) + g.balancer = newGroupBalancer(g, coordinator.kafkaConfig, &groupMonitor{cluster: s.config.Info.Name, monitor: s.monitor}) go g.balancer.run() return g } diff --git a/providers/asyncapi3/kafka/store/log.go b/providers/asyncapi3/kafka/store/log.go index 8d38c64a0..00267aef6 100644 --- a/providers/asyncapi3/kafka/store/log.go +++ b/providers/asyncapi3/kafka/store/log.go @@ -11,21 +11,29 @@ import ( type LogRecord func(log *KafkaMessageLog, traits events.Traits) type KafkaMessageLog struct { - Offset int64 `json:"offset"` - Key LogValue `json:"key"` - Message LogValue `json:"message"` - SchemaId int `json:"schemaId"` - MessageId string `json:"messageId"` - Partition int `json:"partition"` - Headers map[string]LogValue `json:"headers"` - ProducerId int64 `json:"producerId"` - ProducerEpoch int16 `json:"producerEpoch"` - SequenceNumber int32 `json:"sequenceNumber"` - Deleted bool `json:"deleted"` - Api string `json:"api"` - ClientId string `json:"clientId"` - ScriptFile string `json:"script"` - Actions []*common.Action `json:"actions"` + Topic string `json:"topic"` + Offset int64 `json:"offset"` + Key *LogValue `json:"key"` + Message *LogValue `json:"message"` + SchemaId int `json:"schemaId"` + MessageId string `json:"messageId"` + Partition int `json:"partition"` + Headers map[string]*LogValue `json:"headers"` + ProducerId int64 `json:"producerId"` + ProducerEpoch int16 `json:"producerEpoch"` + SequenceNumber int32 `json:"sequenceNumber"` + Deleted bool `json:"deleted"` + Api string `json:"api"` + ClientId string `json:"clientId"` + ScriptFile string `json:"script"` + Actions []*common.Action `json:"actions"` +} + +type KafkaMessageInfo struct { + Topic string `json:"topic"` + Key *LogValue `json:"key"` + Message *LogValue `json:"message"` + Deleted bool `json:"deleted"` } type LogValue struct { @@ -34,10 +42,21 @@ type LogValue struct { } func (l *KafkaMessageLog) Title() string { + if l.Key == nil { + return "" + } if l.Key.Value != "" { return l.Key.Value - } else { - return string(l.Key.Binary) + } + return string(l.Key.Binary) +} + +func (l *KafkaMessageLog) Info() any { + return KafkaMessageInfo{ + Topic: l.Topic, + Key: l.Key, + Message: l.Message, + Deleted: l.Deleted, } } @@ -54,23 +73,30 @@ func (l *KafkaMessageLog) IndexFields() map[string]any { "partition": l.Partition, } - if l.Key.Value != "" { - m["key"] = l.Key.Value - } else { - m["key"] = string(l.Key.Binary) + if l.Key != nil { + if l.Key.Value != "" { + m["key"] = l.Key.Value + } else { + m["key"] = string(l.Key.Binary) + } } - if l.Message.Value != "" { - m["message"] = l.Message.Value - } else { - m["message"] = string(l.Message.Binary) + if l.Message != nil { + if l.Message.Value != "" { + m["message"] = l.Message.Value + } else { + m["message"] = string(l.Message.Binary) + } } var headers []headerIndex for k, h := range l.Headers { - val := h.Value - if val == "" { - val = string(h.Binary) + val := "" + if h != nil { + val = h.Value + if val == "" { + val = string(h.Binary) + } } headers = append(headers, headerIndex{ Key: k, @@ -84,8 +110,8 @@ func (l *KafkaMessageLog) IndexFields() map[string]any { func newKafkaLog(record *kafka.Record) *KafkaMessageLog { return &KafkaMessageLog{ - Key: LogValue{Binary: kafka.Read(record.Key)}, - Message: LogValue{Binary: kafka.Read(record.Value)}, + Key: &LogValue{Binary: kafka.Read(record.Key)}, + Message: &LogValue{Binary: kafka.Read(record.Value)}, Headers: convertHeader(record.Headers), ProducerId: record.ProducerId, ProducerEpoch: record.ProducerEpoch, @@ -98,13 +124,13 @@ type KafkaRequestData interface { } func (s *Store) logRequest(h *kafka.Header, req KafkaRequest) func(res any) { - log := &KafkaRequestLogEvent{Api: s.cluster, Request: req} + log := &KafkaRequestLogEvent{Api: s.config.Info.Name, Request: req} log.Header.set(h) return func(res any) { log.Response = res t := events.NewTraits(). WithNamespace("kafka"). - WithName(s.cluster). + WithName(s.config.Info.Name). With("type", "request"). With("clientId", h.ClientId) _ = s.eh.Push(log, t) @@ -122,6 +148,10 @@ func (l *KafkaRequestLogEvent) Title() string { return l.Request.Title() } +func (l *KafkaRequestLogEvent) Info() any { + return l +} + type KafkaRequest interface { Title() string } @@ -154,6 +184,10 @@ func (r *KafkaJoinGroupRequest) Title() string { return fmt.Sprintf("JoinGroup %s", r.GroupName) } +func (r *KafkaJoinGroupRequest) Info() any { + return r +} + type KafkaJoinGroupResponse struct { KafkaResponseError GenerationId int32 `json:"generationId"` @@ -181,6 +215,10 @@ func (r *KafkaSyncGroupRequest) Title() string { return fmt.Sprintf("SyncGroup %s", r.GroupName) } +func (r *KafkaSyncGroupRequest) Info() any { + return r +} + type KafkaSyncGroupResponse struct { KafkaResponseError ProtocolType string `json:"protocolType"` @@ -196,6 +234,10 @@ func (r *KafkaListOffsetsRequest) Title() string { return "ListOffsets" } +func (r *KafkaListOffsetsRequest) Info() any { + return r +} + type KafkaListOffsetsRequestPartition struct { Partition int `json:"partition"` Timestamp int64 `json:"timestamp"` @@ -226,6 +268,10 @@ func (r *KafkaFindCoordinatorRequest) Title() string { return "FindCoordinator" } +func (r *KafkaFindCoordinatorRequest) Info() any { + return r +} + type KafkaFindCoordinatorResponse struct { KafkaResponseError Host string `json:"host"` @@ -244,6 +290,10 @@ func (r *KafkaInitProducerIdRequest) Title() string { return "InitProducerId" } +func (r *KafkaInitProducerIdRequest) Info() any { + return r +} + type KafkaInitProducerIdResponse struct { KafkaResponseError ProducerId int64 `json:"producerId"` diff --git a/providers/asyncapi3/kafka/store/log_test.go b/providers/asyncapi3/kafka/store/log_test.go index 3ce4ee878..61ec252ab 100644 --- a/providers/asyncapi3/kafka/store/log_test.go +++ b/providers/asyncapi3/kafka/store/log_test.go @@ -11,9 +11,9 @@ func TestKafkaLog_Title(t *testing.T) { v := store.KafkaMessageLog{} require.Equal(t, "", v.Title()) - v = store.KafkaMessageLog{Key: store.LogValue{Value: "foo"}} + v = store.KafkaMessageLog{Key: &store.LogValue{Value: "foo"}} require.Equal(t, "foo", v.Title()) - v = store.KafkaMessageLog{Key: store.LogValue{Binary: []byte("foo")}} + v = store.KafkaMessageLog{Key: &store.LogValue{Binary: []byte("foo")}} require.Equal(t, "foo", v.Title()) } diff --git a/providers/asyncapi3/kafka/store/metadata.go b/providers/asyncapi3/kafka/store/metadata.go index b50150edd..a8c8d8e4b 100644 --- a/providers/asyncapi3/kafka/store/metadata.go +++ b/providers/asyncapi3/kafka/store/metadata.go @@ -3,6 +3,7 @@ package store import ( "mokapi/kafka" "mokapi/kafka/metaData" + "mokapi/providers/asyncapi3" "path" ) @@ -10,16 +11,14 @@ func (s *Store) metadata(rw kafka.ResponseWriter, req *kafka.Request) error { r := req.Message.(*metaData.Request) brokers := s.Brokers() - ctx := kafka.ClientFromContext(req.Context) - ctx.AllowAutoTopicCreation = r.AllowAutoTopicCreation res := &metaData.Response{ Brokers: make([]metaData.ResponseBroker, 0, len(brokers)), Topics: make([]metaData.ResponseTopic, 0, len(r.Topics)), - ClusterId: s.cluster, + ClusterId: s.config.Info.Name, } - // Mokapi does no leader management, therefore only the current server is returned as the broker. + // Mokapi does no leader management; therefore, only the current server is returned as the broker. host, port := parseHostAndPort(req.Host) b := s.getBrokerByPort(req.Host) if b != nil && b.Host != "" { @@ -42,6 +41,13 @@ func (s *Store) metadata(rw kafka.ResponseWriter, req *kafka.Request) error { if topic != nil && isTopicAvailable(topic, b) { return topic, kafka.None } + if topic == nil && r.AllowAutoTopicCreation { + var err error + topic, err = s.NewTopic(name, b) + if err == nil { + return topic, kafka.None + } + } return nil, kafka.UnknownTopicOrPartition } } else { @@ -86,10 +92,14 @@ func (s *Store) metadata(rw kafka.ResponseWriter, req *kafka.Request) error { } func isTopicAvailable(t *Topic, b *Broker) bool { - if len(t.Config.Servers) == 0 { + return isChannelAvailable(t.Config, b) +} + +func isChannelAvailable(ch *asyncapi3.Channel, b *Broker) bool { + if len(ch.Servers) == 0 { return true } - for _, s := range t.Config.Servers { + for _, s := range ch.Servers { name := path.Base(s.Ref) if b != nil && name == b.Name { return true diff --git a/providers/asyncapi3/kafka/store/metadata_test.go b/providers/asyncapi3/kafka/store/metadata_test.go index ad3237056..60faeedf2 100644 --- a/providers/asyncapi3/kafka/store/metadata_test.go +++ b/providers/asyncapi3/kafka/store/metadata_test.go @@ -22,11 +22,11 @@ import ( func TestMetadata(t *testing.T) { testcases := []struct { name string - fn func(t *testing.T, s *store.Store) + test func(t *testing.T, s *store.Store) }{ { - "default", - func(t *testing.T, s *store.Store) { + name: "default", + test: func(t *testing.T, s *store.Store) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithInfo("Cluster Test", "", ""), asyncapi3test.WithServer("", "kafka", "127.0.0.1:9092"), @@ -60,13 +60,11 @@ func TestMetadata(t *testing.T) { require.Len(t, res.Topics[0].Partitions[0].ReplicaNodes, 0) require.Len(t, res.Topics[0].Partitions[0].IsrNodes, 0) require.False(t, res.Topics[0].IsInternal) - - require.False(t, kafka.ClientFromContext(r.Context).AllowAutoTopicCreation) }, }, { - "with specific topic and two partitions", - func(t *testing.T, s *store.Store) { + name: "with specific topic and two partitions", + test: func(t *testing.T, s *store.Store) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithChannel("foo", asyncapi3test.WithKafkaChannelBinding(asyncapi3.TopicBindings{Partitions: 2})), asyncapi3test.WithChannel("foo2", asyncapi3test.WithKafkaChannelBinding(asyncapi3.TopicBindings{Partitions: 2})), @@ -87,8 +85,8 @@ func TestMetadata(t *testing.T) { }, }, { - "with invalid topic", - func(t *testing.T, s *store.Store) { + name: "with invalid topic", + test: func(t *testing.T, s *store.Store) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithChannel("foo"))) @@ -105,8 +103,8 @@ func TestMetadata(t *testing.T) { }, }, { - "create auto topic true", - func(t *testing.T, s *store.Store) { + name: "create auto topic true", + test: func(t *testing.T, s *store.Store) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithChannel("foo"))) @@ -115,13 +113,11 @@ func TestMetadata(t *testing.T) { AllowAutoTopicCreation: true, }) s.ServeMessage(rr, r) - - require.True(t, kafka.ClientFromContext(r.Context).AllowAutoTopicCreation) }, }, { - "with invalid topic name", - func(t *testing.T, s *store.Store) { + name: "with invalid topic name", + test: func(t *testing.T, s *store.Store) { for _, name := range []string{"", ".", "..", "event?", strings.Repeat("a", 250)} { testName := name if len(name) > 10 { @@ -142,8 +138,8 @@ func TestMetadata(t *testing.T) { }, }, { - "only server with empty protocol (backward compatibility) and kafka", - func(t *testing.T, s *store.Store) { + name: "only server with empty protocol (backward compatibility) and kafka", + test: func(t *testing.T, s *store.Store) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), asyncapi3test.WithServer("bar", "amqp", "127.0.0.1:9093"), @@ -162,8 +158,8 @@ func TestMetadata(t *testing.T) { }, }, { - "2.2.0 assigning channels to servers", - func(t *testing.T, s *store.Store) { + name: "2.2.0 assigning channels to servers", + test: func(t *testing.T, s *store.Store) { cfg := asyncapi3test.NewConfig( asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), asyncapi3test.WithServer("bar", "kafka", "127.0.0.1:9093"), @@ -172,21 +168,19 @@ func TestMetadata(t *testing.T) { err := cfg.Parse(&dynamic.Config{Data: cfg}, &dynamictest.Reader{}) require.NoError(t, err) s.Update(cfg) + rr := kafkatest.NewRecorder() r := kafkatest.NewRequest("kafkatest", 4, &metaData.Request{}) r.Host = "127.0.0.1:9092" s.ServeMessage(rr, r) - res, ok := rr.Message.(*metaData.Response) require.True(t, ok) - require.Len(t, res.Topics, 1) require.Equal(t, "foo", res.Topics[0].Name) r = kafkatest.NewRequest("kafkatest", 4, &metaData.Request{}) r.Host = "127.0.0.1:9093" s.ServeMessage(rr, r) - res, ok = rr.Message.(*metaData.Response) require.True(t, ok) require.Len(t, res.Topics, 0) @@ -194,12 +188,127 @@ func TestMetadata(t *testing.T) { r = kafkatest.NewRequest("kafkatest", 4, &metaData.Request{}) r.Host = "foo.bar:9093" s.ServeMessage(rr, r) - res, ok = rr.Message.(*metaData.Response) require.True(t, ok) require.Len(t, res.Topics, 0) }, }, + { + name: "allow topic creation", + test: func(t *testing.T, s *store.Store) { + cfg := asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), + asyncapi3test.WithChannel("orders.{region}.{eventType}", + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + asyncapi3test.WithParameter( + "eventType", &asyncapi3.Parameter{ + Enum: []string{"created", "processed"}, + }, + ), + ), + ) + + err := cfg.Parse(&dynamic.Config{Data: cfg}, &dynamictest.Reader{}) + require.NoError(t, err) + s.Update(cfg) + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 4, &metaData.Request{ + AllowAutoTopicCreation: true, + Topics: []metaData.TopicName{{Name: "orders.us.created"}}, + }) + r.Host = "127.0.0.1:9092" + s.ServeMessage(rr, r) + + res, ok := rr.Message.(*metaData.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "orders.us.created", res.Topics[0].Name) + + topic := s.Topic("orders.us.created") + require.Equal(t, "orders.us.created", topic.Name) + }, + }, + { + name: "allow topic creation using address", + test: func(t *testing.T, s *store.Store) { + cfg := asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), + asyncapi3test.WithChannel("channel-1", + asyncapi3test.WithChannelAddress("orders.{region}.{eventType}"), + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + asyncapi3test.WithParameter( + "eventType", &asyncapi3.Parameter{ + Enum: []string{"created", "processed"}, + }, + ), + ), + ) + + err := cfg.Parse(&dynamic.Config{Data: cfg}, &dynamictest.Reader{}) + require.NoError(t, err) + s.Update(cfg) + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 4, &metaData.Request{ + AllowAutoTopicCreation: true, + Topics: []metaData.TopicName{{Name: "orders.us.created"}}, + }) + r.Host = "127.0.0.1:9092" + s.ServeMessage(rr, r) + + res, ok := rr.Message.(*metaData.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "orders.us.created", res.Topics[0].Name) + + topic := s.Topic("orders.us.created") + require.Equal(t, "orders.us.created", topic.Name) + }, + }, + { + name: "do not allow topic creation", + test: func(t *testing.T, s *store.Store) { + cfg := asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "kafka", "127.0.0.1:9092"), + asyncapi3test.WithChannel("orders.{region}.{eventType}", + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + asyncapi3test.WithParameter( + "eventType", &asyncapi3.Parameter{ + Enum: []string{"created", "processed"}, + }, + ), + ), + ) + + err := cfg.Parse(&dynamic.Config{Data: cfg}, &dynamictest.Reader{}) + require.NoError(t, err) + s.Update(cfg) + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 4, &metaData.Request{ + AllowAutoTopicCreation: false, + Topics: []metaData.TopicName{{Name: "orders.us.created"}}, + }) + r.Host = "127.0.0.1:9092" + s.ServeMessage(rr, r) + + res, ok := rr.Message.(*metaData.Response) + require.True(t, ok) + require.Len(t, res.Topics, 1) + require.Equal(t, "orders.us.created", res.Topics[0].Name) + require.Equal(t, kafka.UnknownTopicOrPartition, res.Topics[0].ErrorCode) + }, + }, } t.Parallel() @@ -210,7 +319,7 @@ func TestMetadata(t *testing.T) { s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), &eventstest.Handler{}, monitor.NewKafka()) defer s.Close() - tc.fn(t, s) + tc.test(t, s) }) } } diff --git a/providers/asyncapi3/kafka/store/offset_commit.go b/providers/asyncapi3/kafka/store/offset_commit.go index c012eebea..6522208db 100644 --- a/providers/asyncapi3/kafka/store/offset_commit.go +++ b/providers/asyncapi3/kafka/store/offset_commit.go @@ -82,6 +82,6 @@ func (s *Store) processMetricsOffsetCommit(ctx context.Context, g *Group, topic } lag := float64(partition.Offset() - g.Commits[topic][partition.Index]) - m.Lags.WithLabel(s.cluster, g.Name, topic, strconv.Itoa(partition.Index)).Set(lag) - m.Commits.WithLabel(s.cluster, g.Name, topic, strconv.Itoa(partition.Index)).Set(float64(g.Commits[topic][partition.Index])) + m.Lags.WithLabel(s.config.Info.Name, g.Name, topic, strconv.Itoa(partition.Index)).Set(lag) + m.Commits.WithLabel(s.config.Info.Name, g.Name, topic, strconv.Itoa(partition.Index)).Set(float64(g.Commits[topic][partition.Index])) } diff --git a/providers/asyncapi3/kafka/store/offset_fetch.go b/providers/asyncapi3/kafka/store/offset_fetch.go index f63af80ae..0ad6d9cb6 100644 --- a/providers/asyncapi3/kafka/store/offset_fetch.go +++ b/providers/asyncapi3/kafka/store/offset_fetch.go @@ -82,7 +82,7 @@ func validateConsumer(t *Topic, clientId, groupId string) (error, kafka.ErrorCod if op.Action != "receive" { continue } - if op.Bindings.Kafka.ClientId != nil { + if op.Bindings != nil && op.Bindings.Kafka.ClientId != nil { p := parser.Parser{Schema: op.Bindings.Kafka.ClientId} _, err = p.Parse(clientId) if err != nil { @@ -91,7 +91,7 @@ func validateConsumer(t *Topic, clientId, groupId string) (error, kafka.ErrorCod continue } } - if op.Bindings.Kafka.GroupId != nil { + if op.Bindings != nil && op.Bindings.Kafka.GroupId != nil { p := parser.Parser{Schema: op.Bindings.Kafka.GroupId} _, err = p.Parse(groupId) if err != nil { diff --git a/providers/asyncapi3/kafka/store/offset_fetch_test.go b/providers/asyncapi3/kafka/store/offset_fetch_test.go index 487c547cf..1e23ab728 100644 --- a/providers/asyncapi3/kafka/store/offset_fetch_test.go +++ b/providers/asyncapi3/kafka/store/offset_fetch_test.go @@ -321,6 +321,44 @@ func TestOffsetFetch_Validation(t *testing.T) { name string fn func(t *testing.T, s *store.Store, hook *test.Hook) }{ + { + "no binding set", + func(t *testing.T, s *store.Store, hook *test.Hook) { + b := kafkatest.NewBroker(kafkatest.WithHandler(s)) + defer b.Close() + + ch := asyncapi3test.NewChannel() + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithServer("", "kafka", b.Addr), + asyncapi3test.AddChannel("foo", ch), + asyncapi3test.WithOperation("foo", + asyncapi3test.WithOperationAction("receive"), + asyncapi3test.WithOperationChannel(ch), + ))) + + err := b.Client().JoinSyncGroup("foo", "bar", 3, 3) + require.NoError(t, err) + + res, err := b.Client().OffsetFetch(3, &offsetFetch.Request{ + GroupId: "bar", + Topics: []offsetFetch.RequestTopic{ + { + Name: "foo", + PartitionIndexes: []int32{0}, + }, + }}) + + require.Equal(t, kafka.None, res.ErrorCode) + require.Len(t, res.Topics, 1) + require.Len(t, res.Topics[0].Partitions, 1) + + p := res.Topics[0].Partitions[0] + require.Equal(t, kafka.None, p.ErrorCode) + require.Equal(t, int64(-1), p.CommittedOffset) + + require.Equal(t, 5, len(hook.Entries)) + }, + }, { "invalid clientId", func(t *testing.T, s *store.Store, hook *test.Hook) { diff --git a/providers/asyncapi3/kafka/store/partition.go b/providers/asyncapi3/kafka/store/partition.go index 7bfd86dbd..a5c14f609 100644 --- a/providers/asyncapi3/kafka/store/partition.go +++ b/providers/asyncapi3/kafka/store/partition.go @@ -98,6 +98,9 @@ func newPartition(index int, brokers Brokers, logger LogRecord, trigger Trigger, } func (p *Partition) Read(offset int64, maxBytes int) (kafka.RecordBatch, kafka.ErrorCode) { + if p == nil { + return kafka.RecordBatch{}, kafka.UnknownTopicOrPartition + } batch := kafka.NewRecordBatch() if offset < p.StartOffset() { return batch, kafka.OffsetOutOfRange @@ -222,6 +225,7 @@ func (p *Partition) write(batch kafka.RecordBatch, opts WriteOptions) (WriteResu baseTime = r.Time } + kLog.Topic = p.Topic.Name kLog.ClientId = opts.ClientId kLog.ScriptFile = opts.ScriptFile diff --git a/providers/asyncapi3/kafka/store/produce_test.go b/providers/asyncapi3/kafka/store/produce_test.go index 20fccfeed..e2c11bb90 100644 --- a/providers/asyncapi3/kafka/store/produce_test.go +++ b/providers/asyncapi3/kafka/store/produce_test.go @@ -13,8 +13,10 @@ import ( "mokapi/providers/asyncapi3/asyncapi3test" "mokapi/providers/asyncapi3/kafka/store" "mokapi/runtime/events" + "mokapi/runtime/metrics" "mokapi/runtime/monitor" "mokapi/schema/json/schema/schematest" + "mokapi/try" "testing" "time" @@ -26,11 +28,11 @@ import ( func TestProduce(t *testing.T) { testcases := []struct { name string - fn func(t *testing.T, s *store.Store, sm *events.StoreManager) + test func(t *testing.T, s *store.Store, sm *events.StoreManager, m *monitor.Kafka) }{ { - "default", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "default", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithServer("foo", "kafka", "127.0.0.1"), asyncapi3test.WithChannel("foo"))) @@ -140,8 +142,8 @@ func TestProduce(t *testing.T) { }, }, { - "Base Offset", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "Base Offset", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithChannel("foo"))) _, _ = s.Topic("foo").Partition(0).Write(kafka.RecordBatch{Records: []*kafka.Record{ @@ -181,8 +183,8 @@ func TestProduce(t *testing.T) { }, }, { - "invalid message value format", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "invalid message value format", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { s.Update(asyncapi3test.NewConfig( asyncapi3test.WithChannel("foo", asyncapi3test.WithMessage("foo", @@ -222,8 +224,8 @@ func TestProduce(t *testing.T) { }, }, { - "invalid client id", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "invalid client id", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithMessage("foo", asyncapi3test.WithContentType("application/json"), @@ -274,8 +276,8 @@ func TestProduce(t *testing.T) { }, }, { - "valid client id", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "valid client id", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithMessage("foo", asyncapi3test.WithContentType("application/json"), @@ -324,8 +326,8 @@ func TestProduce(t *testing.T) { }, }, { - "no partitions", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "no partitions", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithKafkaChannelBinding(asyncapi3.TopicBindings{Partitions: 0})) s.Update(asyncapi3test.NewConfig( @@ -356,8 +358,8 @@ func TestProduce(t *testing.T) { }, }, { - "using producer Id", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "using producer Id", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithMessage("foo", asyncapi3test.WithContentType("application/json"), @@ -423,8 +425,8 @@ func TestProduce(t *testing.T) { }, }, { - "using producer Id but invalid sequence number, to high", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "using producer Id but invalid sequence number, to high", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithMessage("foo", asyncapi3test.WithContentType("application/json"), @@ -481,8 +483,8 @@ func TestProduce(t *testing.T) { }, }, { - "call produce twice for same sequence number", - func(t *testing.T, s *store.Store, sm *events.StoreManager) { + name: "call produce twice for same sequence number", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, _ *monitor.Kafka) { ch := asyncapi3test.NewChannel( asyncapi3test.WithMessage("foo", asyncapi3test.WithContentType("application/json"), @@ -538,15 +540,83 @@ func TestProduce(t *testing.T) { require.Equal(t, "kafka: failed to write to topic 'foo' partition 0: message sequence number already received: 0", hook.LastEntry().Message) }, }, + { + name: "produce to channel using parameter", + test: func(t *testing.T, s *store.Store, sm *events.StoreManager, m *monitor.Kafka) { + s.Update(asyncapi3test.NewConfig( + asyncapi3test.WithServer("foo", "kafka", "127.0.0.1"), + asyncapi3test.WithChannel("products.{region}", + asyncapi3test.WithParameter( + "region", &asyncapi3.Parameter{ + Enum: []string{"us", "eu"}, + }, + ), + ), + )) + _, err := s.NewTopic("products.us", s.Brokers()[0]) + require.NoError(t, err) + + g := s.GetOrCreateGroup("foo", s.Brokers()[0]) + g.Commit("foo", 0, 0) + sm.SetStore(5, events.NewTraits().WithNamespace("kafka")) + + rr := kafkatest.NewRecorder() + r := kafkatest.NewRequest("kafkatest", 3, &produce.Request{ + Topics: []produce.RequestTopic{ + {Name: "products.us", Partitions: []produce.RequestPartition{ + { + Record: kafka.RecordBatch{ + Records: []*kafka.Record{ + { + Offset: 0, + Time: time.Now(), + Key: kafka.NewBytes([]byte("foo")), + Value: kafka.NewBytes([]byte("bar")), + Headers: nil, + }, + }, + }, + }, + }, + }}}) + r.WithContext(monitor.NewKafkaContext(r.Context, m)) + s.ServeMessage(rr, r) + + res, ok := rr.Message.(*produce.Response) + require.True(t, ok) + require.Equal(t, "products.us", res.Topics[0].Name) + require.Equal(t, kafka.None, res.Topics[0].Partitions[0].ErrorCode) + require.Equal(t, int64(0), res.Topics[0].Partitions[0].BaseOffset) + + logs := sm.GetEvents(events.NewTraits().WithNamespace("kafka").WithName("test").With("topic", "products.{region}")) + require.Len(t, logs, 1) + require.Equal(t, "namespace=kafka, name=test, clientId=kafkatest, instance=products.us, partition=0, topic=products.{region}, type=message", logs[0].Traits.String()) + data := logs[0].Data.(*store.KafkaMessageLog) + require.Equal(t, "products.us", data.Topic) + require.Equal(t, "foo", string(data.Key.Binary)) + require.Equal(t, "bar", string(data.Message.Binary)) + require.Equal(t, int64(0), data.Offset) + require.Equal(t, "kafkatest", data.ClientId) + require.Equal(t, "kafkatest", logs[0].Traits.Get("clientId")) + + try.WaitFor(t, func() bool { + n := m.Messages.Sum(metrics.NewQuery()) + return n > 0 + }) + + require.Equal(t, 1.0, m.Messages.Sum(metrics.NewQuery(metrics.ByLabel("instance", "products.us")))) + require.Greater(t, m.LastMessage.Value(metrics.NewQuery(metrics.ByLabel("instance", "products.us"))), 1.0) + }, + }, } for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { sm := &events.StoreManager{} - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), sm, monitor.NewKafka()) + m := monitor.NewKafka() + s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), sm, m) defer s.Close() - tc.fn(t, s, sm) + tc.test(t, s, sm, m) }) } } diff --git a/providers/asyncapi3/kafka/store/store.go b/providers/asyncapi3/kafka/store/store.go index 5bb7b050e..66ca03847 100644 --- a/providers/asyncapi3/kafka/store/store.go +++ b/providers/asyncapi3/kafka/store/store.go @@ -25,6 +25,7 @@ import ( "net/url" "slices" "strconv" + "strings" "sync" "time" @@ -36,12 +37,12 @@ type Store struct { brokers []*Broker topics map[string]*Topic groups map[string]*Group - cluster string eventEmitter common.KafkaEventEmitter eh events.Handler producers map[int64]*ProducerState monitor *monitor.Kafka clients map[string]*kafka.ClientContext + config *asyncapi3.Config nextPID int64 m sync.RWMutex @@ -78,9 +79,17 @@ func (s *Store) Close() { } func (s *Store) Topic(name string) *Topic { + if s == nil { + return nil + } + s.m.RLock() defer s.m.RUnlock() + if s.topics == nil { + return nil + } + if t, ok := s.topics[name]; ok { return t } @@ -88,8 +97,50 @@ func (s *Store) Topic(name string) *Topic { return nil } -func (s *Store) NewTopic(name string, config *asyncapi3.Channel, ops []*asyncapi3.Operation) (*Topic, error) { - return s.addTopic(name, config, ops) +func (s *Store) NewTopic(name string, b *Broker) (*Topic, error) { + var errs []error + for _, ref := range s.config.Channels { + if ref.Value == nil { + continue + } + ch := ref.Value + if len(ch.Parameters) == 0 { + continue + } + if err := ch.IsNameValid(name); err != nil { + errs = append(errs, err) + continue + } + if !ch.IsChannelAvailable("kafka") { + errs = append(errs, fmt.Errorf("channel '%s' is not available for Kafka protocol", ch.Name)) + continue + } + if b != nil { + if !isChannelAvailable(ch, b) { + errs = append(errs, fmt.Errorf("channel '%s' is not available on server '%s'", ch.Name, b.Name)) + } + } + + return s.addTopic(name, ch, s.getOperations(ch)) + } + + switch len(errs) { + case 0: + return nil, fmt.Errorf("topic '%s' does not match any channel available on server '%s'", name, b.Name) + case 1: + return nil, errs[0] + default: + list := slices.Collect(func(yield func(string) bool) { + for _, err := range errs { + yield(fmt.Sprintf("- %s", err.Error())) + } + }) + slices.SortStableFunc(list, func(s1 string, s2 string) int { + return strings.Compare(s1, s2) + }) + return nil, fmt.Errorf("topic '%s' does not match any channel available on server '%s':\n%s", name, b.Name, strings.Join(list, "\n")) + } + } func (s *Store) Topics() []*Topic { @@ -138,7 +189,7 @@ func (s *Store) GetOrCreateGroup(name string, broker *Broker) *Group { } func (s *Store) Update(c *asyncapi3.Config) { - s.cluster = c.Info.Name + s.config = c if c.Servers != nil { for it := c.Servers.Iter(); it.Next(); { name := it.Key() @@ -149,7 +200,7 @@ func (s *Store) Update(c *asyncapi3.Config) { if b := s.getBroker(name); b != nil { host, port := parseHostAndPort(server.Value.Host) if len(host) == 0 { - log.Errorf("unable to update broker '%v' to cluster '%v': missing host in url '%v'", name, s.cluster, server.Value.Host) + log.Errorf("unable to update broker '%v' to cluster '%v': missing host in url '%v'", name, s.config.Info.Name, server.Value.Host) continue } b.Host = host @@ -175,6 +226,9 @@ func (s *Store) Update(c *asyncapi3.Config) { if !ch.Value.IsChannelAvailable("kafka") { continue } + if len(ch.Value.Parameters) > 0 { + continue + } if ch.Value.Address != "" { n = ch.Value.Address @@ -183,8 +237,8 @@ func (s *Store) Update(c *asyncapi3.Config) { if t, ok := s.topics[n]; ok { t.update(ch.Value, s) } else { - if _, err := s.addTopic(n, ch.Value, getOperations(ch.Value, c)); err != nil { - log.Errorf("unable to add topic '%v' to broker '%v': %v", n, s.cluster, err) + if _, err := s.addTopic(n, ch.Value, s.getOperations(ch.Value)); err != nil { + log.Errorf("unable to add topic '%v' to broker '%v': %v", n, s.config.Info.Name, err) } } } @@ -248,7 +302,7 @@ func (s *Store) ServeMessage(rw kafka.ResponseWriter, req *kafka.Request) { case *apiVersion.Request: err = s.apiversion(rw, req) case *createTopics.Request: - err = s.createtopics(rw, req) + err = s.createTopics(rw, req) case *initProducerId.Request: err = s.initProducerID(rw, req) default: @@ -264,10 +318,6 @@ func (s *Store) addTopic(name string, channel *asyncapi3.Channel, ops []*asyncap s.m.Lock() defer s.m.Unlock() - if channel.Address != "" { - name = channel.Address - } - if _, ok := s.topics[name]; ok { return nil, fmt.Errorf("topic %v already exists", name) } @@ -329,9 +379,9 @@ func (s *Store) getBrokerByPort(addr string) *Broker { } func (s *Store) log(log *KafkaMessageLog, traits events.Traits) { - log.Api = s.cluster + log.Api = s.config.Info.Name t := traits.WithNamespace("kafka"). - WithName(s.cluster). + WithName(s.config.Info.Name). With("type", "message") if log.ClientId != "" { t = t.With("clientId", log.ClientId) @@ -346,7 +396,7 @@ func (s *Store) trigger(topic string, partition int, record *kafka.Record, schem } r := &common.KafkaEventRecord{ - Api: s.cluster, + Api: s.config.Info.Name, Topic: topic, Partition: partition, Offset: record.Offset, @@ -441,8 +491,14 @@ func parseHostAndPort(s string) (host string, port int) { } func (s *Store) UpdateMetrics(m *monitor.Kafka, topic *Topic, partition *Partition, batch kafka.RecordBatch) { - m.Messages.WithLabel(s.cluster, topic.Name).Add(float64(len(batch.Records))) - m.LastMessage.WithLabel(s.cluster, topic.Name).Set(float64(time.Now().Unix())) + topicName := topic.Config.ResolveAddress() + labels := []string{s.config.Info.Name, topicName} + if topic.Name != topicName { + labels = append(labels, topic.Name) + } + + m.Messages.WithLabel(labels...).Add(float64(len(batch.Records))) + m.LastMessage.WithLabel(labels...).Set(float64(time.Now().Unix())) for name, g := range s.groups { gt, ok := g.Commits[topic.Name] @@ -454,18 +510,22 @@ func (s *Store) UpdateMetrics(m *monitor.Kafka, topic *Topic, partition *Partiti continue } lag := float64(partition.Offset() - commit) - m.Lags.WithLabel(s.cluster, name, topic.Name, strconv.Itoa(partition.Index)).Set(lag) + m.Lags.WithLabel(s.config.Info.Name, name, topic.Name, strconv.Itoa(partition.Index)).Set(lag) } } -func getOperations(channel *asyncapi3.Channel, config *asyncapi3.Config) []*asyncapi3.Operation { +func (s *Store) getOperations(channel *asyncapi3.Channel) []*asyncapi3.Operation { var ops []*asyncapi3.Operation - for _, op := range config.Operations { + for _, op := range s.config.Operations { if op == nil || op.Value == nil { continue } - if op.Value.Channel.Value == channel { - ops = append(ops, op.Value) + v := op.Value + if v.Channel == nil || v.Channel.Value == nil { + continue + } + if v.Channel.Value == channel { + ops = append(ops, v) } } return ops diff --git a/providers/asyncapi3/kafka/store/store_test.go b/providers/asyncapi3/kafka/store/store_test.go index 84b1bbd44..ebac01e93 100644 --- a/providers/asyncapi3/kafka/store/store_test.go +++ b/providers/asyncapi3/kafka/store/store_test.go @@ -61,26 +61,6 @@ func TestStore(t *testing.T) { require.Len(t, topic.Partitions, 1) }, }, - { - "create topic", - func(t *testing.T, sm *events.StoreManager) { - s := store.New(asyncapi3test.NewConfig(), enginetest.NewEngine(), sm, monitor.NewKafka()) - defer s.Close() - topic, err := s.NewTopic("foo", asyncapi3test.NewChannel(), []*asyncapi3.Operation{}) - require.NoError(t, err) - require.Equal(t, "foo", topic.Name) - require.Equal(t, 1, len(topic.Partitions)) - }, - }, - { - "create topic, already exists", - func(t *testing.T, sm *events.StoreManager) { - s := store.New(asyncapi3test.NewConfig(asyncapi3test.WithChannel("foo")), enginetest.NewEngine(), sm, monitor.NewKafka()) - defer s.Close() - _, err := s.NewTopic("foo", asyncapi3test.NewChannel(), []*asyncapi3.Operation{}) - require.Error(t, err, "topic foo already exists") - }, - }, { "update topic add partition", func(t *testing.T, sm *events.StoreManager) { @@ -136,6 +116,17 @@ func TestStore(t *testing.T) { require.Equal(t, "Validation error count 1:\n\t- #/type: invalid type, expected string but got number", wr.Records[0].BatchIndexErrorMessage) }, }, + { + "channel using parameters", + func(t *testing.T, sm *events.StoreManager) { + s := store.New(asyncapi3test.NewConfig( + asyncapi3test.WithChannel("foo.{region}", + asyncapi3test.WithParameter("region", &asyncapi3.Parameter{})), + ), enginetest.NewEngine(), sm, monitor.NewKafka()) + defer s.Close() + require.Equal(t, 0, len(s.Topics())) + }, + }, } t.Parallel() diff --git a/providers/asyncapi3/kafka/store/topic.go b/providers/asyncapi3/kafka/store/topic.go index ba6364361..0490a4c99 100644 --- a/providers/asyncapi3/kafka/store/topic.go +++ b/providers/asyncapi3/kafka/store/topic.go @@ -27,7 +27,7 @@ type Operation struct { } func (t *Topic) Partition(index int) *Partition { - if index >= len(t.Partitions) { + if t == nil || index >= len(t.Partitions) { return nil } return t.Partitions[index] @@ -62,25 +62,39 @@ func (t *Topic) update(config *asyncapi3.Channel, s *Store) { numPartitions = config.Bindings.Kafka.Partitions } - for i, p := range t.Partitions { + t.updatePartitions(numPartitions) +} + +func (t *Topic) updatePartitions(numPartitions int) { + n := max(numPartitions, len(t.Partitions)) + + for i := range n { + var p *Partition if i >= numPartitions { + p = t.Partitions[i] p.delete() } else { - p.validator = newValidator(config) + if i >= len(t.Partitions) { + s := t.Store() + p = newPartition(i, s.brokers, t.log, s.trigger, t) + t.Partitions = append(t.Partitions, p) + } else { + p = t.Partitions[i] + } + p.validator = newValidator(t.Config) } } - for i := len(t.Partitions); i < numPartitions; i++ { - part := newPartition(i, s.brokers, t.log, s.trigger, t) - part.validator = newValidator(config) - t.Partitions = append(t.Partitions, part) - } - t.Partitions = t.Partitions[:numPartitions] } func (t *Topic) log(r *KafkaMessageLog, traits events.Traits) { - t.logger(r, traits.With("topic", t.Name)) + topicName := t.Config.ResolveAddress() + traits.With("topic", t.Config.ResolveAddress()) + if t.Name != topicName { + traits.With("instance", t.Name) + } + t.logger(r, traits) } func (t *Topic) Store() *Store { diff --git a/providers/asyncapi3/kafka/store/validation.go b/providers/asyncapi3/kafka/store/validation.go index be5a124fb..865dc12e2 100644 --- a/providers/asyncapi3/kafka/store/validation.go +++ b/providers/asyncapi3/kafka/store/validation.go @@ -123,9 +123,9 @@ func newMessageValidator(messageId string, msg *asyncapi3.Message, channel *asyn func (mv *messageValidator) Validate(record *kafka.Record) (*KafkaMessageLog, error) { r := &KafkaMessageLog{ - Key: LogValue{}, - Message: LogValue{}, - Headers: make(map[string]LogValue), + Key: &LogValue{}, + Message: &LogValue{}, + Headers: make(map[string]*LogValue), MessageId: mv.messageId, ProducerId: record.ProducerId, ProducerEpoch: record.ProducerEpoch, @@ -143,11 +143,10 @@ func (mv *messageValidator) Validate(record *kafka.Record) (*KafkaMessageLog, er if mv.payload != nil { if _, err := mv.payload.Validate(record.Value); err != nil { return r, err - } else { - b := kafka.Read(record.Value) - r.Message.Value = string(b) - r.Message.Binary = b } + b := kafka.Read(record.Value) + r.Message.Value = string(b) + r.Message.Binary = b } else { r.Message.Binary = kafka.Read(record.Value) } @@ -183,7 +182,7 @@ type schemaValidator struct { contentType string } -func (v *schemaValidator) Validate(data io.Reader) (interface{}, error) { +func (v *schemaValidator) Validate(data io.Reader) (any, error) { return encoding.DecodeFrom(data, encoding.WithContentType(media.ParseContentType(v.contentType)), encoding.WithParser(v.parser)) } @@ -191,7 +190,7 @@ type headerValidator struct { schema *asyncapi3.SchemaRef } -func (v *headerValidator) Validate(headers []kafka.RecordHeader) (map[string]LogValue, error) { +func (v *headerValidator) Validate(headers []kafka.RecordHeader) (map[string]*LogValue, error) { return parseHeader(headers, v.schema) } @@ -230,15 +229,15 @@ func readSchemaId(payload kafka.Bytes, encoding string) (int, error) { return int(version), err } -func convertHeader(headers []kafka.RecordHeader) map[string]LogValue { - result := map[string]LogValue{} +func convertHeader(headers []kafka.RecordHeader) map[string]*LogValue { + result := map[string]*LogValue{} for _, header := range headers { - result[header.Key] = LogValue{Binary: header.Value} + result[header.Key] = &LogValue{Binary: header.Value} } return result } -func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[string]LogValue, error) { +func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[string]*LogValue, error) { if sr == nil || sr.Value == nil { return convertHeader(headers), nil } @@ -247,7 +246,7 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str m[header.Key] = header.Value } - result := map[string]LogValue{} + result := map[string]*LogValue{} switch s := sr.Value.(type) { case *schema.Schema: if s.Properties == nil { @@ -258,7 +257,7 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str for it := s.Properties.Iter(); it.Next(); { if b, ok := m[it.Key()]; ok { t := it.Value().Type - var v interface{} + var v any var err error switch { case t.IsString(): @@ -296,7 +295,7 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str if err != nil { return nil, err } - result[it.Key()] = LogValue{Value: fmt.Sprintf("%v", v)} + result[it.Key()] = &LogValue{Value: fmt.Sprintf("%v", v)} done[it.Key()] = true } else if slices.Contains(s.Required, it.Key()) { return nil, fmt.Errorf("required property '%s' is missing in header", it.Key()) @@ -314,9 +313,9 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str for _, h := range headers { if _, ok := done[h.Key]; !ok { if utf8.Valid(h.Value) { - result[h.Key] = LogValue{Value: string(h.Value), Binary: h.Value} + result[h.Key] = &LogValue{Value: string(h.Value), Binary: h.Value} } else { - result[h.Key] = LogValue{Binary: h.Value} + result[h.Key] = &LogValue{Binary: h.Value} } } } @@ -328,7 +327,7 @@ func parseHeader(headers []kafka.RecordHeader, sr *asyncapi3.SchemaRef) (map[str return nil, err } b, _ := json.Marshal(val) - result[f.Name] = LogValue{Value: string(b)} + result[f.Name] = &LogValue{Value: string(b)} } } } diff --git a/providers/asyncapi3/kafka/store/validation_test.go b/providers/asyncapi3/kafka/store/validation_test.go index 787870430..802a23a5d 100644 --- a/providers/asyncapi3/kafka/store/validation_test.go +++ b/providers/asyncapi3/kafka/store/validation_test.go @@ -57,26 +57,28 @@ func TestValidation(t *testing.T) { // latest message is first require.Equal(t, &store.KafkaMessageLog{ + Topic: "foo", Offset: 1, - Key: store.LogValue{Value: "", Binary: []byte("key-bar")}, - Message: store.LogValue{Value: "", Binary: []byte("bar")}, + Key: &store.LogValue{Value: "", Binary: []byte("key-bar")}, + Message: &store.LogValue{Value: "", Binary: []byte("bar")}, SchemaId: 0, MessageId: "foo", Partition: 0, - Headers: map[string]store.LogValue{}, + Headers: map[string]*store.LogValue{}, Deleted: false, Api: "test", }, e[0].Data.(*store.KafkaMessageLog)) require.Equal(t, &store.KafkaMessageLog{ + Topic: "foo", Offset: 0, - Key: store.LogValue{Value: "", Binary: []byte("key-foo")}, - Message: store.LogValue{Value: "", Binary: []byte("foo")}, + Key: &store.LogValue{Value: "", Binary: []byte("key-foo")}, + Message: &store.LogValue{Value: "", Binary: []byte("foo")}, SchemaId: 0, MessageId: "foo", Partition: 0, - Headers: map[string]store.LogValue{}, + Headers: map[string]*store.LogValue{}, Deleted: false, Api: "test", }, diff --git a/providers/asyncapi3/mqtt/store/log.go b/providers/asyncapi3/mqtt/store/log.go index b972b45ef..fb55fdced 100644 --- a/providers/asyncapi3/mqtt/store/log.go +++ b/providers/asyncapi3/mqtt/store/log.go @@ -8,7 +8,7 @@ import ( type LogMessage struct { Topic string `json:"topic"` - Message LogValue `json:"message"` + Message *LogValue `json:"message"` Retain bool `json:"retain"` MessageId string `json:"messageId"` Api string `json:"api"` @@ -17,6 +17,11 @@ type LogMessage struct { Actions []*common.Action `json:"actions"` } +type EventMqttInfo struct { + Topic string `json:"topic"` + Message *LogValue `json:"message"` +} + type LogValue struct { Value string `json:"value"` Binary []byte `json:"binary"` @@ -26,6 +31,13 @@ func (l *LogMessage) Title() string { return l.Topic } +func (l *LogMessage) Info() any { + return EventMqttInfo{ + Topic: l.Topic, + Message: l.Message, + } +} + type RequestLogEvent struct { Api string `json:"api"` Type mqtt.Type `json:"type"` @@ -37,6 +49,10 @@ func (r *RequestLogEvent) Title() string { return r.Request.Title() } +func (r *RequestLogEvent) Info() any { + return r +} + type RequestLog interface { Title() string } @@ -54,6 +70,10 @@ func (r *ConnectRequest) Title() string { return "Connect" } +func (r *ConnectRequest) Info() any { + return r +} + type PublishMessage struct { QoS byte `json:"qos"` Retain bool `json:"retain"` @@ -75,6 +95,10 @@ func (r *SubscribeRequest) Title() string { return "Subscribe" } +func (r *SubscribeRequest) Info() any { + return r +} + type SubscribeTopic struct { Name string `json:"name"` QoS byte `json:"qos"` @@ -92,6 +116,10 @@ func (r *DisconnectRequest) Title() string { return "Disconnect" } +func (r *DisconnectRequest) Info() any { + return r +} + func (s *Store) logRequest(req RequestLog, res any, ctx *mqtt.ClientContext) { log := &RequestLogEvent{Api: s.cfg.Info.Name, Request: req} diff --git a/providers/asyncapi3/mqtt/store/publish.go b/providers/asyncapi3/mqtt/store/publish.go index 931d3d55d..5bdddca2b 100644 --- a/providers/asyncapi3/mqtt/store/publish.go +++ b/providers/asyncapi3/mqtt/store/publish.go @@ -146,6 +146,9 @@ func (s *Store) Topic(topic string) (*Topic, bool) { func (s *Store) logMessage(messageId string, topic *Topic, publish *mqtt.PublishRequest, actions []*engine.Action, args PublishArgs) { topicName := topic.cfg.ResolveAddress() labels := []string{s.cfg.Info.Name, topicName} + if topic.Name != topicName { + labels = append(labels, topic.Name) + } s.monitor.Messages.WithLabel(labels...).Add(1) s.monitor.LastMessage.WithLabel(labels...).Set(float64(time.Now().Unix())) @@ -157,20 +160,14 @@ func (s *Store) logMessage(messageId string, topic *Topic, publish *mqtt.Publish With("topic", topicName). With("type", "message"). With("clientId", args.ClientId) - if len(topic.cfg.Parameters) > 0 { - params, err := topic.cfg.ExtractParams(topic.Name) - if err != nil { - log.Errorf("mqtt: failed to log message: %s", err) - } - for k, v := range params { - traits.With(k, v) - } + if topic.Name != topicName { + traits.With("instance", topic.Name) } err := s.eh.Push(&LogMessage{ Topic: topic.Name, MessageId: messageId, - Message: LogValue{ + Message: &LogValue{ Value: string(publish.Data), Binary: publish.Data, }, diff --git a/providers/asyncapi3/mqtt/store/publish_test.go b/providers/asyncapi3/mqtt/store/publish_test.go index 19ca9e628..752ae6cf5 100644 --- a/providers/asyncapi3/mqtt/store/publish_test.go +++ b/providers/asyncapi3/mqtt/store/publish_test.go @@ -358,8 +358,8 @@ func TestPublish(t *testing.T) { asyncapi3test.WithParameter( "sensorId", &asyncapi3.Parameter{}, ), - )), - ) + ), + )) publisher := newClient("publisher", s) defer publisher.close() @@ -395,11 +395,19 @@ func TestPublish(t *testing.T) { require.Equal(t, "sensors/1234z/data", d.Topic) require.Equal(t, "publisher", d.ClientId) require.Equal(t, "hello world", d.Message.Value) - require.Equal(t, "namespace=mqtt, name=test-server, clientId=publisher, sensorId=1234z, topic=sensors/{sensorId}/data, type=message", evts[0].Traits.String()) + require.Equal(t, "namespace=mqtt, name=test-server, clientId=publisher, instance=sensors/1234z/data, topic=sensors/{sensorId}/data, type=message", evts[0].Traits.String()) - require.Equal(t, float64(1), m.Messages.Sum(metrics.NewQuery())) - require.Equal(t, float64(1), m.Messages.WithLabel("test-server", "sensors/{sensorId}/data").Value()) - require.Greater(t, m.LastMessage.WithLabel("test-server", "sensors/{sensorId}/data").Value(), float64(1)) + require.Equal(t, 1.0, m.Messages.Sum( + metrics.NewQuery(metrics.ByLabel("topic", "sensors/{sensorId}/data"))), + ) + require.Equal(t, 0.0, m.Messages.WithLabel("test-server", "sensors/{sensorId}/data").Value()) + require.Greater(t, m.LastMessage.WithLabel("test-server", "sensors/{sensorId}/data", "sensors/1234z/data").Value(), float64(1)) + require.Equal(t, 1.0, m.Messages.Sum( + metrics.NewQuery( + metrics.ByLabel("topic", "sensors/{sensorId}/data"), + metrics.ByLabel("instance", "sensors/1234z/data"), + ), + )) }, }, } @@ -407,7 +415,6 @@ func TestPublish(t *testing.T) { logrus.SetOutput(io.Discard) for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { eh := &eventstest.Handler{} m := monitor.NewMqtt() diff --git a/providers/asyncapi3/mqtt/store/store.go b/providers/asyncapi3/mqtt/store/store.go index 5ab6386da..f377f9675 100644 --- a/providers/asyncapi3/mqtt/store/store.go +++ b/providers/asyncapi3/mqtt/store/store.go @@ -58,6 +58,10 @@ func (s *Store) Update(cfg *asyncapi3.Config) { if !ch.Value.IsChannelAvailable("mqtt") { continue } + if len(ch.Value.Parameters) > 0 { + continue + } + if s.Topics == nil { s.Topics = make(map[string]*Topic) } @@ -66,9 +70,8 @@ func (s *Store) Update(cfg *asyncapi3.Config) { address = ch.Value.Address } - if len(ch.Value.Parameters) == 0 { - s.Topics[address] = &Topic{Name: address, cfg: ch.Value} - } + s.Topics[address] = &Topic{Name: address, cfg: ch.Value} + } } diff --git a/providers/asyncapi3/websocket/channel.go b/providers/asyncapi3/websocket/channel.go index f9cb69900..448381c3f 100644 --- a/providers/asyncapi3/websocket/channel.go +++ b/providers/asyncapi3/websocket/channel.go @@ -105,13 +105,21 @@ func (c *Channel) readLoop(ctx context.Context, conn *websocket.Conn, client *Cl channelName := c.cfg.ResolveAddress() l := messageLog(c, data, messageId, client, Send) - // log event before run event engine to have the correct log order if event handler sends a message - c.log(l, events.NewTraits(). + traits := events.NewTraits(). With("channel", channelName). With("type", "message"). - With("clientId", client.Id)) + With("clientId", client.Id) + if c.Name != channelName { + traits = traits.With("instance", c.Name) + } + + // log event before run event engine to have the correct log order if event handler sends a message + c.log(l, traits) labels := []string{c.api, channelName} + if c.Name != channelName { + labels = append(labels, c.Name) + } c.monitor.Messages.WithLabel(labels...).Add(1) c.monitor.LastMessage.WithLabel(labels...).Set(float64(time.Now().Unix())) diff --git a/providers/asyncapi3/websocket/client.go b/providers/asyncapi3/websocket/client.go index 4c991835c..6e1a786c3 100644 --- a/providers/asyncapi3/websocket/client.go +++ b/providers/asyncapi3/websocket/client.go @@ -50,19 +50,27 @@ func (c *Client) sendMessage(message any) error { l := messageLog(c.channel, data, messageId, c, Receive) channelName := c.channel.cfg.ResolveAddress() - c.channel.log(l, events.NewTraits(). + traits := events.NewTraits(). With("channel", channelName). With("type", "message"). - With("clientId", c.Id)) + With("clientId", c.Id) + if c.channel.Name != channelName { + traits = traits.With("instance", c.channel.Name) + } + + c.channel.log(l, traits) labels := []string{c.channel.api, channelName} + if c.channel.Name != channelName { + labels = append(labels, c.channel.Name) + } c.channel.monitor.Messages.WithLabel(labels...).Add(1) c.channel.monitor.LastMessage.WithLabel(labels...).Set(float64(time.Now().Unix())) if err != nil { c.channel.monitor.MessagesError.WithLabel(labels...).Add(1) b, _ := json.Marshal(message) - l.Message = LogValue{ + l.Message = &LogValue{ Value: string(b), Binary: b, } diff --git a/providers/asyncapi3/websocket/log.go b/providers/asyncapi3/websocket/log.go index 1550b6be3..c96b57150 100644 --- a/providers/asyncapi3/websocket/log.go +++ b/providers/asyncapi3/websocket/log.go @@ -14,7 +14,7 @@ const ( type MessageLog struct { Channel string `json:"channel"` - Message LogValue `json:"message"` + Message *LogValue `json:"message"` MessageId string `json:"messageId"` Direction Direction `json:"direction"` Client ClientLog `json:"client"` @@ -24,6 +24,14 @@ type MessageLog struct { ValidationErrors map[string]string `json:"validationErrors,omitempty"` } +type MessageLogInfo struct { + Channel string `json:"channel"` + Message *LogValue `json:"message"` + Direction Direction `json:"direction"` + Client ClientLog `json:"client"` + ValidationErrors map[string]string `json:"validationErrors,omitempty"` +} + type ClientLog struct { Id string `json:"id"` Query map[string]any `json:"query"` @@ -41,10 +49,20 @@ func (l *MessageLog) Title() string { return "" } +func (l *MessageLog) Info() any { + return MessageLogInfo{ + Channel: l.Channel, + Message: l.Message, + Direction: l.Direction, + Client: l.Client, + ValidationErrors: l.ValidationErrors, + } +} + func messageLog(c *Channel, data []byte, messageId string, client *Client, direction Direction) *MessageLog { l := &MessageLog{ Channel: c.Name, - Message: LogValue{ + Message: &LogValue{ Value: string(data), Binary: data, }, @@ -87,6 +105,10 @@ func (l *ConnectLog) Title() string { return fmt.Sprintf("Connect to %s", l.Channel) } +func (l *ConnectLog) Info() any { + return l +} + type CloseLog struct { Type string `json:"type"` Channel string `json:"channel"` @@ -111,3 +133,7 @@ func closeLog(c *Channel, client *Client, reason, closedBy string) *CloseLog { func (l *CloseLog) Title() string { return fmt.Sprintf("Connect to %s", l.Channel) } + +func (l *CloseLog) Info() any { + return l +} diff --git a/providers/asyncapi3/websocket/store_test.go b/providers/asyncapi3/websocket/store_test.go index 7427bc2df..7d0d202f2 100644 --- a/providers/asyncapi3/websocket/store_test.go +++ b/providers/asyncapi3/websocket/store_test.go @@ -12,6 +12,7 @@ import ( "mokapi/providers/asyncapi3/websocket" "mokapi/runtime" "mokapi/runtime/events" + "mokapi/runtime/metrics" "mokapi/runtime/runtimetest" "mokapi/schema/json/schema/schematest" "mokapi/try" @@ -162,7 +163,7 @@ func TestStore(t *testing.T) { _, _, err = c.Read(ctx) require.ErrorContains(t, err, "invalid json format: invalid character 'h' looking for beginning of value") - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 2 }) @@ -208,7 +209,7 @@ func TestStore(t *testing.T) { _ = c.Close(ws.StatusNormalClosure, "bye") - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 2 }) @@ -288,7 +289,7 @@ export default function() { err = c.Write(ctx, ws.MessageText, []byte(`{"text": "hello"}`)) require.NoError(t, err) - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 2 }) @@ -414,7 +415,7 @@ export default function() { err = c.Write(ctx, ws.MessageText, []byte("hello")) require.NoError(t, err) - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 1 }) @@ -510,7 +511,7 @@ export default function() { err = c.Write(ctx, ws.MessageText, []byte("hello")) require.NoError(t, err) - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 1 }) @@ -602,12 +603,24 @@ export default function() { err = c.Write(ctx, ws.MessageText, []byte("hello")) require.NoError(t, err) - waitFor(t, func() bool { + try.WaitFor(t, func() bool { evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) return len(evts) > 1 }) + require.Equal(t, 1.0, app.Monitor.Websocket.Messages.Sum( + metrics.NewQuery(metrics.ByLabel("channel", "/chats/{chatId}")), + )) + require.Equal(t, 1.0, app.Monitor.Websocket.Messages.Sum( + metrics.NewQuery( + metrics.ByLabel("channel", "/chats/{chatId}"), + metrics.ByLabel("instance", "/chats/1234"), + ), + )) + evts := app.Events.GetEvents(events.NewTraits().WithNamespace("websocket")) + require.Equal(t, "/chats/{chatId}", evts[0].Traits.Get("channel")) + require.Equal(t, "/chats/1234", evts[0].Traits.Get("instance")) require.Len(t, evts, 2) evt := evts[0] d := evt.Data.(*websocket.MessageLog) @@ -654,18 +667,3 @@ export default function() { }) } } - -func waitFor(t *testing.T, check func() bool) { - t.Helper() - deadline := time.Now().Add(30 * time.Second) - - for { - if check() { - break - } - if time.Now().After(deadline) { - t.Fatal("waitFor deadline") - } - time.Sleep(20 * time.Millisecond) - } -} diff --git a/providers/directory/log.go b/providers/directory/log.go index 3e1ad1293..4456728bb 100644 --- a/providers/directory/log.go +++ b/providers/directory/log.go @@ -5,8 +5,17 @@ import ( "mokapi/engine/common" "mokapi/ldap" "mokapi/runtime/events" + "strings" ) +type LogInfo struct { + Operation string `json:"operation"` + Dn string `json:"dn,omitempty"` + Criteria string `json:"criteria,omitempty"` + Status string `json:"status,omitempty"` + Duration int64 `json:"duration"` +} + type SearchLog struct { Request *SearchRequest `json:"request"` Response *SearchResponse `json:"response"` @@ -15,16 +24,16 @@ type SearchLog struct { } type SearchRequest struct { - Operation string `json:"operation"` - BaseDN string `json:"baseDN"` - Scope string `json:"scope"` - DereferencePolicy int64 `json:"dereferencePolicy"` - SizeLimit int64 `json:"sizeLimit"` - TimeLimit int64 `json:"timeLimit"` - TypesOnly bool `json:"typesOnly"` - Filter string `json:"filter"` - Attributes []string `json:"attributes"` - Controls []interface{} `json:"controls"` + Operation string `json:"operation"` + BaseDN string `json:"baseDN"` + Scope string `json:"scope"` + DereferencePolicy int64 `json:"dereferencePolicy"` + SizeLimit int64 `json:"sizeLimit"` + TimeLimit int64 `json:"timeLimit"` + TypesOnly bool `json:"typesOnly"` + Filter string `json:"filter"` + Attributes []string `json:"attributes"` + Controls []any `json:"controls"` } type SearchResponse struct { @@ -321,14 +330,40 @@ func (l *BindLog) Title() string { return fmt.Sprintf("%s %s %s", l.Request.Auth, l.Request.Name, l.Request.Password) } +func (l *BindLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Name, + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *UnbindLog) Title() string { return "" } +func (l *UnbindLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Duration: l.Duration, + } +} + func (l *SearchLog) Title() string { return l.Request.Filter } +func (l *SearchLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.BaseDN, + Criteria: l.Request.Filter, + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *SearchLog) IndexFields() map[string]any { m := map[string]any{ "operation": l.Request.Operation, @@ -343,18 +378,78 @@ func (l *CompareLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Value) } +func (l *CompareLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Dn, + Criteria: fmt.Sprintf("%s == %s", l.Request.Attribute, l.Request.Value), + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *ModifyLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Dn) } +type ModifyInfo struct { + Operation string `json:"operation"` + Dn string `json:"dn,omitempty"` + Criteria []string `json:"criteria,omitempty"` + Status string `json:"status,omitempty"` + Duration int64 `json:"duration"` +} + +func (l *ModifyLog) Info() any { + criteria := make([]string, 0, len(l.Request.Items)) + for _, i := range l.Request.Items { + criteria = append(criteria, fmt.Sprintf("%s %s", strings.ToLower(i.Modification), i.Attribute.Type)) + } + return ModifyInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Dn, + Criteria: criteria, + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *ModifyDNLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Dn) } +func (l *ModifyDNLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Dn, + Criteria: l.Request.NewRdn, + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *DeleteLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Dn) } +func (l *DeleteLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Dn, + Status: l.Response.Status, + Duration: l.Duration, + } +} + func (l *AddLog) Title() string { return fmt.Sprintf("%s %s", l.Request.Operation, l.Request.Dn) } + +func (l *AddLog) Info() any { + return LogInfo{ + Operation: l.Request.Operation, + Dn: l.Request.Dn, + Status: l.Response.Status, + Duration: l.Duration, + } +} diff --git a/providers/mail/log.go b/providers/mail/log.go index 8a8fa07a3..67b912e04 100644 --- a/providers/mail/log.go +++ b/providers/mail/log.go @@ -19,6 +19,14 @@ type Log struct { Actions []*common.Action `json:"actions"` } +type LogInfo struct { + MessageId string `json:"messageId"` + From string `json:"from"` + To []string `json:"to"` + Subject string `json:"subject"` + Duration int64 `json:"duration"` +} + func NewLogEvent(msg *smtp.Message, ctx *smtp.ClientContext, eh events.Handler, traits events.Traits) *Log { event := &Log{ From: ctx.From, @@ -46,6 +54,16 @@ func (l *Log) Title() string { return fmt.Sprintf("%s", l.Subject) } +func (l *Log) Info() any { + return LogInfo{ + MessageId: l.MessageId, + From: l.From, + To: l.To, + Subject: l.Subject, + Duration: l.Duration, + } +} + func (l *Log) IndexFields() map[string]any { m := map[string]any{ "from": l.From, diff --git a/providers/openapi/handler_test.go b/providers/openapi/handler_test.go index 2d49d91ce..c20b4567f 100644 --- a/providers/openapi/handler_test.go +++ b/providers/openapi/handler_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "maps" "mokapi/config/dynamic" "mokapi/config/dynamic/dynamictest" "mokapi/engine/common" @@ -785,9 +786,7 @@ func TestResolveEndpoint(t *testing.T) { h := openapi.NewHandler(config, f, e) err := h.ServeHTTP(rw, r) if err != nil { - for k, v := range err.Header { - rw.Header()[k] = v - } + maps.Copy(rw.Header(), err.Header) http.Error(rw, err.Message, err.StatusCode) } }, config) @@ -1199,9 +1198,7 @@ func TestHandler_Event(t *testing.T) { r = r.WithContext(ctx) httpErr := h.ServeHTTP(rw, r) if httpErr != nil { - for k, v := range httpErr.Header { - rw.Header()[k] = v - } + maps.Copy(rw.Header(), httpErr.Header) http.Error(rw, httpErr.Message, httpErr.StatusCode) } }, config, sm) @@ -1239,7 +1236,6 @@ func TestHandler_Log(t *testing.T) { } for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { test.NewNullLogger() @@ -1259,9 +1255,7 @@ func TestHandler_Log(t *testing.T) { h := openapi.NewHandler(config, f, m) err := h.ServeHTTP(rw, r) if err != nil { - for k, v := range err.Header { - rw.Header()[k] = v - } + maps.Copy(rw.Header(), err.Header) http.Error(rw, err.Message, err.StatusCode) } }, config, m) @@ -1474,8 +1468,8 @@ func TestHandler_Event_TypeScript(t *testing.T) { require.Contains(t, rr.Body.String(), `{"foo":`) result := sm.GetEvents(events.NewTraits().WithNamespace("http")) - log := result[0].Data.(*openapi.HttpLog) - require.Equal(t, "no configuration was found for HTTP status code 404, https://swagger.io/docs/specification/describing-responses", log.Actions[0].Error.Message) + hl := result[0].Data.(*openapi.HttpLog) + require.Equal(t, "no configuration was found for HTTP status code 404, https://swagger.io/docs/specification/describing-responses", hl.Actions[0].Error.Message) }, }, { @@ -1592,14 +1586,13 @@ func TestHandler_Event_TypeScript(t *testing.T) { require.Contains(t, rr.Body.String(), `{"foo":`) result := sm.GetEvents(events.NewTraits().WithNamespace("http")) - log := result[0].Data.(*openapi.HttpLog) - require.Equal(t, "content type 'text/plain' is not specified for HTTP status code 404", log.Actions[0].Error.Message) + hl := result[0].Data.(*openapi.HttpLog) + require.Equal(t, "content type 'text/plain' is not specified for HTTP status code 404", hl.Actions[0].Error.Message) }, }, } for _, tc := range testcases { - tc := tc t.Run(tc.name, func(t *testing.T) { tc.test(t) }) @@ -1662,10 +1655,10 @@ func TestHandler_Parameter(t *testing.T) { } type testEngine struct { - emit func(event string, args ...interface{}) []*common.Action + emit func(event string, args ...any) []*common.Action } -func (e *testEngine) Emit(event string, args ...interface{}) []*common.Action { +func (e *testEngine) Emit(event string, args ...any) []*common.Action { if e.emit != nil { return e.emit(event, args...) } diff --git a/providers/openapi/log.go b/providers/openapi/log.go index 388a1c6ec..ee93412ad 100644 --- a/providers/openapi/log.go +++ b/providers/openapi/log.go @@ -24,6 +24,13 @@ type HttpLog struct { ClientIP string `json:"clientIP"` } +type HttpLogInfo struct { + Url string `json:"url"` + StatusCode int `json:"statusCode"` + Duration int64 `json:"duration"` + Deprecated bool `json:"deprecated"` +} + type HttpRequestLog struct { Method string `json:"method"` Url string `json:"url"` @@ -112,6 +119,15 @@ func (l *HttpLog) Title() string { return l.Request.Url } +func (l *HttpLog) Info() any { + return HttpLogInfo{ + Url: l.Request.Url, + StatusCode: l.Response.StatusCode, + Duration: l.Duration, + Deprecated: l.Deprecated, + } +} + func (l *HttpLog) IndexFields() map[string]any { m := map[string]any{ "request": l.Request, diff --git a/providers/openapi/parameter_query.go b/providers/openapi/parameter_query.go index 680a59b5a..0a78b1c3a 100644 --- a/providers/openapi/parameter_query.go +++ b/providers/openapi/parameter_query.go @@ -46,7 +46,7 @@ func parseQuery(param *Parameter, u *url.URL) (*RequestParameterValue, error) { } } -func parseQueryObject(param *Parameter, u *url.URL) (string, interface{}, error) { +func parseQueryObject(param *Parameter, u *url.URL) (string, any, error) { if param.Style == "form" && param.IsExplode() { raw := u.RawQuery if len(raw) == 0 { @@ -69,7 +69,7 @@ func parseQueryObject(param *Parameter, u *url.URL) (string, interface{}, error) return raw, i, err } else if param.Style == "deepObject" && param.IsExplode() { paramRegex := regexp.MustCompile(fmt.Sprintf(`^%s\[(?P.+)\]$`, regexp.QuoteMeta(param.Name))) - obj := map[string]interface{}{} + obj := map[string]any{} raw := strings.Builder{} for k, values := range u.Query() { @@ -82,9 +82,18 @@ func parseQueryObject(param *Parameter, u *url.URL) (string, interface{}, error) if prop == nil && !param.Schema.IsFreeForm() && !param.Schema.IsDictionary() { return "", nil, fmt.Errorf("property '%v' not defined in schema: %s", name, param.Schema) } - s := strings.Join(values, ",") - raw.WriteString(fmt.Sprintf("%v[%v]=%v", param.Name, name, s)) - if v, err := p.ParseWith(s, schema.ConvertToJsonSchema(prop)); err != nil { + + var data any + if prop != nil && prop.Type.IsArray() { + data = values + } else if len(values) == 1 { + data = values[0] + } else { + data = values + } + + raw.WriteString(fmt.Sprintf("%v[%v]=%v", param.Name, name, strings.Join(values, ","))) + if v, err := p.ParseWith(data, schema.ConvertToJsonSchema(prop)); err != nil { return "", nil, err } else { obj[name] = v diff --git a/providers/openapi/parameter_query_test.go b/providers/openapi/parameter_query_test.go index 3c3f4d624..57b0a2d23 100644 --- a/providers/openapi/parameter_query_test.go +++ b/providers/openapi/parameter_query_test.go @@ -649,6 +649,38 @@ func TestParseQueryDeepObjectOtherParameters(t *testing.T) { query: url.Values{"filter[unknown]": {"guest"}}, wantErr: "parse query parameter 'filter' failed: property 'unknown' not defined in schema: schema type=object properties=[role] free-form=false", }, + { + name: "property is array using multiple values", + params: openapi.Parameters{{Value: &openapi.Parameter{ + Name: "filter", + Type: openapi.ParameterQuery, + Schema: schematest.New("object", + schematest.WithProperty("role", schematest.New("array", schematest.WithItems("string"))), + ), + Style: "deepObject", + Explode: new(true), + }}}, + query: url.Values{ + "filter[role]": {"admin", "user"}, + }, + want: map[string]any{"filter": map[string]any{"role": []any{"admin", "user"}}}, + }, + { + name: "property is array using comma separated values", + params: openapi.Parameters{{Value: &openapi.Parameter{ + Name: "filter", + Type: openapi.ParameterQuery, + Schema: schematest.New("object", + schematest.WithProperty("role", schematest.New("array", schematest.WithItems("string"))), + ), + Style: "deepObject", + Explode: new(true), + }}}, + query: url.Values{ + "filter[role]": {"admin,user"}, + }, + want: map[string]any{"filter": map[string]any{"role": []any{"admin,user"}}}, + }, } for _, tc := range testcases { diff --git a/runtime/events/events.go b/runtime/events/events.go index af3b5cd3f..9fad52309 100644 --- a/runtime/events/events.go +++ b/runtime/events/events.go @@ -18,6 +18,7 @@ type Handler interface { type EventData interface { Title() string + Info() any } type StoreManager struct { diff --git a/runtime/events/eventstest/event.go b/runtime/events/eventstest/event.go index 0f8f9faba..90f7ecc54 100644 --- a/runtime/events/eventstest/event.go +++ b/runtime/events/eventstest/event.go @@ -3,7 +3,7 @@ package eventstest import "mokapi/runtime/events" type Event struct { - Name string + Name string `json:"name"` Api string `json:"api"` } @@ -11,6 +11,10 @@ func (e *Event) Title() string { return e.Name } +func (e *Event) Info() any { + return e +} + func (e *Event) IndexFields() map[string]any { return map[string]any{ "name": e.Name, diff --git a/runtime/index.go b/runtime/index.go index d02116281..e90c60bf8 100644 --- a/runtime/index.go +++ b/runtime/index.go @@ -175,6 +175,9 @@ func (s *SearchIndex) runWorker(ctx context.Context) { if batchSize == 0 { return } + if s.idx == nil { + return + } if err := s.idx.Batch(batch); err != nil { println("batch failed:", err.Error()) diff --git a/runtime/logs.go b/runtime/logs.go index fe021daca..a6ebc8c14 100644 --- a/runtime/logs.go +++ b/runtime/logs.go @@ -56,6 +56,10 @@ func (d *LogData) Title() string { return fmt.Sprintf("%v: %v", d.Level, msg) } +func (d *LogData) Info() any { + return d +} + func extractTraits(entry *log.Entry) events.Traits { traits := events.NewTraits() if ns, ok := entry.Data["namespace"].(string); ok { diff --git a/runtime/metrics/counter.go b/runtime/metrics/counter.go index e1dd692fe..4f3e1d04c 100644 --- a/runtime/metrics/counter.go +++ b/runtime/metrics/counter.go @@ -17,7 +17,7 @@ func NewCounter(opt ...Options) *Counter { for _, opt := range opt { opt(o) } - return &Counter{info: &Info{Namespace: o.namespace, Name: o.name, labels: o.labels}} + return &Counter{info: &Info{Namespace: o.namespace, Name: o.name}} } func (c *Counter) Add(v float64) { @@ -73,13 +73,20 @@ func NewCounterMap(opts ...Options) *CounterMap { info: info, counters: make(map[uint32]*Counter), newCounter: func(values []string) *Counter { - if len(o.labelNames) != len(values) { + if !o.areLabelsValid(values) { panic(fmt.Sprintf("invalid labels values for %v", info)) } labels := make([]*Label, 0) for i, v := range values { + var name string + if i < len(o.labelNames) { + name = o.labelNames[i] + } else { + name = o.optionalNames[i-len(o.labelNames)] + } + labels = append(labels, &Label{ - Name: o.labelNames[i], + Name: name, Value: v, }) } diff --git a/runtime/metrics/gauge.go b/runtime/metrics/gauge.go index 9d28fe1a4..3ec839914 100644 --- a/runtime/metrics/gauge.go +++ b/runtime/metrics/gauge.go @@ -17,7 +17,7 @@ func NewGauge(opt ...Options) *Gauge { opt(o) } return &Gauge{ - info: &Info{Namespace: o.namespace, Name: o.name, labels: o.labels}, + info: &Info{Namespace: o.namespace, Name: o.name}, } } @@ -67,13 +67,20 @@ func NewGaugeMap(opts ...Options) *GaugeMap { info: info, gauges: make(map[uint32]*Gauge), newGauge: func(values []string) *Gauge { - if len(o.labelNames) != len(values) { + if !o.areLabelsValid(values) { panic(fmt.Sprintf("invalid labels values for %v", info.String())) } labels := make([]*Label, 0) for i, v := range values { + var name string + if i < len(o.labelNames) { + name = o.labelNames[i] + } else { + name = o.optionalNames[i-len(o.labelNames)] + } + labels = append(labels, &Label{ - Name: o.labelNames[i], + Name: name, Value: v, }) } diff --git a/runtime/metrics/metrics.go b/runtime/metrics/metrics.go index a899e8c80..9e997b668 100644 --- a/runtime/metrics/metrics.go +++ b/runtime/metrics/metrics.go @@ -79,10 +79,10 @@ func (i *Info) Match(query *Query) bool { type Options func(o *options) type options struct { - namespace string - name string - labels []*Label - labelNames []string + namespace string + name string + labelNames []string + optionalNames []string } func WithName(name string) Options { @@ -104,6 +104,21 @@ func WithFQName(namespace, name string) Options { } } +func WithOptionalNames(names ...string) Options { + return func(o *options) { + o.optionalNames = names + } +} + +func (o options) areLabelsValid(values []string) bool { + minLabels := len(o.labelNames) + maxLabels := len(o.labelNames) + len(o.optionalNames) + if len(values) < minLabels || len(values) > maxLabels { + return false + } + return true +} + func hash(values []string) uint32 { s := strings.Join(values, "_") diff --git a/runtime/metrics/metrics_test.go b/runtime/metrics/metrics_test.go index c717e7c03..aeb9d2057 100644 --- a/runtime/metrics/metrics_test.go +++ b/runtime/metrics/metrics_test.go @@ -60,6 +60,26 @@ func TestNewCounterMap(t *testing.T) { require.Equal(t, 0.0, c.Value(&Query{Labels: []*Label{{Name: "foo", Value: "foo"}}})) }, }, + { + name: "missing label", + f: func(t *testing.T) { + c := NewCounterMap(WithName("foo"), WithLabelNames("foo")) + require.PanicsWithValue(t, "invalid labels values for foo", func() { + c.WithLabel().Add(1) + }) + }, + }, + { + name: "using optional label", + f: func(t *testing.T) { + c := NewCounterMap(WithName("foo"), WithLabelNames("foo"), WithOptionalNames("bar")) + c.WithLabel("foo1").Add(1) + require.Equal(t, 1.0, c.Value(&Query{Labels: []*Label{{Name: "foo", Value: "foo1"}}})) + c.WithLabel("foo2", "bar").Add(1) + require.Equal(t, 1.0, c.Value(&Query{Labels: []*Label{{Name: "foo", Value: "foo1"}}})) + require.Equal(t, 1.0, c.Value(&Query{Labels: []*Label{{Name: "foo", Value: "foo2"}, {Name: "bar", Value: "bar"}}})) + }, + }, { name: "json", f: func(t *testing.T) { diff --git a/runtime/monitor/kafka.go b/runtime/monitor/kafka.go index b1ad40bd7..40774fe03 100644 --- a/runtime/monitor/kafka.go +++ b/runtime/monitor/kafka.go @@ -18,17 +18,23 @@ type Kafka struct { func NewKafka() *Kafka { messages := metrics.NewCounterMap( metrics.WithFQName("kafka", "messages_total"), - metrics.WithLabelNames("service", "topic")) + metrics.WithLabelNames("service", "topic"), + metrics.WithOptionalNames("instance"), + ) lastMessage := metrics.NewGaugeMap( metrics.WithFQName("kafka", "message_timestamp"), - metrics.WithLabelNames("service", "topic")) + metrics.WithLabelNames("service", "topic"), + metrics.WithOptionalNames("instance"), + ) lag := metrics.NewGaugeMap( metrics.WithFQName("kafka", "consumer_group_lag"), - metrics.WithLabelNames("service", "group", "topic", "partition")) + metrics.WithLabelNames("service", "group", "topic", "partition"), + ) commits := metrics.NewGaugeMap( metrics.WithFQName("kafka", "consumer_group_commit"), - metrics.WithLabelNames("service", "group", "topic", "partition")) + metrics.WithLabelNames("service", "group", "topic", "partition"), + ) lastRebalancing := metrics.NewGaugeMap( metrics.WithFQName("kafka", "rebalance_timestamp"), diff --git a/runtime/monitor/mqtt.go b/runtime/monitor/mqtt.go index da7c38cdc..a480b39dd 100644 --- a/runtime/monitor/mqtt.go +++ b/runtime/monitor/mqtt.go @@ -15,10 +15,14 @@ type Mqtt struct { func NewMqtt() *Mqtt { messages := metrics.NewCounterMap( metrics.WithFQName("mqtt", "messages_total"), - metrics.WithLabelNames("service", "topic")) + metrics.WithLabelNames("service", "topic"), + metrics.WithOptionalNames("instance"), + ) lastMessage := metrics.NewGaugeMap( metrics.WithFQName("mqtt", "message_timestamp"), - metrics.WithLabelNames("service", "topic")) + metrics.WithLabelNames("service", "topic"), + metrics.WithOptionalNames("instance"), + ) return &Mqtt{ Messages: messages, diff --git a/runtime/monitor/websocket.go b/runtime/monitor/websocket.go index 7c2b7706d..6e8ce3cfe 100644 --- a/runtime/monitor/websocket.go +++ b/runtime/monitor/websocket.go @@ -16,13 +16,19 @@ type Websocket struct { func NewWebsocket() *Websocket { messages := metrics.NewCounterMap( metrics.WithFQName("websocket", "messages_total"), - metrics.WithLabelNames("service", "channel")) + metrics.WithLabelNames("service", "channel"), + metrics.WithOptionalNames("instance"), + ) messagesError := metrics.NewCounterMap( metrics.WithFQName("websocket", "messages_error_total"), - metrics.WithLabelNames("service", "channel")) + metrics.WithLabelNames("service", "channel"), + metrics.WithOptionalNames("instance"), + ) lastMessage := metrics.NewGaugeMap( metrics.WithFQName("websocket", "message_timestamp"), - metrics.WithLabelNames("service", "channel")) + metrics.WithLabelNames("service", "channel"), + metrics.WithOptionalNames("instance"), + ) return &Websocket{ Messages: messages, diff --git a/runtime/runtime_websocket.go b/runtime/runtime_websocket.go index 1959eb052..af51dfff6 100644 --- a/runtime/runtime_websocket.go +++ b/runtime/runtime_websocket.go @@ -275,8 +275,8 @@ func (s *WebsocketStore) updateEventStore(k *WebsocketInfo) { if _, ok := k.seenTopics[channelName]; ok { continue } - s.monitor.Mqtt.Messages.WithLabel(k.Config.Info.Name, channelName) - s.monitor.Mqtt.LastMessage.WithLabel(k.Config.Info.Name, channelName) + s.monitor.Websocket.Messages.WithLabel(k.Config.Info.Name, channelName) + s.monitor.Websocket.LastMessage.WithLabel(k.Config.Info.Name, channelName) traits := events.NewTraits().WithNamespace("websocket").WithName(k.Config.Info.Name).With("channel", channelName) s.events.SetStore(int(eventStore.Size), traits) k.seenTopics[channelName] = true diff --git a/schema/json/generator/request.go b/schema/json/generator/request.go index 1adedfee9..bb9866c9b 100644 --- a/schema/json/generator/request.go +++ b/schema/json/generator/request.go @@ -1,6 +1,7 @@ package generator import ( + "maps" "mokapi/schema/json/schema" ) @@ -59,8 +60,8 @@ func (r *Request) With(path []string, s *schema.Schema, example []any) *Request } type Context struct { - Values Values `json:"values"` - snapshots []Values + Values map[string]any `json:"values"` + snapshots []map[string]any // represents how many steps current context is away from the root level int } @@ -73,7 +74,10 @@ func newContext() *Context { func (c *Context) Snapshot() { if c.level != 0 { - c.snapshots = append(c.snapshots, c.Values.Snapshot()) + m := map[string]any{} + maps.Copy(m, c.Values) + c.Values = map[string]any{} + c.snapshots = append(c.snapshots, m) } c.level++ } @@ -96,14 +100,3 @@ func (c *Context) Has(key string) bool { func (c *Context) Value(key string) any { return c.Values[key] } - -func (s *Values) Snapshot() Values { - snapshot := map[string]any{} - for k, v := range *s { - if v != nil { - snapshot[k] = v - } - } - *s = map[string]any{} - return snapshot -} diff --git a/try/time.go b/try/time.go new file mode 100644 index 000000000..e478ae084 --- /dev/null +++ b/try/time.go @@ -0,0 +1,29 @@ +package try + +import ( + "testing" + "time" +) + +func MustTime(s string) time.Time { + t, err := time.Parse(time.RFC3339, s) + if err != nil { + panic(err) + } + return t +} + +func WaitFor(t *testing.T, check func() bool) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + + for { + if check() { + break + } + if time.Now().After(deadline) { + t.Fatal("waitFor deadline") + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/webui/build-sitemap.js b/webui/build-sitemap.js index a61c47b77..86558b427 100644 --- a/webui/build-sitemap.js +++ b/webui/build-sitemap.js @@ -1,6 +1,6 @@ -const fs = require('fs'); -const path = require('path'); -const util = require('util'); +import fs from 'node:fs'; +import path from 'node:path'; +import util from 'node:util'; const pages = [ { @@ -48,7 +48,7 @@ function writeItem(item) { let xml = ''; if (item.path && item.source) { - stats = fs.statSync(path.join(docsPath, item.source)); + const stats = fs.statSync(path.join(docsPath, item.source)); const url = `https://mokapi.io${item.path}`; const node = util.format(urlTemplate, url, stats.mtime.toISOString()) xml += node + '\n' @@ -84,7 +84,7 @@ try { // write docs const data = fs.readFileSync(path.join(docsPath, 'config.json'), 'utf8'); - docs = JSON.parse(data) + const docs = JSON.parse(data) content += writeObject(docs) // trim leading and trailing whitespace and new lines diff --git a/webui/e2e/mocks/common.yml b/webui/e2e/mocks/common.yml index e6fd78fae..085a411f9 100644 --- a/webui/e2e/mocks/common.yml +++ b/webui/e2e/mocks/common.yml @@ -53,64 +53,6 @@ components: type: string value: type: number - Events: - type: array - items: - $ref: '#/components/schemas/Event' - Event: - type: object - properties: - id: - type: string - traits: - type: object - additionalProperties: - type: string - time: - type: string - data: - anyOf: - - $ref: '#/components/schemas/KafkaEvent' - - description: any - type: object - additionalProperties: true - KafkaEvent: - type: object - properties: - offset: - type: integer - key: - $ref: '#/components/schemas/KafkaValue' - message: - $ref: '#/components/schemas/KafkaValue' - schemaId: - type: integer - partition: - type: integer - headers: - type: object - additionalProperties: - type: object - properties: - value: - type: string - binary: - type: string - deleted: - type: boolean - producerId: - type: number - producerEpoch: - type: number - sequenceNumber: - type: number - KafkaValue: - type: object - properties: - value: - type: string - binary: - type: string Error: type: object properties: diff --git a/webui/e2e/mocks/dashboard.ts b/webui/e2e/mocks/dashboard.ts new file mode 100644 index 000000000..a9723377d --- /dev/null +++ b/webui/e2e/mocks/dashboard.ts @@ -0,0 +1,335 @@ +import { app, sleep } from 'mokapi' +import { get, post, fetch } from 'mokapi/http' +import { clusters as kafkaClusters, events as kafkaEvents, configs as kafkaConfigs } from './kafka' +import { apps as httpServices, events as httpEvents, configs as httpConfigs } from './http' +import { services as mailServices, mailEvents, getMail, getAttachment } from './mail' +import { servers as ldapServers, searches } from './ldap' +import { clusters as mqttClusters, events as mqttEvents } from './mqtt' +import { services as websocketServices, events as websocketEvents } from './websocket' +import { sum, max } from './metrics' +import { getInfo, matchTraits } from './helpers' +import { jobs, config as jobConfig } from './jobs' +import { match, type Query } from '../../lib/events' + +export default async function() { + const apiBaseUrl = `http://localhost:8091/mokapi` + const { version, buildTime } = get(`${apiBaseUrl}/api/info`).json() + + app.http() + .use((_, res) => { + res.headers["Access-Control-Allow-Methods"] = "*" + res.headers["Access-Control-Allow-Headers"] = "*" + res.headers["Access-Control-Allow-Origin"] = "*" + res.headers['Mokapi-Version'] = version + res.headers['Mokapi-Build-Time'] = buildTime + }) + .operation('info', (_, res) => { + res.data = { + version: "0.11.0", + activeServices: ["http", "kafka", "ldap", "mail", "mqtt", "websocket"], + search: { enabled: true } + } + }) + .operation('services', (req, res) => { + res.data = getServices(req.query['type']) + }) + .operation('events', (req, res) => { + const page = req.query['page'] ? parseInt(req.query['page']) : 1 + const size = req.query['size'] ? parseInt(req.query['size']) : 25 + + const items = events.filter(x => match(x, query(req.query))) + res.data = { + items: items.map(eventInfo).slice((page - 1) * size, page * size), + page: page, + size: size, + total: items.length + } + }) + .operation('event', (req, res) => { + const event = events.find(x => x.id === req.path['id']) + if (!event){ + res.rebuild(404) + return + } + res.data = event + }) + .operation('example', (req, res) => { + const resp = post(`${apiBaseUrl}/api/schema/example`, req.body) + res.statusCode = resp.statusCode + res.headers = resp.headers + res.body = resp.body + }) + .operation('validate', (req, res) => { + sleep(1000) + let url = `${apiBaseUrl}/api/schema/validate` + if (req.query['outputFormat']) { + url += '?outputFormat=' + req.query['outputFormat'] + } + const resp = post(url, req.body) + res.statusCode = resp.statusCode + res.headers = resp.headers + res.body = resp.body + res.data = null + }) + .operation('configs', (req, res) => { + res.data = configs + }) + .operation('config', (req, res) => { + const config = configs.find(x => x.id === req.path.id) + if (!config) { + res.rebuild(404) + return + } + res.data = config + }) + .operation('configData', (req, res) => { + const config = configData[req.path.id] + if (!config) { + res.rebuild(404) + return + } + res.data = config.data + res.headers = config.headers + }) + + for (const config of configs) { + if (!config.data || !config.data.startsWith('http://localhost:8090')) { + continue + } + const r = await fetch(config.data) + configData[config.id] = { + data: r.body, + headers: { + 'Content-Disposition': `inline; filename="${config.filename}"` + } + } + } +} + +const configs = [ + ...Object.values(httpConfigs), + ...Object.values(kafkaConfigs), + ...Object.values(jobConfig) +] + +const configData: Record }> = {} + + +function getServices(type: string) { + const services = [] + if (!type || type === 'http') { + services.push(...httpServices.map(x => { + return { + ...getInfo(x, 'http'), + metrics: { + requests_total: sum('http_requests_total', { name: 'service', value: x.name }), + requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: x.name }), + request_timestamp: max('http_request_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'kafka') { + services.push(...kafkaClusters.map(x => { + return { + ...getInfo(x, 'kafka'), + metrics: { + messages_total: sum('kafka_messages_total', { name: 'service', value: x.name }), + message_timestamp: max('kafka_message_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'mail') { + services.push(...mailServices.map(x => { + return { + ...getInfo(x, 'mail'), + metrics: { + mails_total: sum('mail_mails_total', { name: 'service', value: x.name }), + mail_timestamp: max('mail_mail_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'ldap') { + services.push(...ldapServers.map(x => { + return { + ...getInfo(x, 'ldap'), + metrics: { + requests_total: sum('ldap_requests_total', { name: 'service', value: x.name }), + request_timestamp: max('ldap_request_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'mqtt') { + services.push(...mqttClusters.map(x => { + return { + ...getInfo(x, 'mqtt'), + metrics: { + messages_total: sum('mqtt_messages_total', { name: 'service', value: x.name }), + message_timestamp: max('mqtt_message_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + if (!type || type === 'websocket') { + services.push(...websocketServices.map(x => { + return { + ...getInfo(x, 'websocket'), + metrics: { + messages_total: sum('websocket_messages_total', { name: 'service', value: x.name }), + message_timestamp: max('websocket_message_timestamp', { name: 'service', value: x.name }) + } + } + })) + } + + return services +} + +const events = [ + ...httpEvents, + ...kafkaEvents, + ...mqttEvents, + ...websocketEvents, + ...mailEvents, + ...searches, + ...jobs +] + +function eventInfo(event: ServiceEvent) { + const {data, ...info} = event + switch (event.traits.namespace) { + case 'http': + return { + ...info, + data: { + url: event.data.request.url, + statusCode: event.data.response.statusCode, + duration: event.data.duration, + deprecated: event.data.deprecated + } + } + case 'kafka': + switch (event.traits.type) { + case 'message': + return { + ...info, + data: { + topic: event.data.topic, + key: event.data.key, + message: event.data.message, + deleted: event.data.deleted + } + } + case 'request': + return event + } + case 'mqtt': + switch (event.traits.type) { + case 'message': + return { + ...info, + data: { + topic: event.data.topic, + message: event.data.message + } + } + case 'request': + return event + } + case 'websocket': + return event + case 'ldap': + return { + ...info, + data: { + operation: event.data.request.operation, + dn: dn(event), + criteria: criteria(event), + status: event.data.response.status ?? '-', + duration: event.data.duration, + } + } + case 'mail': + return { + ...info, + data: { + messageId: event.data.messageId, + from: event.data.from, + to: event.data.to, + subject: event.data.subject, + duration: event.data.duration + } + } + case 'job': + return event + } +} + +function dn(e: ServiceEvent) { + switch (e.data.request.operation) { + case 'Bind': + return e.data.request.name; + case 'Search': + return e.data.request.baseDN + case 'Modify': + case 'Add': + case 'Delete': + case 'ModifyDN': + case 'Compare': + return e.data.request.dn + } +} + +function criteria(e: ServiceEvent) { + switch (e.data.request.operation) { + case 'Search': + return e.data.request.filter + case 'Compare': + return `${e.data.request.attribute} == ${e.data.request.value}` + case 'Modify': + const result = [] + for (const item of e.data.request.items) { + result.push(`${item.modification.toLocaleLowerCase()} ${item.attribute.type}`) + } + return result + case 'ModifyDN': + return e.data.request.newRdn + default: + undefined; + } +} + +function query(q: Record): Query { + const result: Query = { labels: Object.keys(q.traits).map(x => ({ name: x, value: q.traits[x] })) } + + if (q.http) { + result.http = { + url: q.http.url, + statusCode: q.http.statusCode, + clientIP: q.http.clientIP + } + + if (q.http.method) { + result.http.methods = q.http.method.split(',') + } + if (q.http.requestHeader) { + result.http.requestHeaders = [] + for (const h of q.http.requestHeader) { + const kv = h.split(':', 2) + result.http.requestHeaders.push({ name: kv[0], value: kv.length == 1 ? '' : kv[1] }) + } + } + if (q.http.responseHeader) { + result.http.responseHeaders = [] + for (const h of q.http.responseHeader) { + const kv = h.split(':', 2) + result.http.responseHeaders.push({ name: kv[0], value: kv.length == 1 ? '' : kv[1] }) + } + } + } + + return result +} \ No newline at end of file diff --git a/webui/e2e/mocks/dashboard.yml b/webui/e2e/mocks/dashboard.yml index eec8c6a69..1ee852bfe 100644 --- a/webui/e2e/mocks/dashboard.yml +++ b/webui/e2e/mocks/dashboard.yml @@ -280,7 +280,7 @@ paths: /api/services/mail/messages/{messageId}/attachments/{name}: get: summary: Retrieve mail attachment's content - operationId: smtpMailAttachment + operationId: smtpMessageAttachment parameters: - name: messageId in: path @@ -302,51 +302,10 @@ paths: type: string "404": description: mail attachment not found - /api/services/mail/mails/{messageId}: - get: - summary: Retrieve the mail with the given messageId - operationId: smtpMail - deprecated: true - parameters: - - name: messageId - in: path - required: true - schema: - type: string - responses: - "200": - description: mail - content: - application/json: - schema: - $ref: './mail.yml#/components/schemas/SmtpMessage' - "404": - description: mail not found - /api/services/mail/mails/{messageId}/attachments/{name}: - get: - summary: Retrieve mail attachment's content - operationId: smtpMailAttachment - parameters: - - name: messageId - in: path - required: true - schema: - type: string - - name: name - in: path - required: true - schema: - type: string - responses: - "200": - description: mail - content: - '*/*': - schema: {} - "404": - description: mail attachment not found /api/services/kafka: $ref: './kafka.yml#/components/pathItems/Clusters' + /api/services/kafka/{cluster}/channels/{channel}: + $ref: './kafka.yml#/components/pathItems/Channel' /api/services/kafka/{cluster}/topics: $ref: './kafka.yml#/components/pathItems/Topics' /api/services/kafka/{cluster}/topics/{topic}: @@ -382,43 +341,9 @@ paths: /api/services/kafka/{cluster}/clients/{clientId}: $ref: './kafka.yml#/components/pathItems/Client' /api/events: - get: - summary: list of events - operationId: events - parameters: - - name: traits - in: query - required: false - schema: - type: object - additionalProperties: - type: string - responses: - "200": - description: name of kafka cluster - content: - application/json: - schema: - $ref: 'common.yml#/components/schemas/Events' + $ref: './events.yaml#/components/pathItems/Events' /api/events/{id}: - get: - summary: Information about the kafka cluster - operationId: event - parameters: - - name: id - in: path - required: true - schema: - type: string - responses: - "200": - description: name of kafka cluster - content: - application/json: - schema: - $ref: 'common.yml#/components/schemas/Event' - "404": - description: no event exists for specified id + $ref: './events.yaml#/components/pathItems/Event' /api/metrics: get: summary: get list of metrics diff --git a/webui/e2e/mocks/events.yaml b/webui/e2e/mocks/events.yaml new file mode 100644 index 000000000..afc349a10 --- /dev/null +++ b/webui/e2e/mocks/events.yaml @@ -0,0 +1,125 @@ +components: + pathItems: + Events: + get: + summary: list of events + operationId: events + parameters: + - $ref: '#/components/parameters/Traits' + - $ref: '#/components/parameters/Http' + responses: + "200": + description: name of kafka cluster + content: + application/json: + schema: + $ref: '#/components/schemas/Events' + Event: + get: + summary: Information about the kafka cluster + operationId: event + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Specify multiple values separated by commas or by repeating the parameter (OR logic). Case-insensitive. + responses: + "200": + description: name of kafka cluster + content: + application/json: + schema: + $ref: '#/components/schemas/Event' + "404": + description: no event exists for specified id + parameters: + Traits: + name: traits + in: query + required: false + style: deepObject + explode: true + schema: + type: object + additionalProperties: + type: string + Http: + name: http + in: query + required: false + style: deepObject + explode: true + schema: + $ref: '#/components/schemas/HttpEventFilter' + schemas: + Events: + type: object + properties: + items: + type: array + items: + $ref: '#/components/schemas/Event' + page: + type: integer + size: + type: integer + total: + type: integer + Event: + type: object + properties: + id: + type: string + traits: + type: object + additionalProperties: + type: string + time: + type: string + data: + anyOf: + - $ref: 'kafka.yml#/components/schemas/KafkaEvent' + - description: any + type: object + additionalProperties: true + HttpEventFilter: + type: object + properties: + method: + type: string + description: | + Filters by HTTP method. Specify multiple methods separated by commas or by repeating the parameter (OR logic). + Case-insensitive. + pattern: '^[A-Za-z]+(,[A-Za-z]+)*$' + url: + type: string + requestHeader: + type: array + items: + type: string + description: | + Format Name:ValueRegex. Without :, only checks that the header exists; :Regex matches against all headers. Repeat the parameter to combine filters with AND. + example: 'Content-Type:application/json' + statusCode: + type: string + examples: + - 200,201 + - '-200' + - 200-299 + responseHeader: + type: array + items: + type: string + description: | + Format Name:ValueRegex. Without :, only checks that the header exists; :Regex matches against all headers. Repeat the parameter to combine filters with AND. + example: 'Content-Type:application/json' + clientIP: + type: string + examples: + - 192.168.0.1 + - 192.168.0.1,10.0.0.1 + - '!127.0.0.1' + - 192.168.0.0/24 + diff --git a/webui/e2e/mocks/helpers.ts b/webui/e2e/mocks/helpers.ts new file mode 100644 index 000000000..c0502cb7a --- /dev/null +++ b/webui/e2e/mocks/helpers.ts @@ -0,0 +1,28 @@ +export function getInfo(config: Config, type: string) { + const info: Info = { + name: config.name, + description: config.description, + version: config.version, + type: type, + } + if (config.contact) { + info.contact = { + name: config.contact.name, + email: config.contact.email, + url: config.contact.url + } + } + if (config.status) { + info.status = config.status + } + return info +} + +export function matchTraits(e: ServiceEvent, traits: Traits) { + for (let t in traits) { + if (e.traits[t] !== traits[t]) { + return false + } + } + return true +} \ No newline at end of file diff --git a/webui/e2e/mocks/services_http.js b/webui/e2e/mocks/http.ts similarity index 79% rename from webui/e2e/mocks/services_http.js rename to webui/e2e/mocks/http.ts index e69630042..92b4827f2 100644 --- a/webui/e2e/mocks/services_http.js +++ b/webui/e2e/mocks/http.ts @@ -1,4 +1,99 @@ -import {metrics} from "metrics" +import { app } from 'mokapi' +import { metrics, sum, max} from './metrics' + +export default function () { + app.http() + .operation('serviceHttp', (req, res) => { + const name = req.path['name'] + const service = apps.find(x => x.name === name) + if (!service) { + res.rebuild(404) + } else { + res.data = { + name: service.name, + description: service.description, + version: service.version, + ...service.contact ? { contact: service.contact } : {}, + servers: service.servers, + paths: service.paths.map(p => { + return { + path: p.path, + ...('summary' in p ? { summary: p.summary } : {}), + ...('description' in p ? { description: p.description } : {}), + status: p.status, + ...('errors' in p ? { errors: p.errors } : {}), + operations: p.operations.map(o => { + return { + method: o.method, + ...('summary' in o ? { summary: o.summary } : {}), + ...('description' in o ? { description: o.description } : {}), + ...('operationId' in o ? { operationId: o.operationId } : {}), + deprecated: ('deprecated' in o ? o.deprecated : false), + ...('tags' in o ? { tags: o.tags } : {}), + status: o.status, + ...('errors' in o ? { errors: o.errors } : {}), + metrics: { + requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path }, { name: 'method', value: o.method.toUpperCase() }), + requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path }, { name: 'method', value: o.method.toUpperCase() }), + request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path }, { name: 'method', value: o.method.toUpperCase() }) + } + } + }) + } + }), + } + } + }) + .operation('httpOperations', (req, res) => { + const name = req.path['name'] + const service = apps.find(x => x.name === name) + if (!service) { + res.rebuild(404) + return + } + const method = req.query['method'] + const path = req.query['path'] + let operations = [] + for (const p of service.paths) { + if (!path || p.path === path) { + for (const o of p.operations) { + if (!method || o.method.toLowerCase() === method.toLowerCase()) { + operations.push({ + ...o, + metrics: { + requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), + request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }) + } + }) + } + } + } + } + res.data = operations + }) + .operation('httpOpenApi', (req, res) => { + const serviceName = req.path['name'] + const format = req.path['format'] + const service = apps.find(x => x.name === serviceName) + if (!service) { + res.rebuild(404) + return + } + switch (format) { + case 'yaml': + res.headers['Content-Type'] = 'application/yaml' + res.data = service + return + case 'json': + res.headers['Content-Type'] = 'application/json' + res.data = service + return + default: + throw new Error(`Unsupported format: ${format}`) + } + }) +} const category = { type: "object", @@ -11,7 +106,7 @@ const category = { type: "string" } }, - xml: {name: "Category"} + xml: { name: "Category" } } const tag = { @@ -25,7 +120,7 @@ const tag = { type: "string" } }, - xml: {name: "Tag"} + xml: { name: "Tag" } } const pet = { @@ -70,7 +165,7 @@ const pet = { enum: ["available", "pending", "sold"] } }, - xml: {name: "Pet"} + xml: { name: "Pet" } } const book = { @@ -160,7 +255,7 @@ export let apps = [ { type: "query", name: "$format", - schema: {type: "string"}, + schema: { type: "string" }, required: false, deprecated: false, explode: false, @@ -414,7 +509,7 @@ export let apps = [ headers: [ { name: "Expires-After", - schema: {type: "string", format: "date-time"}, + schema: { type: "string", format: "date-time" }, description: "date in UTC when token expires", } ], @@ -626,7 +721,7 @@ export let events = [ path: "/pet", method: 'POST' }, - time: '2023-02-13T08:49:25.482366+01:00', + time: '2023-02-13T08:49:25.482366+01:00', data: { deprecated: true, request: { @@ -665,10 +760,10 @@ export let events = [ event: "http" }, logs: [ - { level: 'info', message: 'a custom log message, see https://mokapi.io'}, - { level: 'warn', message: 'a warn log message'}, - { level: 'error', message: 'a error log message'}, - { level: 'debug', message: 'a debug log message'} + { level: 'info', message: 'a custom log message, see https://mokapi.io' }, + { level: 'warn', message: 'a warn log message' }, + { level: 'error', message: 'a error log message' }, + { level: 'debug', message: 'a debug log message' } ], parameters: [ JSON.stringify({ @@ -689,7 +784,7 @@ export let events = [ headers: { "Content-Type": "application/json" }, - data: {"foo":"bar"} + data: { "foo": "bar" } }) ] } @@ -725,7 +820,7 @@ export let events = [ statusCode: 201, headers: { "Content-Type": "application/json", - "Set-Cookie": "sessionId=38afes7a8" + "Set-Cookie": "sessionId=38afes6v3" }, size: 512 }, @@ -876,5 +971,78 @@ export let events = [ duration: 133, clientIP: '192.0.1.127' } + }, + { + id: "6f954c79-c8ff-45bb-bddf-c88da57d53d9", + traits: { + namespace: "http", + name: "Swagger Petstore", + path: "/requestWithParams", + method: 'GET' + }, + time: '2023-02-13T10:05:25.583366+01:00', + data: { + request: { + method: "GET", + url: "http://127.0.0.1:18080/requestWithParams", + contentType: "application/json", + parameters: [], + body: '"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor"' + }, + response: { + statusCode: 200, + headers: { + "Content-Type": "application/json", + }, + body: `"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor"`, + size: 512 + }, + duration: 133, + actions: [ + { + duration: 20, + tags: { + name: "dashboard", + file: "examples/mokapi/http_handler.js", + fileKey: "b6fea8ac-56c7-4e73-a9c0-6887640bdca8", + event: "http" + }, + logs: [], + parameters: [ + JSON.stringify({ + method: "POST", + url: "http://127.0.0.1:18080/pet", + parameters: [ + { + name: 'Accept-Encoding', + type: 'header', + raw: 'gzip, deflate' + } + ], + contentType: "application/xml", + body: { + truncated: true, + size: 1000, + hash: 'abc', + preview: 'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor' + } + }), + JSON.stringify({ + statusCode: 200, + headers: { + "Content-Type": "application/json" + }, + data: { + truncated: true, + size: 1000, + hash: 'abc', + preview: 'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor' + } + }) + ] + } + ], + clientIP: '192.0.1.127' + } } ] \ No newline at end of file diff --git a/webui/e2e/mocks/http.yml b/webui/e2e/mocks/http.yml index 8458a9707..1eb559f8b 100644 --- a/webui/e2e/mocks/http.yml +++ b/webui/e2e/mocks/http.yml @@ -89,9 +89,9 @@ components: metrics: type: object properties: - http_requests_total: + requests_total: type: integer - http_requests_errors_total: + requests_errors_total: type: integer request_timestamp: type: integer diff --git a/webui/e2e/mocks/http_handler.js b/webui/e2e/mocks/http_handler.js index a18e40437..68205a113 100644 --- a/webui/e2e/mocks/http_handler.js +++ b/webui/e2e/mocks/http_handler.js @@ -1,448 +1,21 @@ import { on, sleep } from 'mokapi' import { clusters as kafkaClusters, events as kafkaEvents, configs as kafkaConfigs } from './kafka.ts' -import { apps as httpServices, events as httpEvents, configs as httpConfigs } from './services_http.js' -import { services as mailServices, mailEvents, getMail, getAttachment } from './mail.js' -import { server as ldapServers, searches } from './ldap.js' -import { clusters as mqttClusters, events as mqttEvents } from './mqtt.js' -import { services as websocketServices, events as websocketEvents } from './websocket.js' +import { apps as httpServices, events as httpEvents, configs as httpConfigs } from './http' +import { services as mailServices, mailEvents, getMail, getAttachment } from './mail' +import { server as ldapServers, searches } from './ldap' +import { clusters as mqttClusters, events as mqttEvents } from './mqtt' +import { services as websocketServices, events as websocketEvents } from './websocket' import { metrics, sum, max } from './metrics.ts' import { get, post, fetch } from 'mokapi/http' -import { base64 } from 'mokapi/encoding'; -import { yaml } from 'mokapi/yaml' const configs = {} -const jobs = [] export default async function() { const apiBaseUrl = `http://localhost:8091/mokapi` - const res = get(`${apiBaseUrl}/api/info`) - const { version, buildTime } = res.json() on('http', function(request, response) { - response.headers["Access-Control-Allow-Methods"] = "*" - response.headers["Access-Control-Allow-Headers"] = "*" - response.headers["Access-Control-Allow-Origin"] = "*" - response.headers['Mokapi-Version'] = version - response.headers['Mokapi-Build-Time'] = buildTime switch (request.operationId) { - case 'info': { - - response.data = { - version: "0.11.0", - activeServices: ["http", "kafka", "ldap", "mail", "mqtt", "websocket"], - search: { enabled: true } } - return - } - case 'services': - response.data = getServices(request.query['type']) - return - case 'serviceHttp': - const name = request.path['name'] - const service = httpServices.find(x => x.name === name) - if (service) { - response.data = { - name: service.name, - description: service.description, - version: service.version, - ...service.contact ? { contact: service.contact } : {}, - servers: service.servers, - paths: service.paths.map(p => { - return { - path: p.path, - ...p.summary ? { summary: p.summary } : {}, - ...p.description ? { description: p.description } : {}, - status: p.status, - ...p.errors ? { errors: p.errors } : {}, - operations: p.operations.map(o => { - return { - method: o.method, - ...o.summary ? {summary: o.summary} : {}, - ...o.description ? {description: o.description} : {}, - ...o.operationId ? { operationId: o.operationId } : {}, - deprecated: o.deprecated ?? false, - ...o.tags ? { tags: o.tags } : {}, - status: o.status, - ...o.errors ? { errors: o.errors } : {}, - metrics: { - http_requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), - http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), - http_request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }) - } - } - }) - } - }), - } - } else{ - response.statusCode = 404 - } - return - case 'httpOperations': { - const name = request.path['name'] - const service = httpServices.find(x => x.name === name) - if (!service) { - response.statusCode = 404 - return - } - const method = request.query['method'] - const path = request.query['path'] - let operations = [] - for (const p of service.paths) { - if (!path || p.path === path) { - for (const o of p.operations) { - if (!method || o.method.toLowerCase() === method.toLowerCase()) { - operations.push({ - ...o, - metrics: { - http_requests_total: sum('http_requests_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), - http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }), - http_request_timestamp: max('http_request_timestamp', { name: 'service', value: service.name }, { name: 'endpoint', value: p.path}, { name: 'method', value: o.method.toUpperCase() }) - } - }) - } - } - } - } - response.data = operations - return - } - case 'httpOpenApi': { - console.log(request) - const serviceName = request.path['name'] - const format = request.path['format'] - const service = httpServices.find(x => x.name === serviceName) - if (!service) { - response.statusCode = 404 - return - } - switch (format) { - case 'yaml': - response.headers['Content-Type'] = 'application/yaml' - response.data = service - return - case 'json': - response.headers['Content-Type'] = 'application/json' - response.data = service - return - default: - throw new Error(`Unsupported format: ${format}`) - } - } - case 'serviceKafka': - response.data = { - name: kafkaClusters[0].name, - description: kafkaClusters[0].description, - version: kafkaClusters[0].version, - contact: kafkaClusters[0].contact, - servers: kafkaClusters[0].servers, - topics: kafkaClusters[0].topics.map(x => { - return { - name: x.name, - ...x.summary === undefined ? {} : { summary: x.summary }, - metrics: { - kafka_messages_total: metrics.find(m => m.name === `kafka_messages_total{service="Kafka World",topic="${x.name}"}`).value, - kafka_message_timestamp: metrics.find(m => m.name === `kafka_message_timestamp{service="Kafka World",topic="${x.name}"}`).value - } - } - }), - groups: kafkaClusters[0].groups.map(x => { - return { - name: x.name, - generation: x.generation, - state: x.state, - protocol: x.protocol, - members: x.members.length ?? 0, - metrics: x.metrics - } - }), - clients: kafkaClusters[0].clients.map(c => { - return { - clientId: c.clientId, - address: c.address, - software: c.software, - } - }), - configs: kafkaClusters[0].configs.map(c => { - return { - id: c.id, - url: c.url, - provider: c.provider, - time: c.time - } - }) - } - return - case 'kafkaGroup': - response.data = kafkaClusters[0].groups.find(x => x.name === request.path['group']) - return - case 'serviceMail': - response.data = mailServices[0] - return - case 'kafkaClusters': - response.data = kafkaClusters.map(x => getInfo(x, 'kafka')) - return - case 'kafkaTopics': - response.data = kafkaClusters[0].topics - return - case 'kafkaTopic': - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - response.data = { - ...topic, - groups: kafkaClusters[0].groups.filter(g => g.topics.includes(topic.name)).map(x => { - return { - name: x.name, - generation: x.generation, - state: x.state, - protocol: x.protocol, - members: x.members.length ?? 0, - metrics: x.metrics - } - }), - } - if (response.data === undefined) { - response.statusCode = 404 - } - return - case 'kafkaProduceTopic': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - } else { - response.data = { - offsets: [ - { - partition: 0, - offset: 1234, - } - ] - } - } - return - } - case 'kafkaPartitions': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - return - } - response.data = topic.partitions - return - } - case 'kafkaPartition': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - return - } - const id = request.path.partitionId - if (id >= topic.partitions.length) { - response.statusCode = 404 - } else { - response.data = topic.partitions[id] - } - return - } - case 'kafkaProducePartition': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - return - } - const id = request.path.partitionId - if (id >= topic.partitions.length) { - response.statusCode = 404 - } else { - response.data = { - offsets: [ - { - partition: id, - offset: 1234, - } - ] - } - } - return - } - case 'kafkaOffsets': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - return - } - const id = request.path.partitionId - if (id >= topic.partitions.length) { - response.statusCode = 404 - } else { - const events = getEvents({topic: topic.name}).filter(x => x.data.partition === id) - if (request.header['Accept'] === 'application/json') { - response.data = events.map(x => ({ - key: x.data.key.value, - value: JSON.parse(x.data.message.value) - })) - } else { - response.data = events.map(x => ({ - key: base64.encode(x.data.key.value), - value: base64.encode(x.data.message.value) - })) - } - } - return - } - case 'kafkaOffset': { - const topic = kafkaClusters[0].topics.find(x => x.name === request.path.topic) - if (!topic) { - response.statusCode = 404 - return - } - const id = request.path.partitionId - if (id >= topic.partitions.length) { - response.statusCode = 404 - } else { - const events = getEvents({topic: topic.name}).filter(x => x.data.partition === id && x.data.offset === request.path.offset) - if (events.length === 0) { - response.statusCode = 404 - return - } - if (request.header['Accept'] === 'application/json') { - response.data = { - key: events[0].data.key.value, - value: JSON.parse(events[0].data.message.value) - } - } else { - response.data = { - key: base64.encode(events[0].data.key.value), - value: base64.encode(events[0].data.message.value) - } - } - } - return - } - case 'kafkaClient': - response.data = kafkaClusters[0].clients.find(x => x.clientId === request.path['clientId']) - return - case 'mailboxes': - response.data = mailServices[0].mailboxes - return - case 'mailbox': - for (const mb of mailServices[0].mailboxes) { - if (mb.name === request.path.mailbox) { - response.data = { - ...mb, - folders: mb.folders.map(x => x.name) - } - return - } - } - response.statusCode = 404 - return - case 'smtpMessages': - for (const mb of mailServices[0].mailboxes) { - if (mb.name === request.path.mailbox) { - const result = [] - for (const f of mb.folders) { - result.push(...f.messages) - } - response.data = result - return - } - } - response.statusCode = 404 - return - case 'smtpMail': - case 'smtpMessage': - const messageId = request.path['messageId'] - const mail = getMail(messageId) - if (mail == null) { - response.statusCode = 404 - } else { - response.data = { - service: mailServices[0].name, - data: mail - } - } - return - case 'smtpMailAttachment': - case 'smtpAttachment': - const attachment = getAttachment(request.path['messageId'], request.path['name']) - if (attachment == null) { - response.statusCode = 404 - } else { - response.data = attachment.data - response.headers['Content-Type'] = attachment.contentType - } - return - case 'serviceLdap': - response.data = ldapServers[0] - return - case 'metrics': - let names = request.query['names'] - if (!names) { - response.data = metrics - } else{ - response.data = metrics.filter(x => names.includes(x.name)) - } - return - case 'metricsType': - let type = request.path['type'] - response.data = metrics.filter(x => x.name.startsWith(type + '_')) - return - case 'events': - response.data = getEvents(request.query['traits']) - return - case 'event': - let id = request.path["id"] - let e = getEvent(id) - if (!e){ - response.statusCode = 404 - }else{ - response.data = e - } - return - case 'example': { - const res = post(`${apiBaseUrl}/api/schema/example`, request.body) - response.statusCode = res.statusCode - response.headers = res.headers - response.body = res.body - return - } - case 'validate': - sleep(1000) - let url = `${apiBaseUrl}/api/schema/validate` - if (request.query['outputFormat']) { - url += '?outputFormat=' + request.query['outputFormat'] - } - const res = post(url, request.body) - console.log(res.statusCode) - response.statusCode = res.statusCode - response.headers = res.headers - response.body = res.body - response.data = null - return - case 'configs': - response.data = getConfigs() - return - case 'config': - const config = getConfig(request.path.id) - if (config) { - response.data = config - return - } else { - response.statusCode = 404 - response.data = '' - return - } - case 'configData': - const configData = configs[request.path.id] - if (configData) { - response.data = configData.data - response.headers = configData.headers - return - } else { - response.statusCode = 404 - response.data = '' - return - } case 'fakerTree': { const resp = get(`${apiBaseUrl}/mokapi/api/faker/tree`) response.body = resp.body @@ -479,246 +52,6 @@ export default async function() { } } } - - jobs.push( - { - id: "8234", - traits: { - namespace: "job", - name: "cron", - }, - time: new Date(Date.now()).toISOString(), - data: { - schedule: '3s', - maxRuns: -1, - runs: 1, - nextRun: new Date(Date.now() + 60 * 60 * 1000).toISOString(), - duration: 2300, - tags: { - name: "cron", - file: "cron.js", - fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", - } - } - }, - { - id: "8134", - traits: { - namespace: "job", - name: "cron", - }, - time: new Date(Date.now() - 60 * 60 * 1000).toISOString(), - data: { - schedule: '3s', - maxRuns: 10, - runs: 1, - nextRun: new Date(Date.now()).toISOString(), - duration: 2300, - logs: [ - { - level: 'error', - message: 'an error message, see https://mokapi.io' - } - ], - error: { - message: 'There was an error' - }, - tags: { - name: "cron", - file: "cron.js", - fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", - } - }, - - }, - { - id: "81344", - traits: { - namespace: "job", - name: "cron", - }, - time: new Date(Date.now() - 60 * 60 * 1000).toISOString(), - data: { - schedule: '3s', - runs: 1, - nextRun: new Date(Date.now()).toISOString(), - duration: 2300, - logs: [ - { - level: 'warn', - message: 'a warning message, see https://mokapi.io' - } - ], - tags: { - name: "cron", - file: "cron.js", - fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", - } - }, - - } - ) -} - -function getServices(type) { - const services = [] - if (!type || type === 'http') { - services.push(...httpServices.map(x => { - return { - ...getInfo(x, 'http'), - metrics: { - http_requests_total: sum('http_requests_total', { name: 'service', value: x.name }), - http_requests_errors_total: sum('http_requests_errors_total', { name: 'service', value: x.name }), - http_request_timestamp: max('http_request_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - if (!type || type === 'kafka') { - services.push(...kafkaClusters.map(x => { - return { - ...getInfo(x, 'kafka'), - metrics: { - kafka_messages_total: sum('kafka_messages_total', { name: 'service', value: x.name }), - kafka_message_timestamp: max('kafka_message_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - if (!type || type === 'mail') { - services.push(...mailServices.map(x => { - return { - ...getInfo(x, 'mail'), - metrics: { - mail_mails_total: sum('mail_mails_total', { name: 'service', value: x.name }), - mail_mail_timestamp: max('mail_mail_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - if (!type || type === 'ldap') { - services.push(...ldapServers.map(x => { - return { - ...getInfo(x, 'ldap'), - metrics: { - ldap_requests_total: sum('ldap_requests_total', { name: 'service', value: x.name }), - ldap_request_timestamp: max('ldap_request_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - if (!type || type === 'mqtt') { - services.push(...mqttClusters.map(x => { - return { - ...getInfo(x, 'mqtt'), - metrics: { - mqtt_messages_total: sum('mqtt_messages_total', { name: 'service', value: x.name }), - mqtt_message_timestamp: max('mqtt_message_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - if (!type || type === 'websocket') { - services.push(...websocketServices.map(x => { - return { - ...getInfo(x, 'websocket'), - metrics: { - websocket_messages_total: sum('websocket_messages_total', { name: 'service', value: x.name }), - websocket_message_timestamp: max('websocket_message_timestamp', { name: 'service', value: x.name }) - } - } - })) - } - - return services -} - -function getEvent(id) { - for (const e of httpEvents){ - if (e.id === id){ - return e - } - } - for (const e of kafkaEvents){ - if (e.id === id){ - return e - } - } - for (const e of mqttEvents){ - if (e.id === id){ - return e - } - } - for (const e of mailEvents){ - if (e.id === id){ - return e - } - } - for (const e of searches){ - if (e.id === id){ - return e - } - } - for (const e of jobs) { - if (e.id === id) { - return e - } - } - for (const e of websocketEvents) { - if (e.id === id) { - return e - } - } - return null -} - -function getEvents(traits) { - const result = [] - for (const e of httpEvents){ - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of kafkaEvents){ - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of mqttEvents){ - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of websocketEvents) { - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of mailEvents) { - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of searches) { - if (matchEvent(e, traits)) { - result.push(e) - } - } - for (const e of jobs) { - if (matchEvent(e, traits)) { - result.push(e) - } - } - return result -} - -function matchEvent(e, traits) { - for (let t in traits) { - if (e.traits[t] !== traits[t]) { - return false - } - } - return true } function getConfig(id) { @@ -742,26 +75,6 @@ function getConfigs() { ] } -function getInfo(config, type) { - const info = { - name: config.name, - description: config.description, - version: config.version, - type: type, - } - if (config.contact) { - info.contact = { - name: config.contact.name, - email: config.contact.email, - url: config.contact.url - } - } - if (config.status) { - info.status = config.status - } - return info -} - const jobConfig = { '6f5cab53-9f51-4bbe-bb32-fe41ee71d542': { id: '6f5cab53-9f51-4bbe-bb32-fe41ee71d542', diff --git a/webui/e2e/mocks/jobs.ts b/webui/e2e/mocks/jobs.ts new file mode 100644 index 000000000..79ee9ff14 --- /dev/null +++ b/webui/e2e/mocks/jobs.ts @@ -0,0 +1,89 @@ +export const jobs = [ + { + id: "8234", + traits: { + namespace: "job", + name: "cron", + }, + time: new Date(Date.now()).toISOString(), + data: { + schedule: '3s', + maxRuns: -1, + runs: 1, + nextRun: new Date(Date.now() + 60 * 60 * 1000).toISOString(), + duration: 2300, + tags: { + name: "cron", + file: "cron.js", + fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", + } + } + }, + { + id: "8134", + traits: { + namespace: "job", + name: "cron", + }, + time: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + data: { + schedule: '3s', + maxRuns: 10, + runs: 1, + nextRun: new Date(Date.now()).toISOString(), + duration: 2300, + logs: [ + { + level: 'error', + message: 'an error message, see https://mokapi.io' + } + ], + error: { + message: 'There was an error' + }, + tags: { + name: "cron", + file: "cron.js", + fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", + } + }, + + }, + { + id: "81344", + traits: { + namespace: "job", + name: "cron", + }, + time: new Date(Date.now() - 60 * 60 * 1000).toISOString(), + data: { + schedule: '3s', + runs: 1, + nextRun: new Date(Date.now()).toISOString(), + duration: 2300, + logs: [ + { + level: 'warn', + message: 'a warning message, see https://mokapi.io' + } + ], + tags: { + name: "cron", + file: "cron.js", + fileKey: "6f5cab53-9f51-4bbe-bb32-fe41ee71d542", + } + }, + + } +] + +export const config = { + '6f5cab53-9f51-4bbe-bb32-fe41ee71d542': { + id: '6f5cab53-9f51-4bbe-bb32-fe41ee71d542', + url: 'file://cron.js', + provider: 'file', + time: '2025-05-01T08:49:25.482366+01:00', + data: "import { every } from 'mokapi'; export default function() { every('1s', function() {}); }", + filename: 'cron.js' + } +} \ No newline at end of file diff --git a/webui/e2e/mocks/kafka.ts b/webui/e2e/mocks/kafka.ts index fed386f8b..7792b23a4 100644 --- a/webui/e2e/mocks/kafka.ts +++ b/webui/e2e/mocks/kafka.ts @@ -1,5 +1,201 @@ -import { metrics, filter as filterMetrics, parseLabels } from './metrics.ts' +import { app } from 'mokapi' +import { sum, max, metrics, filter as filterMetrics, parseLabels } from './metrics' import { base64 } from 'mokapi/encoding'; +import { getInfo, matchTraits } from './helpers'; + +export default function () { + app.http() + .operation('serviceKafka', (_, res) => { + const channels = clusters[0].topics.filter(x => !clusters[0].topics.some(y => y.instances && y.instances.find(z => z.name === x.name))) + res.data = { + name: clusters[0].name, + description: clusters[0].description, + version: clusters[0].version, + contact: clusters[0].contact, + servers: clusters[0].servers, + topics: channels.map(x => getChannelInfo(x)), + groups: clusters[0].groups.map(x => { + return { + name: x.name, + generation: x.generation, + state: x.state, + protocol: x.protocol, + members: x.members.length ?? 0, + metrics: x.metrics + } + }), + clients: clusters[0].clients.map(c => { + return { + clientId: c.clientId, + address: c.address, + software: c.software, + } + }), + configs: clusters[0].configs.map(c => { + return { + id: c.id, + url: c.url, + provider: c.provider, + time: c.time + } + }) + } + }) + .operation('kafkaChannel', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path['channel']) + if (!topic) { + res.rebuild(404) + return + } + const { partitions, ...channel } = topic + res.data = { + ...channel, + ...getChannelInfo(topic) + } + }) + .operation('kafkaGroup', (req, res) => { + res.data = clusters[0].groups.find(x => x.name === req.path['group']) + }) + .operation('kafkaClusters', (req, res) => { + res.data = clusters.map(x => getInfo(x, 'kafka')) + }) + .operation('kafkaTopics', (req, res) => { + res.data = clusters[0].topics + }) + .operation('kafkaTopic', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic || topic.instances) { + res.rebuild(404) + return + } + const { parameters, ...data } = topic + res.data = { + ...data, + ...('parameters' in topic ? { parameters: topic.parameters }: {}), + groups: clusters[0].groups.filter(g => g.topics.includes(topic.name)).map(x => { + return { + name: x.name, + generation: x.generation, + state: x.state, + protocol: x.protocol, + members: x.members.length ?? 0, + metrics: x.metrics + } + }), + } + }) + .operation('kafkaProduceTopic', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic) { + res.rebuild(404) + return + } + res.data = { + offsets: [ + { + partition: 0, + offset: 1234, + } + ] + } + }) + .operation('kafkaPartitions', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic) { + res.rebuild(404) + return + } + res.data = topic.partitions + }) + .operation('kafkaPartition', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic || !topic.partitions) { + res.rebuild(404) + return + } + const id = req.path.partitionId + if (id >= topic.partitions.length) { + res.rebuild(404) + return + } + res.data = topic.partitions[id] + }) + .operation('kafkaProducePartition', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic || !topic.partitions) { + res.rebuild(404) + return + } + const id = req.path.partitionId + if (id >= topic.partitions.length) { + res.rebuild(404) + return + } + res.data = { + offsets: [ + { + partition: id, + offset: 1234, + } + ] + } + }) + .operation('kafkaOffsets', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic || !topic.partitions) { + res.rebuild(404) + return + } + const id = req.path.partitionId + if (id >= topic.partitions.length) { + res.rebuild(404) + return + } + const data = events.filter(x => matchTraits(x, { topic: topic.name, type: 'message' }) && x.data.partition === id) + if (req.header['Accept'] === 'application/json') { + res.data = data.map(x => ({ + key: x.data.key?.value, + value: JSON.parse(x.data.message?.value ?? '') + })) + } else { + res.data = data.map(x => ({ + key: base64.encode(x.data.key?.value ?? ''), + value: base64.encode(x.data.message?.value ?? '') + })) + } + }) + .operation('kafkaOffset', (req, res) => { + const topic = clusters[0].topics.find(x => x.name === req.path.topic) + if (!topic || !topic.partitions) { + res.rebuild(404) + return + } + const id = req.path.partitionId + if (id >= topic.partitions.length) { + res.rebuild(404) + return + } + const data = events.filter(x => matchTraits(x, { topic: topic.name, type: 'message' }) && x.data.partition === id && x.data.offset === req.path.offset) + if (data.length === 0) { + res.rebuild(404) + return + } + if (req.header['Accept'] === 'application/json') { + res.data = { + key: events[0].data.key?.value, + value: JSON.parse(events[0].data.message?.value ?? '') + } + } else { + res.data = { + key: base64.encode(data[0].data.key?.value ?? ''), + value: base64.encode(data[0].data.message?.value ?? '') + } + } + }) + .operation('kafkaClient', (req, res) => { + res.data = clusters[0].clients.find(x => x.clientId === req.path['clientId']) + }) +} const Product = { type: 'object', @@ -34,6 +230,15 @@ const Product = { } } +const Order = { + type: 'object', + properties: { + orderId: { type: 'string', format: 'uuid' }, + amount: { type: 'number' }, + timestamp: { type: 'string', format: 'date-time' } + } +} + const UserSignedUp = { type: 'object', properties: { @@ -92,7 +297,7 @@ export let clusters = [ { name: 'Broker', host: 'localhost:9092', - tags: [{name: 'env:test', description: 'This environment is for running internal tests'}], + tags: [{ name: 'env:test', description: 'This environment is for running internal tests' }], description: 'Dashwood contempt on mr unlocked resolved provided of of. Stanhill wondered it it welcomed oh. Hundred no prudent he however smiling at an offence. If earnestly extremity he he propriety something admitting convinced ye.', configs: { 'log.retention.bytes': 1024, @@ -133,7 +338,7 @@ export let clusters = [ title: 'Shop New Order notification', summary: 'A message containing details of a new order', description: 'More info about how the order notifications are **created** and **used**.', - key: { schema: {type: 'string'} }, + key: { schema: { type: 'string' } }, payload: { schema: Product }, contentType: 'application/json' } @@ -164,13 +369,13 @@ export let clusters = [ 'userSignedUp': { name: 'userSignedUp', title: 'title', - key: { schema: {type: 'string'} }, + key: { schema: { type: 'string' } }, payload: { schema: UserSignedUp, format: 'application/vnd.oai.openapi+json;version=3.0.0' }, contentType: 'application/xml' }, 'second': { name: 'second', - key: { schema: {type: 'string'} }, + key: { schema: { type: 'string' } }, payload: { schema: SecondMessage }, contentType: 'application/json' } @@ -192,7 +397,7 @@ export let clusters = [ 'avro': { name: 'avro', title: 'avro', - key: { schema: {type: 'string'}, format: 'application/vnd.apache.avro;version=1.9.0' }, + key: { schema: { type: 'string' }, format: 'application/vnd.apache.avro;version=1.9.0' }, payload: { schema: { type: 'record', fields: [ @@ -204,20 +409,62 @@ export let clusters = [ contentType: 'avro/binary' }, } + }, + { + name: 'orders.{region}.{eventType}', + messages: { + 'orderMessage': { + name: 'OrderEvent', + title: 'Order Event Payload', + payload: { schema: Order }, + contentType: 'application/json' + } + }, + parameters: [ 'region', 'eventType' ], + instances: [ + { + name: 'orders.us.created', + parameters: { + region: 'us', + eventType: 'created' + }, + } + ], + }, + { + name: 'orders.us.created', + messages: { + 'orderMessage': { + name: 'OrderEvent', + title: 'Order Event Payload', + payload: { schema: Order }, + contentType: 'application/json' + } + }, + parameters: { eventType: 'created', region: 'eu' }, + partitions: [ + { + id: 0, + startOffset: 0, + offset: 0, + leader: { name: 'foo', addr: 'localhost:9002' }, + segments: 1 + } + ], } ], groups: [ { name: 'foo', generation: 0, - members:[ + members: [ { name: 'julie', clientId: 'consumer-1', addr: '127.0.0.1:15001', software: 'mokapi 1.0', heartbeat: 1654771269, - partitions: { 'mokapi.shop.products': [ 0,1 ], 'mokapi.shop.userSignedUp': [ 0 ] } + partitions: { 'mokapi.shop.products': [0, 1], 'mokapi.shop.userSignedUp': [0] } }, { name: 'hermann', @@ -225,30 +472,30 @@ export let clusters = [ addr: '127.0.0.1:15002', software: 'mokapi 1.0', heartbeat: 1654872269, - partitions: { 'mokapi.shop.products': [ 2 ], 'mokapi.shop.userSignedUp': [ ] } + partitions: { 'mokapi.shop.products': [2], 'mokapi.shop.userSignedUp': [] } } ], leader: 'julie', state: 'Stable', protocol: 'Range', - topics: [ 'mokapi.shop.products', 'mokapi.shop.userSignedUp' ], + topics: ['mokapi.shop.products', 'mokapi.shop.userSignedUp'], metrics: { - kafka_rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="foo"}`)?.value || 0, + rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="foo"}`)?.value || 0, topics: { - 'mokapi.shop.products': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'foo'}, { name: 'topic', value: 'mokapi.shop.products'}).map(m => { + 'mokapi.shop.products': filterMetrics('kafka_consumer_group_commit', { name: 'group', value: 'foo' }, { name: 'topic', value: 'mokapi.shop.products' }).map(m => { const labels = parseLabels(m) - return { + return { partition: labels.partition, - kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="${labels.partition}"}`)?.value || 0, - kafka_consumer_group_commit: m.value + consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.products",partition="${labels.partition}"}`)?.value || 0, + consumer_group_commit: m.value } }), - 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'foo'}, { name: 'topic', value: 'mokapi.shop.userSignedUp'}).map(m => { + 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', { name: 'group', value: 'foo' }, { name: 'topic', value: 'mokapi.shop.userSignedUp' }).map(m => { const labels = parseLabels(m) - return { + return { partition: labels.partition, - kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, - kafka_consumer_group_commit: m.value + consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, + consumer_group_commit: m.value } }) } @@ -264,22 +511,22 @@ export let clusters = [ addr: '127.0.0.1:15003', software: 'mokapi 1.0', heartbeat: 1654721269, - partitions: { 'mokapi.shop.userSignedUp': [ 0 ] } + partitions: { 'mokapi.shop.userSignedUp': [0] } } ], leader: 'george', state: 'Stable', protocol: 'Range', - topics: [ 'mokapi.shop.userSignedUp' ], + topics: ['mokapi.shop.userSignedUp'], metrics: { - kafka_rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="mokapi.shop.userSignedUp"}`)?.value || 0, + rebalance_timestamp: metrics.find(m => m.name === `kafka_rebalance_timestamp{service="Kafka World",group="mokapi.shop.userSignedUp"}`)?.value || 0, topics: { - 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', {name: 'group', value: 'bar'}, { name: 'topic', value: 'mokapi.shop.userSignedUp'}).map(m => { + 'mokapi.shop.userSignedUp': filterMetrics('kafka_consumer_group_commit', { name: 'group', value: 'bar' }, { name: 'topic', value: 'mokapi.shop.userSignedUp' }).map(m => { const labels = parseLabels(m) - return { + return { partition: labels.partition, - kafka_consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, - kafka_consumer_group_commit: m.value + consumer_group_lag: metrics.find(x => x.name === `kafka_consumer_group_lag{service="Kafka World",group="foo",topic="mokapi.shop.userSignedUp",partition="${labels.partition}"}`)?.value || 0, + consumer_group_commit: m.value } }) } @@ -288,7 +535,7 @@ export let clusters = [ ], metrics: metrics.filter(x => x.name.includes("kafka")), configs: [ - configs[ 'b6fea8ac-56c7-4e73-a9c0-6337640bdca8' ] + configs['b6fea8ac-56c7-4e73-a9c0-6337640bdca8'] ], clients: [ { @@ -320,41 +567,42 @@ export let clusters = [ ] export let events = [ - { - id: '5d549ffe-97cd-477d-8db6-27ed4963c08d', - traits: { - namespace: 'kafka', - name: 'Kafka World', - type: 'message', - topic: 'mokapi.shop.products' - }, - time: '2023-02-13T09:49:25.482366+01:00', - data: { - offset: 0, - key: { - value: 'GGOEWXXX0827' - }, - message: { - value: JSON.stringify( - { - id: 'GGOEWXXX0827', - name: 'Waze Women\'s Short Sleeve Tee', - description: 'Made of soft tri-blend jersey fabric, this great t-shirt will help you find your Waze. Made in USA.', - features: '

Jersey knit

\\n

37.5% cotton, 50% polyester, 12.5% rayon

\\n

Made in the USA

', - price: '18.99', - keywords: 'Waze Women\'s Short Sleeve Tee, Waze Short Sleeve Tee, Waze Women\'s Tees, Waze Women\'s tee, waze ladies tees, waze ladies tee, waze short sleeve tees, waze short sleeve tee', - url: 'Waze+Womens+Short+Sleeve+Tee', - category: 'apparel', - subcategory: 'apparel' - }) - }, - partition: 0, - messageId: 'shopOrder', - headers: { - foo: { value: 'bar', binary: '' }, - bar: { value: '', binary: 'dmFsdWUgYXMgYmluYXJ5'} - }, - actions: [ + { + id: '5d549ffe-97cd-477d-8db6-27ed4963c08d', + traits: { + namespace: 'kafka', + name: 'Kafka World', + type: 'message', + topic: 'mokapi.shop.products' + }, + time: '2023-02-13T09:49:25.482366+01:00', + data: { + topic: 'mokapi.shop.products', + offset: 0, + key: { + value: 'GGOEWXXX0827' + }, + message: { + value: JSON.stringify( + { + id: 'GGOEWXXX0827', + name: 'Waze Women\'s Short Sleeve Tee', + description: 'Made of soft tri-blend jersey fabric, this great t-shirt will help you find your Waze. Made in USA.', + features: '

Jersey knit

\\n

37.5% cotton, 50% polyester, 12.5% rayon

\\n

Made in the USA

', + price: '18.99', + keywords: 'Waze Women\'s Short Sleeve Tee, Waze Short Sleeve Tee, Waze Women\'s Tees, Waze Women\'s tee, waze ladies tees, waze ladies tee, waze short sleeve tees, waze short sleeve tee', + url: 'Waze+Womens+Short+Sleeve+Tee', + category: 'apparel', + subcategory: 'apparel' + }) + }, + partition: 0, + messageId: 'shopOrder', + headers: { + foo: { value: 'bar', binary: '' }, + bar: { value: '', binary: 'dmFsdWUgYXMgYmluYXJ5' } + }, + actions: [ { duration: 20, tags: { @@ -364,9 +612,9 @@ export let events = [ event: "kafka" } } - ] - } - }, + ] + } + }, { id: '670ce171-808c-4f8e-a7cd-209b008aada5', traits: { @@ -377,6 +625,7 @@ export let events = [ }, time: '2023-02-13T09:49:25.482366+01:00', data: { + topic: 'mokapi.shop.products', offset: 1, key: { value: 'GGOEWXXX0828' @@ -413,6 +662,7 @@ export let events = [ }, time: '2025-02-13T09:49:25.482366+01:00', data: { + topic: 'mokapi.shop.avro', offset: 1, key: { binary: 'VEVTVDEgw6Qg8JCAgA==' @@ -428,5 +678,82 @@ export let events = [ partition: 0, messageId: 'avro' } + }, + { + id: '84490626-b806-4420-898c-d6b28d1f4ab1', + traits: { + namespace: 'kafka', + name: 'Kafka World', + type: 'request', + clientId: 'consumer', + }, + time: '2025-02-13T09:49:25.482366+01:00', + data: { + "api": "Kafka World", + "header": { + "requestKey": 10, + "requestName": "FindCoordinator", + "version": 2 + }, + "request": { + "key": "mokapi.shop.products", + "keyType": 0 + }, + "response": { + "errorCode": "", + "errorMessage": "", + "host": "localhost", + "port": 9092 + } + } + }, + { + id: '3373222f-e854-4dc6-b369-1f67b79cf47d', + traits: { + namespace: 'kafka', + name: 'Kafka World', + type: 'message', + topic: 'orders.{region}.{eventType}', + instance: 'orders.us.created' + }, + time: '2023-02-13T09:49:25.482366+01:00', + data: { + topic: 'orders.us.created', + offset: 0, + key: { + value: 'a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d' + }, + message: { + value: JSON.stringify( + { + orderId: 'a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d', + amount: 149.99, + timestamp: '2026-10-17T10:15:30' + }) + }, + partition: 0, + messageId: 'shopOrder', + } + }, +] + +function getChannelInfo(c: any) { + return { + name: c.name, + ...c.summary === undefined ? {} : { summary: c.summary }, + ...('parameters' in c ? { parameters: c.parameters } : {}), + ...('instances' in c ? { instances: c.instances?.map((i: any) => ( + { + ...i, + metrics: { + messages_total: sum('kafka_messages_total', { name: 'service', value: clusters[0].name }, { name: 'topic', value: c.name }, { name: 'instance', value: i.name }), + message_timestamp: max('kafka_message_timestamp', { name: 'service', value: clusters[0].name }, { name: 'topic', value: c.name }, { name: 'instance', value: i.name }) + }, + } + )) } : {}), + metrics: { + messages_total: metrics.find(m => m.name === `kafka_messages_total{service="Kafka World",topic="${c.name}"}`)?.value ?? 0, + message_timestamp: metrics.find(m => m.name === `kafka_message_timestamp{service="Kafka World",topic="${c.name}"}`)?.value ?? 0 + } } - ] +} \ No newline at end of file diff --git a/webui/e2e/mocks/kafka.yml b/webui/e2e/mocks/kafka.yml index c475d4d69..4de815443 100644 --- a/webui/e2e/mocks/kafka.yml +++ b/webui/e2e/mocks/kafka.yml @@ -23,7 +23,30 @@ components: contact: $ref: './common.yml#/components/schemas/Contact' additionalProperties: false - + Channel: + get: + summary: Returns channel specification + operationId: kafkaChannel + parameters: + - in: path + name: cluster + required: true + schema: + type: string + - in: path + name: channel + required: true + schema: + type: string + responses: + '200': + description: The channel specification + content: + application/json: + schema: + $ref: '#/components/schemas/Channel' + '404': + description: Channel not found Topics: get: summary: Returns the list of topics that belongs to the specified Kafka cluster @@ -42,7 +65,7 @@ components: schema: type: array items: - $ref: '#/components/schemas/TopicInfo' + $ref: '#/components/schemas/ChannelInfo' Topic: parameters: - in: path @@ -65,6 +88,8 @@ components: application/json: schema: $ref: '#/components/schemas/Topic' + '404': + description: Topic not found post: summary: Produce messages to a topic, the partition will be chosen randomly operationId: kafkaProduceTopic @@ -376,7 +401,7 @@ components: topics: type: array items: - $ref: '#/components/schemas/TopicInfo' + $ref: '#/components/schemas/ChannelInfo' groups: type: array items: @@ -385,7 +410,7 @@ components: type: array items: $ref: './common.yml#/components/schemas/ConfigInfo' - TopicInfo: + ChannelInfo: type: object properties: name: @@ -396,25 +421,60 @@ components: type: integer messages: type: integer - Topic: + Channel: type: object properties: name: type: string + title: + type: string summary: type: string - partitions: - type: array - items: - $ref: '#/components/schemas/Partition' + description: + type: string messages: type: object additionalProperties: $ref: '#/components/schemas/MessageConfig' - bindings: - $ref: '#/components/schemas/TopicBindings' + parameters: + type: array + items: + type: string tags: $ref: '#/components/schemas/Tags' + metrics: + type: object + properties: + messages_total: + type: integer + message_timestamp: + type: integer + Topic: + type: object + properties: + name: + type: string + parameters: + type: object + additionalProperties: + type: string + bindings: + $ref: '#/components/schemas/TopicBindings' + partitions: + type: array + items: + $ref: '#/components/schemas/Partition' + groups: + type: array + items: + $ref: '#/components/schemas/GroupInfo' + metrics: + type: object + properties: + messages_total: + type: integer + message_timestamp: + type: integer Partition: type: object properties: @@ -444,7 +504,7 @@ components: metrics: type: object properties: - kafka_rebalance_timestamp: + rebalance_timestamp: type: integer topics: type: object @@ -478,7 +538,7 @@ components: metrics: type: object properties: - kafka_rebalance_timestamp: + rebalance_timestamp: type: integer topics: type: object @@ -601,3 +661,40 @@ components: type: string description: type: string + KafkaEvent: + type: object + properties: + offset: + type: integer + key: + $ref: '#/components/schemas/KafkaValue' + message: + $ref: '#/components/schemas/KafkaValue' + schemaId: + type: integer + partition: + type: integer + headers: + type: object + additionalProperties: + type: object + properties: + value: + type: string + binary: + type: string + deleted: + type: boolean + producerId: + type: number + producerEpoch: + type: number + sequenceNumber: + type: number + KafkaValue: + type: object + properties: + value: + type: string + binary: + type: string diff --git a/webui/e2e/mocks/ldap.js b/webui/e2e/mocks/ldap.ts similarity index 96% rename from webui/e2e/mocks/ldap.js rename to webui/e2e/mocks/ldap.ts index ba2556fe7..275550674 100644 --- a/webui/e2e/mocks/ldap.js +++ b/webui/e2e/mocks/ldap.ts @@ -1,4 +1,13 @@ -export let server = [ +import { app } from 'mokapi' + +export default function () { + app.http() + .operation('serviceLdap', (_, res) => { + res.data = servers[0] + }) +} + +export let servers = [ { name: "LDAP Testserver", description: "This is a sample LDAP server", diff --git a/webui/e2e/mocks/mail.js b/webui/e2e/mocks/mail.ts similarity index 64% rename from webui/e2e/mocks/mail.js rename to webui/e2e/mocks/mail.ts index 31caf47a4..4772f08d7 100644 --- a/webui/e2e/mocks/mail.js +++ b/webui/e2e/mocks/mail.ts @@ -1,3 +1,64 @@ +import { app } from 'mokapi' + +export default function () { + app.http() + .operation('serviceMail', (_, res) => { + res.data = services[0] + }) + .operation('mailboxes', (_, res) => { + res.data = services[0].mailboxes + }) + .operation('mailbox', (req, res) => { + for (const mb of services[0].mailboxes) { + if (mb.name === req.path.mailbox) { + res.data = { + ...mb, + folders: mb.folders.map(x => x.name) + } + return + } + } + res.rebuild(404) + }) + .operation('smtpMessages', (req, res) => { + for (const mb of services[0].mailboxes) { + if (mb.name === req.path.mailbox) { + const result = [] + for (const f of mb.folders) { + result.push(...f.messages) + } + res.data = result + return + } + } + res.rebuild(404) + }) + .operation('smtpMessage', (req, res) => { + const messageId = req.path.messageId + const mail = getMail(messageId) + if (mail == null) { + res.rebuild(404) + return + } + res.data = { + service: services[0].name, + data: mail + } + }) + .operation('smtpMessageAttachment', (req, res) => { + const messageId = req.path.messageId + const mail = getMail(messageId) + if (mail == null) { + res.rebuild(404) + return + } + res.data = { + service: services[0].name, + data: mail + } + }) +} + export let mails = [ { from: [{name: 'Alice', address: 'alice@mokapi.io'}], @@ -111,12 +172,13 @@ export let mailEvents = [ from: 'alice@mokapi.io', to: ['bob@mokapi.io'], messageId: '20230223-084925.763-4196@mokapi.io', - subject: 'A test mail' + subject: 'A test mail', + duration: 30016, }, }, ] -export function getMail(messageId) { +export function getMail(messageId: string) { for (let m of mails) { if (m.messageId === messageId) { return m @@ -125,8 +187,12 @@ export function getMail(messageId) { return null } -export function getAttachment(messageId, attachmentName) { +export function getAttachment(messageId: string, attachmentName: string) { const mail = getMail(messageId) + if (!mail) { + return null + } + for (let a of mail.attachments) { if (a.name === attachmentName) { return a diff --git a/webui/e2e/mocks/metrics.ts b/webui/e2e/mocks/metrics.ts index 466918a84..cf29fa2f2 100644 --- a/webui/e2e/mocks/metrics.ts +++ b/webui/e2e/mocks/metrics.ts @@ -1,6 +1,20 @@ const app_start = new Date() app_start.setHours(new Date().getHours() - 2) +import { app } from 'mokapi' + +export default function () { + app.http() + .operation('metrics', (req, res) => { + let names = req.query['names'] + res.data = !names ? metrics : metrics.filter(x => names.includes(x.name)) + }) + .operation('metricsType', (req, res) => { + let type = req.path['type'] + res.data = metrics.filter(x => x.name.startsWith(type + '_')) + }) +} + export let metrics = [ { name: 'app_start_timestamp', @@ -38,6 +52,7 @@ export let metrics = [ name: 'http_request_timestamp{service="Swagger Petstore",endpoint="/pet/findByStatus",method="POST"}', value: 1652237690 }, + // Kafka { name: 'kafka_messages_total{service="Kafka World",topic="mokapi.shop.products"}', value: 10 @@ -82,6 +97,15 @@ export let metrics = [ name: 'kafka_message_timestamp{service="Kafka World",topic="mokapi.shop.avro"}', value: 1652035690 }, + { + name: 'kafka_messages_total{service="Kafka World",topic="orders.{region}.{eventType}",instance="orders.us.created"}', + value: 1 + }, + { + name: 'kafka_message_timestamp{service="Kafka World",topic="orders.{region}.{eventType}",instance="orders.us.created"}', + value: 1676278165 + }, + // Mails { name: 'mail_mails_total{service="Mail Testserver"}', value: 3 @@ -108,11 +132,11 @@ export let metrics = [ value: 1672182690 }, { - name: 'mqtt_messages_total{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data"}', + name: 'mqtt_messages_total{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data",instance="sensors/123/data"}', value: 1 }, { - name: 'mqtt_message_timestamp{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data"}', + name: 'mqtt_message_timestamp{service="MQTT Temperature Sensor API",topic="sensors/{sensorId}/data",instance="sensors/123/data"}', value: 1771058965 }, // websocket @@ -125,16 +149,16 @@ export let metrics = [ value: 1672182690 }, { - name: 'websocket_messages_total{service="WebSocket Chat API",channel="/chats/{chatId}"}', + name: 'websocket_messages_total{service="WebSocket Chat API",channel="/chats/{chatId}",instance="/chats/1234"}', value: 2 }, { - name: 'websocket_message_timestamp{service="WebSocket Chat API",channel="/chats/{chatId}"}', + name: 'websocket_message_timestamp{service="WebSocket Chat API",channel="/chats/{chatId}",instance="/chats/1234"}', value: 1771058323 }, ] -export function sum(name, ...labels){ +export function sum(name: string, ...labels: Label[]){ if (!metrics){ return 0 } @@ -152,7 +176,7 @@ export function sum(name, ...labels){ return sum } -export function max(name, ...labels) { +export function max(name: string, ...labels: Label[]) { if (!metrics){ return 0 } @@ -173,7 +197,7 @@ export function max(name, ...labels) { return max } -export function filter(name, ...labels) { +export function filter(name: string, ...labels: Label[]) { const result = []; if (!metrics){ return []; @@ -191,7 +215,7 @@ export function filter(name, ...labels) { return result; } -export function parseLabels(metric) { +export function parseLabels(metric: { name: string }) { const match = metric.name.match(/\{(.+)}/); if (!match || match.length < 2 || !match[1]) { return {}; @@ -205,7 +229,7 @@ export function parseLabels(metric) { ); } -function matchLabels(metric, labels){ +function matchLabels(metric: { name: string }, labels: Label[]){ for (var label of labels){ const s = `${label.name}="${label.value}"` if (!metric.name.includes(s)){ diff --git a/webui/e2e/mocks/mqtt.js b/webui/e2e/mocks/mqtt.ts similarity index 81% rename from webui/e2e/mocks/mqtt.js rename to webui/e2e/mocks/mqtt.ts index 4826f8e72..e3d2035a4 100644 --- a/webui/e2e/mocks/mqtt.js +++ b/webui/e2e/mocks/mqtt.ts @@ -1,4 +1,4 @@ -import { metrics } from 'metrics.ts'; +import { sum, max, metrics } from './metrics'; import { on } from 'mokapi'; const Measurement = { @@ -75,12 +75,13 @@ export let clusters = [ contentType: 'application/json', }, }, + parameters: ['sensorId'], instances: [ { name: 'sensors/123/data', parameters: { sensorId: '123' - } + }, } ] }, @@ -106,15 +107,24 @@ export default async function () { topics: clusters[0].topics.map(x => { return { name: x.name, - ...x.title ? { title: x.title } : {}, - ...x.summary ? { summary: x.summary } : {}, - ...x.description ? { description: x.description } : {}, + ...('title' in x ? { title: x.title } : {}), + ...('summary' in x ? { summary: x.summary } : {}), + ...('description' in x ? { description: x.description } : {}), messages: x.messages, - ...x.instances ? { instances: x.instances } : {}, + ...('parameters' in x ? { parameters: x.parameters} : {}), + ...('instances' in x ? { instances: x.instances?.map((i: any) => ( + { + ...i, + metrics: { + messages_total: sum('mqtt_messages_total', { name: 'service', value: clusters[0].name }, { name: 'topic', value: x.name }, { name: 'instance', value: i.name }), + message_timestamp: max('mqtt_message_timestamp', { name: 'service', value: clusters[0].name }, { name: 'topic', value: x.name }, { name: 'instance', value: i.name }) + }, + } + )) } : {}), metrics: { - mqtt_messages_total: metrics.find(m => m.name === `mqtt_messages_total{service="${clusters[0].name}",topic="${x.name}"}`).value, - mqtt_message_timestamp: metrics.find(m => m.name === `mqtt_message_timestamp{service="${clusters[0].name}",topic="${x.name}"}`).value - } + messages_total: sum('mqtt_messages_total', { name: 'service', value: clusters[0].name }, { name: 'topic', value: x.name }), + message_timestamp: max('mqtt_message_timestamp', { name: 'service', value: clusters[0].name }, { name: 'topic', value: x.name }) + }, } }), clients: clusters[0].clients @@ -155,6 +165,7 @@ export let events = [ name: 'MQTT Temperature Sensor API', type: 'message', topic: 'sensors/{sensorId}/data', + instance: 'sensors/123/data', sensorId: '123', clientId: 'mqtt-client-2', }, diff --git a/webui/e2e/mocks/tsconfig.json b/webui/e2e/mocks/tsconfig.json new file mode 100644 index 000000000..2d8745e61 --- /dev/null +++ b/webui/e2e/mocks/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "lib": ["es2020", "dom"], + "paths": { + "mokapi": ["../../../npm/go-mokapi/types/index"], + "mokapi/*": ["../../../npm/go-mokapi/types/*"] + }, + "types": ["../../../npm/go-mokapi/types/global"] + } +} \ No newline at end of file diff --git a/webui/e2e/mocks/types.d.ts b/webui/e2e/mocks/types.d.ts new file mode 100644 index 000000000..439c31afb --- /dev/null +++ b/webui/e2e/mocks/types.d.ts @@ -0,0 +1,36 @@ +interface Config { + name: string; + description?: string; + version?: string; + contact?: Contact; + status?: string; +} + +interface Info { + type: string; + name: string; + description?: string; + version?: string; + contact?: Contact; + status?: string; +} + +interface Contact { + name?: string; + email?: string; + url?: string; +} + +interface Label { + name: string; + value: string; +} + +type Traits = Record; + +interface ServiceEvent { + id: string; + traits: Traits; + time: string; + data: any +} \ No newline at end of file diff --git a/webui/e2e/mocks/websocket.js b/webui/e2e/mocks/websocket.ts similarity index 75% rename from webui/e2e/mocks/websocket.js rename to webui/e2e/mocks/websocket.ts index f4ede3e06..aee99e544 100644 --- a/webui/e2e/mocks/websocket.js +++ b/webui/e2e/mocks/websocket.ts @@ -1,4 +1,4 @@ -import { metrics } from 'metrics.ts'; +import { sum, max, metrics } from './metrics'; import { on } from 'mokapi'; const ChatMessage = { @@ -71,14 +71,15 @@ export let services = [ contentType: 'application/json', } }, + parameters: ['chatId'], instances: [ { - name: 'chats/1234', + name: '/chats/1234', parameters: { chatId: '1234' - } + }, } - ] + ], }, ], metrics: metrics.filter((x) => x.name.includes('websocket')), @@ -96,21 +97,26 @@ export default async function () { version: services[0].version, contact: services[0].contact, servers: services[0].servers, - channels: services[0].channels.map((x) => { + channels: services[0].channels.map((x: any) => { return { name: x.name, - ...(x.title ? { title: x.title } : {}), - ...(x.summary ? { summary: x.summary } : {}), - ...(x.description ? { description: x.description } : {}), + ...('title' in x ? { title: x.title } : {}), + ...('summary' in x ? { summary: x.summary } : {}), + ...('description' in x ? { description: x.description } : {}), messages: x.messages, - ...x.instances ? { instances: x.instances } : {}, + ...('parameters' in x ? { parameters: x.parameters} : {}), + ...('instances' in x ? { instances: x.instances.map((i: any) => ( + { + ...i, + metrics: { + messages_total: sum('websocket_messages_total', { name: 'service', value: services[0].name }, { name: 'channel', value: x.name }, { name: 'instance', value: i.name }), + message_timestamp: max('websocket_message_timestamp', { name: 'service', value: services[0].name }, { name: 'channel', value: x.name }, { name: 'instance', value: i.name }) + }, + } + )) } : {}), metrics: { - websocket_messages_total: metrics.find( - (m) => m.name === `websocket_messages_total{service="${services[0].name}",channel="${x.name}"}` - )?.value, - websocket_message_timestamp: metrics.find( - (m) => m.name === `websocket_message_timestamp{service="${services[0].name}",channel="${x.name}"}` - )?.value, + messages_total: sum('websocket_messages_total', { name: 'service', value: services[0].name }, { name: 'channel', value: x.name }), + message_timestamp: max('websocket_message_timestamp', { name: 'service', value: services[0].name }, { name: 'channel', value: x.name }) }, }; }), @@ -169,6 +175,7 @@ export let events = [ namespace: 'websocket', name: 'WebSocket Chat API', channel: '/chats/{chatId}', + instance: '/chats/1234', type: 'message', clientId: clients[1].id, }, @@ -201,6 +208,7 @@ export let events = [ namespace: 'websocket', name: 'WebSocket Chat API', channel: '/chats/{chatId}', + instance: '/chats/1234', type: 'message', clientId: clients[1].id, }, @@ -230,4 +238,28 @@ export let events = [ ] }, }, + { + id: '9ab700fa-55b6-4c32-9c53-8cd3926dacc4', + traits: { + namespace: 'websocket', + name: 'WebSocket Chat API', + channel: '/chats/{chatId}', + instance: '/chats/1234', + type: 'connection', + clientId: clients[0].id, + }, + time: '2026-02-13T09:50:26.100000+01:00', + data: { + type: 'connect', + channel: '/chats/general', + client: { + id: clients[1].id, + query: {}, + header: {}, + address: clients[0].address, + server: clients[0].serverAddress + }, + api: 'WebSocket Chat API', + }, + }, ]; \ No newline at end of file diff --git a/webui/e2e/mocks/websocket.yml b/webui/e2e/mocks/websocket.yml index d8211ac4e..ebb529ec9 100644 --- a/webui/e2e/mocks/websocket.yml +++ b/webui/e2e/mocks/websocket.yml @@ -84,10 +84,17 @@ components: properties: name: type: string - Parameters: + parameters: type: object additionalProperties: type: string + metrics: + type: object + properties: + messages_total: + type: integer + message_timestamp: + type: integer WebsocketMessageConfig: type: object properties: diff --git a/webui/e2e/tests/components/dashboard.ts b/webui/e2e/tests/components/dashboard.ts index d2dd6a066..757cbd8f7 100644 --- a/webui/e2e/tests/components/dashboard.ts +++ b/webui/e2e/tests/components/dashboard.ts @@ -15,6 +15,7 @@ export function useDashboardTabs(page: Page) { kafka: dashboard.getByRole('link', { name: 'Kafka', exact: true }), mail: dashboard.getByRole('link', { name: 'Mail', exact: true }), ldap: dashboard.getByRole('link', { name: 'LDAP', exact: true }), + jobs: dashboard.getByRole('link', { name: 'Jobs', exact: true }), configs: dashboard.getByRole('link', { name: 'Configs', exact: true }), } } \ No newline at end of file diff --git a/webui/e2e/tests/dashboard-demo/dashboard.spec.ts b/webui/e2e/tests/dashboard-demo/dashboard.spec.ts index db3da1ced..a1afdb9dd 100644 --- a/webui/e2e/tests/dashboard-demo/dashboard.spec.ts +++ b/webui/e2e/tests/dashboard-demo/dashboard.spec.ts @@ -36,7 +36,7 @@ test('Visit Dashboard Demo Overview', async ({ page, baseURL }) => { const row = table.getByRole('row').filter({ hasText: 'Kafka Order Service API' }) await expect(row).toBeVisible() - await expect(await getCellByColumnName(table, 'Messages', row)).toHaveText('2') + await expect(await getCellByColumnName(table, 'Messages', row)).toHaveText('3') }) await test.step('Verify Mail Server is in the Mail table', async() => { diff --git a/webui/e2e/tests/dashboard-demo/kafka.spec.ts b/webui/e2e/tests/dashboard-demo/kafka.spec.ts index ddc56233e..6bc92eab6 100644 --- a/webui/e2e/tests/dashboard-demo/kafka.spec.ts +++ b/webui/e2e/tests/dashboard-demo/kafka.spec.ts @@ -13,6 +13,8 @@ test('Visit Kafka Order Service', async ({ page, baseURL }) => { dashboard = '/dashboard-demo' await page.goto('/dashboard-demo') } + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) + await page.getByText('Kafka Order Service API').click(); @@ -42,23 +44,28 @@ test('Visit Kafka Order Service', async ({ page, baseURL }) => { await page.getByRole('tab', { name: 'Topics' }).click(); const table = page.getByRole('table', { name: 'Topics' }); const rows = table.locator('tbody tr'); - await expect(rows).toHaveCount(2); + await expect(rows).toHaveCount(3); await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('order-topic'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText('The Kafka topic for order events.'); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toHaveText('-'); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('2'); - await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('user-events'); - await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText('The Kafka topic for user events.'); - await expect(await getCellByColumnName(table, 'Last Message', rows.nth(1))).toHaveText('-'); - await expect(await getCellByColumnName(table, 'Messages', rows.nth(1))).toHaveText('0'); + await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('shipments.{region}.updates'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText('Regional topics with shipment tracking events.'); + await expect(await getCellByColumnName(table, 'Last Message', rows.nth(1))).not.toHaveText('-'); + await expect(await getCellByColumnName(table, 'Messages', rows.nth(1))).toHaveText('1'); + + await expect(await getCellByColumnName(table, 'Name', rows.nth(2))).toHaveText('user-events'); + await expect(await getCellByColumnName(table, 'Summary', rows.nth(2))).toHaveText('The Kafka topic for user events.'); + await expect(await getCellByColumnName(table, 'Last Message', rows.nth(2))).toHaveText('-'); + await expect(await getCellByColumnName(table, 'Messages', rows.nth(2))).toHaveText('0'); await test.step('Verify filtering by tags', async () => { const tags = page.getByRole('group', { name: 'Filter topics by tags' }); await expect(tags.getByRole('checkbox', { name: 'All' })).toBeChecked(); await tags.getByRole('checkbox', { name: 'user' }).click(); - await expect(rows).toHaveCount(1); + await expect(rows).toHaveCount(2); }); @@ -134,19 +141,21 @@ test('Visit Kafka Order Service', async ({ page, baseURL }) => { }); + const messages = page.getByRole('table', { name: 'Recent Messages' }); + await test.step('Verify Recent Messages', async () => { await page.getByRole('tab', { name: 'Topics' }).click(); const region = page.getByRole('region', { name: 'Recent Messages' }); await expect(region).toBeVisible(); - const table = region.getByRole('table', { name: 'Recent Messages' }); - const rows = table.locator('tbody tr'); - await expect(rows).toHaveCount(2); - await expect(await getCellByColumnName(table, 'Key', rows.nth(0))).toHaveText('a914817b-c5f0-433e-8280-1cd2fe44234e'); - await expect(await getCellByColumnName(table, 'Value', rows.nth(0))).toContainText('{"orderId":"a914817b-c5f0-433e-8280-1cd2fe44234e","productId":"2a'); - await expect(await getCellByColumnName(table, 'Topic', rows.nth(0))).toHaveText('order-topic'); - await expect(await getCellByColumnName(table, 'Time', rows.nth(0))).not.toBeEmpty(); + + const rows = messages.locator('tbody tr'); + await expect(rows).toHaveCount(3); + await expect(await getCellByColumnName(messages, 'Key', rows.nth(1))).toHaveText('a914817b-c5f0-433e-8280-1cd2fe44234e'); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(1))).toContainText('{"orderId":"a914817b-c5f0-433e-8280-1cd2fe44234e","productId":"2a'); + await expect(await getCellByColumnName(messages, 'Topic', rows.nth(1))).toHaveText('order-topic'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(1))).not.toBeEmpty(); }) @@ -156,8 +165,6 @@ test('Visit Kafka Order Service', async ({ page, baseURL }) => { await page.getByRole('table', { name: 'Topics' }).getByText('order-topic').click(); const info = page.getByRole('region', { name: 'Info' }); await expect(info.getByLabel('Topic', { exact: true })).toHaveText('order-topic'); - await expect(info.getByLabel('Cluster')).toHaveText('Kafka Order Service API'); - await expect(info.getByLabel('Cluster')).toHaveAttribute('href'); await expect(info.getByLabel('Summary')).toHaveText('The Kafka topic for order events.'); await expect(info.getByLabel('Type of API')).toHaveText('Kafka'); @@ -262,8 +269,26 @@ test('Visit Kafka Order Service', async ({ page, baseURL }) => { await expect(value.getByLabel('Lines of Code').nth(0)).toHaveText('44 lines'); await expect(value.getByLabel('Size of Code').nth(0)).toHaveText('877 B'); await expect(value.getByRole('region', { name: 'Content' }).nth(0)).toContainText(`"type": "object", - "properties": {`); +"properties": {`); + + }); + + await test.step('Visit shipments.{region}.updates', async () => { + await breadcrumb.getByRole('link', { name: 'Kafka Order Service API' }).click(); + await page.getByRole('link', { name: 'shipments.{region}.updates' }).click(); + + const region = page.getByRole('region', { name: 'Info' }) + await expect(region.getByLabel('Topic Template')).toHaveText('shipments.{region}.updates'); + await expect(region.getByLabel('Title')).toHaveText('Shipment Updates per Region'); + await expect(region.getByLabel('Summary')).toHaveText('Regional topics with shipment tracking events.') + await expect(region.getByLabel('Description')).toContainText('Each fulfilment region publishes tracking') + await expect(region.getByLabel('Type of API')).toHaveText('Kafka'); + await expect(messages.locator('tbody tr')).toHaveCount(1); + + await page.getByRole('table', { name: 'Topics' }).getByRole('link', { name: 'shipments.eu.updates' }).click(); + await expect(region.getByLabel('Topic', { exact: true })).toHaveText('shipments.eu.updates'); + await expect(messages.locator('tbody tr')).toHaveCount(1); }); }); diff --git a/webui/e2e/tests/dashboard-demo/mail.spec.ts b/webui/e2e/tests/dashboard-demo/mail.spec.ts index f709a62b5..3de66350b 100644 --- a/webui/e2e/tests/dashboard-demo/mail.spec.ts +++ b/webui/e2e/tests/dashboard-demo/mail.spec.ts @@ -74,8 +74,6 @@ test('Visit Mail Server', async ({ page, baseURL }) => { await rows.nth(1).click(); await expect(page.getByLabel('Mailbox Name')).toHaveText('bob.miller@example.com'); - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(page.getByLabel('Username')).toHaveText('bob.miller'); await expect(page.getByLabel('Password')).toHaveText('mysecretpassword123'); @@ -117,8 +115,6 @@ test('Visit Mail Server', async ({ page, baseURL }) => { await page.getByRole('table', { name: 'Mailboxes' }).getByText('alice.johnson@example.com').click(); await expect(page.getByLabel('Mailbox Name')).toHaveText('alice.johnson@example.com'); - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Mail Server'); - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/mail/service/Mail%20Server'); await expect(page.getByLabel('Username')).toHaveText('alice.johnson'); await expect(page.getByLabel('Password')).toHaveText('anothersecretpassword456'); diff --git a/webui/e2e/tests/dashboard-demo/mqtt.spec.ts b/webui/e2e/tests/dashboard-demo/mqtt.spec.ts index 4386270fd..5a530a353 100644 --- a/webui/e2e/tests/dashboard-demo/mqtt.spec.ts +++ b/webui/e2e/tests/dashboard-demo/mqtt.spec.ts @@ -7,6 +7,7 @@ test('Visit MQTT overview', async ({ page, baseURL }) => { } else { await page.goto('/dashboard-demo') } + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) await test.step('Verify Dashboard', async () => { @@ -34,7 +35,7 @@ test('Visit MQTT overview', async ({ page, baseURL }) => { const rows = table.locator('tbody tr'); await expect(rows).toHaveCount(1); - await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('sensors/{sensorId}/data'); + await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('Template sensors/{sensorId}/data, 1 topic'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText(''); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('1'); @@ -43,12 +44,32 @@ test('Visit MQTT overview', async ({ page, baseURL }) => { const topics = page.getByRole('table', { name: 'Topics' }); await topics.getByText('sensors/{sensorId}/data').click(); - await expect(page.getByLabel('Topic', { exact: true })).toHaveText('sensors/{sensorId}/data'); + await expect(page.getByLabel('Topic Template', { exact: true })).toHaveText('sensors/{sensorId}/data'); await expect(page.getByLabel('Description')).not.toBeVisible(); await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + let rows = topics.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(topics, 'Topic', rows.nth(0))).toHaveText('sensors/12345/data'); + await expect(await getCellByColumnName(topics, 'Last Message', rows.nth(0))).not.toBeEmpty(); + await expect(await getCellByColumnName(topics, 'Messages', rows.nth(0))).toHaveText('1') + const messages = page.getByRole('table', { name: 'Messages' }); - const rows = messages.locator('tbody tr'); + + await test.step('Visit sensors/12345/data', async () => { + await rows.nth(0).click(); + + await expect(page.getByLabel('Topic', { exact: true })).toHaveText('sensors/12345/data'); + await expect(page.getByLabel('Description')).not.toBeVisible(); + await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + await expect(messages.locator('tbody tr')).toHaveCount(1); + + await breadcrumb.getByRole('link', { name: 'sensors/{sensorId}/data' }).click(); + + }); + + + rows = messages.locator('tbody tr'); await expect(rows).toHaveCount(1); await expect(await getCellByColumnName(messages, 'Topic', rows.nth(0))).toHaveText('sensors/12345/data'); await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toContainText('{"temp":24,"timestamp":"'); @@ -91,7 +112,7 @@ test('Visit MQTT overview', async ({ page, baseURL }) => { }) await meta.getByLabel('Topic').click(); - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await breadcrumb.getByRole('link', { name: 'Smart Home MQTT API' }).click(); }) }) diff --git a/webui/e2e/tests/dashboard-demo/petstore.spec.ts b/webui/e2e/tests/dashboard-demo/petstore.spec.ts index 3c48ad508..2fd28882b 100644 --- a/webui/e2e/tests/dashboard-demo/petstore.spec.ts +++ b/webui/e2e/tests/dashboard-demo/petstore.spec.ts @@ -124,8 +124,6 @@ test('Visit Petstore Demo', async ({ page, baseURL }) => { await page.getByText('/pet/{petId}', { exact: true }).click() await expect(page.getByLabel('Path', { exact: true })).toHaveText('/pet/{petId}') - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Swagger Petstore') - await expect(page.getByLabel('Service', { exact: true }).getByRole('link')).toHaveAttribute('href', dashboard + '/http/services/Swagger%20Petstore') const region = page.getByRole('region', { name: 'Methods' }); await expect(region).toBeVisible(); @@ -160,7 +158,6 @@ test('Visit Petstore Demo', async ({ page, baseURL }) => { await expect(page.getByLabel('Operation', { exact: true })).toHaveText('DELETE /pet/{petId}'); await expect(page.getByLabel('Operation ID')).toHaveText('deletePet'); - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Swagger Petstore'); await expect(page.getByLabel('Summary')).toHaveText('Deletes a pet'); await test.step('Verify request', async () => { diff --git a/webui/e2e/tests/dashboard-demo/websocket.spec.ts b/webui/e2e/tests/dashboard-demo/websocket.spec.ts index 265f4fa1d..53963f1be 100644 --- a/webui/e2e/tests/dashboard-demo/websocket.spec.ts +++ b/webui/e2e/tests/dashboard-demo/websocket.spec.ts @@ -7,6 +7,7 @@ test('Visit Websocket overview', async ({ page, baseURL }) => { } else { await page.goto('/dashboard-demo') } + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) await test.step('Verify Dashboard', async () => { @@ -30,29 +31,48 @@ test('Visit Websocket overview', async ({ page, baseURL }) => { await expect(region.getByLabel('Name')).toHaveText('WebSocket Chat API') await expect(region.getByLabel('Description')).toHaveText('A real-time Chat API based on WebSockets for bidirectional communication within chat rooms.') - const table = page.getByRole('table', { name: 'Channels' }); + const table = page.getByRole('table', { name: 'Paths' }); const rows = table.locator('tbody tr'); await expect(rows).toHaveCount(1); - await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('/chats/{roomId}'); + await expect(await getCellByColumnName(table, 'Path', rows.nth(0))).toHaveText('Template /chats/{roomId}, 1 path'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText(''); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('3'); await test.step('Visit /chats/{roomId}', async () => { - const topics = page.getByRole('table', { name: 'Channels' }); + const topics = page.getByRole('table', { name: 'Paths' }); await topics.getByText('/chats/{roomId}').click(); const info = page.getByRole('region', { name: 'Info' }) - await expect(info.getByLabel('Channel', { exact: true })).toHaveText('/chats/{roomId}'); + await expect(info.getByLabel('Path Template', { exact: true })).toHaveText('/chats/{roomId}'); await expect(info.getByLabel('Description')).toHaveText('The communication channel for a specific chat room.') await expect(info.getByLabel('Type of API')).toHaveText('Websocket'); + let rows = topics.locator('tbody tr'); + await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(topics, 'Path', rows.nth(0))).toHaveText('/chats/general'); + await expect(await getCellByColumnName(topics, 'Last Message', rows.nth(0))).not.toBeEmpty(); + await expect(await getCellByColumnName(topics, 'Messages', rows.nth(0))).toHaveText('3'); + const messages = page.getByRole('table', { name: 'Messages' }); - const rows = messages.locator('tbody tr'); + + await test.step('Visit /chats/general', async () => { + await rows.nth(0).click(); + + await expect(info.getByLabel('Path', { exact: true })).toHaveText('/chats/general'); + await expect(info.getByLabel('Description')).toHaveText('The communication channel for a specific chat room.') + await expect(info.getByLabel('Type of API')).toHaveText('Websocket'); + await expect(messages.locator('tbody tr')).toHaveCount(3); + + await breadcrumb.getByRole('link', { name: '/chats/{roomId}' }).click(); + + }); + + rows = messages.locator('tbody tr'); await expect(rows).toHaveCount(3); - await expect(await getCellByColumnName(messages, 'Channel', rows.nth(0))).toHaveText('/chats/general'); + await expect(await getCellByColumnName(messages, 'Path', rows.nth(0))).toHaveText('/chats/general'); await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toContainText('{"text":"Hello there!"'); await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); @@ -61,7 +81,7 @@ test('Visit Websocket overview', async ({ page, baseURL }) => { await rows.nth(2).click(); const meta = page.getByRole('region', { name: 'Meta' }) - await expect(meta.getByLabel('Channel')).toHaveText('/chats/general'); + await expect(meta.getByLabel('Path')).toHaveText('/chats/general'); await expect(meta.getByLabel('Time')).not.toBeEmpty(); await expect(meta.getByLabel('Client')).toContainText(':'); await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); @@ -90,8 +110,8 @@ test('Visit Websocket overview', async ({ page, baseURL }) => { }) - await meta.getByLabel('Channel').click(); - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await meta.getByLabel('Path').click(); + await breadcrumb.getByRole('link', { name: 'WebSocket Chat API' }).click(); }) }) diff --git a/webui/e2e/tests/dashboard/dashboard.spec.ts b/webui/e2e/tests/dashboard/dashboard.spec.ts index e2ef0e2cf..6f529545a 100644 --- a/webui/e2e/tests/dashboard/dashboard.spec.ts +++ b/webui/e2e/tests/dashboard/dashboard.spec.ts @@ -1,9 +1,6 @@ import { test, expect } from '../models/fixture-dashboard' -import dayjs from 'dayjs' -import relativeTime from 'dayjs/plugin/relativeTime' import { formatTimestamp } from '../helpers/format' -dayjs.extend(relativeTime) test.describe('Dashboard', () => { test.use({ colorScheme: 'dark' }) @@ -52,7 +49,7 @@ test.describe('Dashboard', () => { await test.step("metric Kafka Messages",async () => { const metricKafkaMessages = dashboard.kafka.metricKafkaMessages await expect(metricKafkaMessages.title).toHaveText('Kafka Messages') - await expect(metricKafkaMessages.value).toHaveText('11') + await expect(metricKafkaMessages.value).toHaveText('12') await expect(metricKafkaMessages.additional).not.toBeVisible() }) @@ -60,8 +57,8 @@ test.describe('Dashboard', () => { const kafkaCells = dashboard.kafka.serviceList.getByRole('cell') await expect(kafkaCells.nth(0)).toHaveText('Kafka World') await expect(kafkaCells.nth(1)).not.toHaveText('') - await expect(kafkaCells.nth(2)).toHaveText(formatTimestamp(1652135690)) - await expect(kafkaCells.nth(3)).toHaveText('11') + await expect(kafkaCells.nth(2)).toHaveText(formatTimestamp(1676278165)) + await expect(kafkaCells.nth(3)).toHaveText('12') }) await test.step("metric SMTP Mails",async () => { diff --git a/webui/e2e/tests/dashboard/http/books.spec.ts b/webui/e2e/tests/dashboard/http/books.spec.ts index 4a67e7ecb..aef221dc8 100644 --- a/webui/e2e/tests/dashboard/http/books.spec.ts +++ b/webui/e2e/tests/dashboard/http/books.spec.ts @@ -80,7 +80,6 @@ test.describe('Visit Books API', () => { await expect(page).toHaveURL(/Books%20API\/books/) await expect(page.getByLabel('Path', { exact: true })).toHaveText('/books'); - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Books API') await expect(page.getByLabel('Type of API')).toHaveText('HTTP') await test.step('Verify methods', async () => { @@ -106,7 +105,6 @@ test.describe('Visit Books API', () => { await expect(page.getByLabel('Operation', { exact: true })).toHaveText('POST /books'); await expect(page.getByLabel('Operation ID')).toHaveText('addBook'); - await expect(page.getByLabel('Service', { exact: true })).toHaveText('Books API'); await expect(page.getByLabel('Type of API')).toHaveText('HTTP'); await expect(page.getByLabel('Summary')).toHaveText('Add a new book'); await expect(page.getByLabel('Description', { exact: true })).not.toBeVisible(); diff --git a/webui/e2e/tests/dashboard/http/filter.spec.ts b/webui/e2e/tests/dashboard/http/filter.spec.ts new file mode 100644 index 000000000..baf5aa006 --- /dev/null +++ b/webui/e2e/tests/dashboard/http/filter.spec.ts @@ -0,0 +1,162 @@ +import { useDashboard } from '../../components/dashboard' +import { test, expect } from '../../models/fixture-dashboard' + +test.describe('Visit HTTP dashboard', () => { + test.use({ colorScheme: 'dark' }) + + test('Verifiy HTTP filters', async ({ page }) => { + const { tabs, open } = useDashboard(page) + await open() + await tabs.http.click() + + const table = page.getByRole('table', { name: 'Recent Requests' }); + let rows = table.locator('tbody tr'); + await expect(rows).toHaveCount(6) + + const filterButton = page.getByRole('button', { name: 'Filter' }) + const dialog = page.getByRole('dialog', { name: 'Filter' }) + const resetFilters = async() => { + await page.evaluate(() => { + localStorage.removeItem('http-requests-filter'); + }); + await page.reload() + } + + await test.step('Verify API names', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'API Name' }).check() + const names = dialog.getByRole('combobox', { name: 'Service' }) + const options = names.getByRole('option') + await expect(options).toHaveCount(2) + + await names.selectOption('Books API') + await expect(rows).toHaveCount(0) + + await names.selectOption('Swagger Petstore') + await expect(rows).toHaveCount(6) + + await dialog.getByRole('button', { name: 'Close' }).click() + }) + + await test.step('Verify HTTP method', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'Method' }).check() + await dialog.getByRole('combobox', { name: 'Method' }).selectOption('POST') + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + + await expect(rows).toHaveCount(2) + }) + + await test.step('Verify URL', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'URL' }).check() + await dialog.getByRole('textbox', { name: 'URL' }).fill('[sS]tatus') + + await expect(rows).toHaveCount(2) + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + + await expect(rows).toHaveCount(2) + }) + + await test.step('Verify request header', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'Request Header' }).check() + await dialog.getByRole('textbox', { name: 'Request Header Name 1' }).fill('LongHeader') + await dialog.getByRole('textbox', { name: 'Request Header Value 1' }).fill('Lorem') + + await expect(rows).toHaveCount(1) + + await dialog.getByRole('checkbox', { name: 'Request Header' }).check() + await dialog.getByRole('textbox', { name: 'Request Header Name 1' }).fill('debug') + await dialog.getByRole('textbox', { name: 'Request Header Value 1' }).fill('bar') + + await dialog.getByRole('textbox', { name: 'Request Header Name 1' }).fill('noSpec') + await dialog.getByRole('textbox', { name: 'Request Header Value 1' }).fill('bar') + + await expect(rows).toHaveCount(1) + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + }) + + await test.step('Verify response status code', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'Status Code' }).check() + await dialog.getByRole('textbox', { name: 'Status Code' }).fill('200') + await expect(rows).toHaveCount(2) + + await dialog.getByRole('textbox', { name: 'Status Code' }).fill('200-400') + await expect(rows).toHaveCount(4) + + await dialog.getByRole('textbox', { name: 'Status Code' }).fill('2xx') + await expect(rows).toHaveCount(4) + + await dialog.getByRole('textbox', { name: 'Status Code' }).fill('!2xx') + await expect(rows).toHaveCount(2) + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + }) + + await test.step('Verify response header', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'Response Header' }).check() + + await dialog.getByRole('textbox', { name: 'Response Header Name 1' }).fill('Content-Type') + await dialog.getByRole('textbox', { name: 'Response Header Value 1' }).fill('application/json') + await expect(rows).toHaveCount(6) + + await resetFilters() + await filterButton.click() + await dialog.getByRole('checkbox', { name: 'Response Header' }).check() + + await dialog.getByRole('textbox', { name: 'Response Header Name 1' }).fill('Set-Cookie') + await expect(rows).toHaveCount(2) + + await dialog.getByRole('textbox', { name: 'Response Header Value 1' }).fill('38afes6v3') + await expect(rows).toHaveCount(1) + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + }) + + await test.step('Verify client IP', async() => { + await resetFilters() + + await filterButton.click() + + await dialog.getByRole('checkbox', { name: 'Client IP' }).check() + + await dialog.getByRole('textbox', { name: 'Client IP' }).fill('127.0.0.1') + await expect(rows).toHaveCount(4) + + await dialog.getByRole('textbox', { name: 'Client IP' }).fill('192.0.1.1/24') + await expect(rows).toHaveCount(2) + + await dialog.getByRole('button', { name: 'Close' }).click() + await expect(filterButton).toContainText('1') + }) + }) +}) \ No newline at end of file diff --git a/webui/e2e/tests/dashboard/jobs.spec.ts b/webui/e2e/tests/dashboard/jobs.spec.ts new file mode 100644 index 000000000..e9ef42461 --- /dev/null +++ b/webui/e2e/tests/dashboard/jobs.spec.ts @@ -0,0 +1,16 @@ +import { useDashboard } from '../components/dashboard' +import { test, expect } from '../models/fixture-dashboard' + +test.describe('Visit Jobs dashboard', () => { + test.use({ colorScheme: 'dark' }) + + test('Verifiy Job table', async ({ page }) => { + const { tabs, open } = useDashboard(page) + await open() + await tabs.jobs.click() + + const table = page.getByRole('table', { name: 'Recent Jobs' }); + let rows = table.locator('> tbody > tr:not(.collapse-row)'); + await expect(rows).toHaveCount(3) + }) +}) \ No newline at end of file diff --git a/webui/e2e/tests/dashboard/kafka/cluster.spec.ts b/webui/e2e/tests/dashboard/kafka/cluster.spec.ts index 0fb348c13..07dac295e 100644 --- a/webui/e2e/tests/dashboard/kafka/cluster.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/cluster.spec.ts @@ -90,8 +90,8 @@ test('Visit Kafka cluster config file', async ({ page, context }) => { const { test: testSourceView } = useSourceView(dashboard.getByRole('region', { name: 'Content' })) await testSourceView({ - lines: '112 lines', - size: '2.85 kB', + lines: '136 lines', + size: '3.31 kB', content: /"name": "Kafka World"/, filename: 'asyncapi.json', clipboard: '"name": "Kafka World"' diff --git a/webui/e2e/tests/dashboard/kafka/cluster.ts b/webui/e2e/tests/dashboard/kafka/cluster.ts index c39a6d822..53bcc2140 100644 --- a/webui/e2e/tests/dashboard/kafka/cluster.ts +++ b/webui/e2e/tests/dashboard/kafka/cluster.ts @@ -7,8 +7,8 @@ export const cluster = { name: 'mokapi', }, description: 'To ours significant why upon tomorrow her faithful many motionless.', - lastMessage: formatTimestamp(1652135690), - messages: '11', + lastMessage: formatTimestamp(1676278165), + messages: '12', brokers: [{ name: 'Broker', url: 'localhost:9092', diff --git a/webui/e2e/tests/dashboard/kafka/orders.us.created.spec.ts b/webui/e2e/tests/dashboard/kafka/orders.us.created.spec.ts new file mode 100644 index 000000000..b2b0f99d4 --- /dev/null +++ b/webui/e2e/tests/dashboard/kafka/orders.us.created.spec.ts @@ -0,0 +1,70 @@ +import { cluster } from './cluster' +import { useDashboard } from '../../components/dashboard' +import { test, expect } from '../../models/fixture-dashboard' +import { getCellByColumnName } from '../../helpers/table' +import { formatDateTime } from '../../helpers/format' + +test('Visit Kafka topic orders.{region}.{eventType}', async ({ page, context }) => { + await context.grantPermissions(["clipboard-read", "clipboard-write"]) + + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) + const messages = page.getByRole('table', { name: 'Recent Messages' }) + const tabList = page.getByRole('region', { name: 'Topic Data' }).getByRole('tablist') + + await test.step('Visit topic "orders.{region}.{eventType}"', async () => { + const { tabs, open } = useDashboard(page) + await open() + await tabs.kafka.click() + + await page.getByRole('table', { name: 'Clusters' }).getByText(cluster.name).click() + await expect(page.getByRole('region', { name: "Info" })).toBeVisible() + + await page.getByRole('table', { name: 'Topics' }).getByText('orders.{region}.{eventType}').click() + + const info = page.getByRole('region', { name: "Info" }) + await expect(info).toBeVisible() + await expect(info.getByLabel('Topic Template')).toHaveText('orders.{region}.{eventType}') + await expect(info.getByLabel('Type of API')).toHaveText('Kafka') + + await expect(tabList.getByRole('tab')).toHaveText(['Messages', 'Configs']) + + const instances = page.getByRole('table', { name: 'Topics' }) + let rows = instances.locator('tbody tr') + await expect(rows).toHaveCount(1) + await expect(await getCellByColumnName(instances, 'Topic', rows.nth(0))).toHaveText('orders.us.created'); + await expect(await getCellByColumnName(instances, 'Last Message', rows.nth(0))).toHaveText(formatDateTime('2023-02-13T09:49:25.482366+01:00')); + await expect(await getCellByColumnName(instances, 'Messages', rows.nth(0))).toHaveText('1'); + + await test.step('Visit topic "orders.us.created"', async () => { + await rows.nth(0).click() + + await expect(info.getByLabel('Topic')).toHaveText('orders.us.created') + await expect(tabList.getByRole('tab')).toHaveText(['Messages', 'Partitions', 'Groups', 'Configs']) + + await expect(messages.locator('thead th')).toHaveText(['Key', 'Value', 'Time']) + rows = messages.locator('tbody tr') + await expect(rows).toHaveCount(1) + await expect(await getCellByColumnName(messages, 'Key', rows.nth(0))).toHaveText('a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d'); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toContainText('"orderId":"a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d"'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).toHaveText(formatDateTime('2023-02-13T09:49:25.482366+01:00')); + + await (breadcrumb.getByRole('link', { name: 'orders.{region}.{eventType}' })).click() + }); + + await expect(messages.locator('thead th')).toHaveText(['Topic','Key', 'Value', 'Time']) + rows = messages.locator('tbody tr') + await expect(rows).toHaveCount(1) + await expect(await getCellByColumnName(messages, 'Topic', rows.nth(0))).toHaveText('orders.us.created'); + await expect(await getCellByColumnName(messages, 'Key', rows.nth(0))).toHaveText('a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d'); + await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toContainText('"orderId":"a1b2c3d4-e5f6-7a8b-9c0d-1e2f3a4b5c6d"'); + await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).toHaveText(formatDateTime('2023-02-13T09:49:25.482366+01:00')); + + await (breadcrumb.getByRole('link', { name: 'Kafka World' })).click() + }) + + await expect(page.getByRole('region', { name: 'Info' }).getByLabel('Name')).toHaveText('Kafka World') + + await test.step('Check info section"', async () => { + + }) +}) diff --git a/webui/e2e/tests/dashboard/kafka/overview.spec.ts b/webui/e2e/tests/dashboard/kafka/overview.spec.ts index 85e51bfbe..1cc02c7eb 100644 --- a/webui/e2e/tests/dashboard/kafka/overview.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/overview.spec.ts @@ -9,7 +9,7 @@ test('Visit Kafka overview', async ({ page }) => { await tabs.kafka.click() await test.step('Check messages metric', async () => { - await expect(page.getByRole('status', { name: 'Kafka Messages' })).toHaveText('11') + await expect(page.getByRole('status', { name: 'Kafka Messages' })).toHaveText('12') }) await test.step('Check clusters', async () => { diff --git a/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts b/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts index b75db8875..132f93394 100644 --- a/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/topic.order.spec.ts @@ -9,6 +9,7 @@ const ace = { edit: (id: any) => { return { setValue: (s: string) => {} } } } test('Visit Kafka topic mokapi.shop.products', async ({ page, context }) => { await context.grantPermissions(["clipboard-read", "clipboard-write"]) + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) const topic = cluster.topics[0] await test.step('Browse to topic "mokapi.shop.products"', async () => { @@ -26,7 +27,6 @@ test('Visit Kafka topic mokapi.shop.products', async ({ page, context }) => { const info = page.getByRole('region', { name: "Info" }) await expect(info).toBeVisible() await expect(info.getByLabel('Topic')).toHaveText(topic.name) - await expect(info.getByLabel('Cluster')).toHaveText(cluster.name) await expect(info.getByLabel('Type of API')).toHaveText('Kafka') if (topic.title) { await expect(info.getByLabel('Title')).toHaveText(topic.title) @@ -42,6 +42,7 @@ test('Visit Kafka topic mokapi.shop.products', async ({ page, context }) => { await useKafkaMessages(page).test(page.getByRole('table', { name: 'Recent Messages' }), false) const tabList = page.getByRole('region', { name: 'Topic Data' }).getByRole('tablist') + await test.step('Check partition"', async () => { await tabList.getByRole('tab', { name: 'Partitions' }).click() const table = page.getByRole('tabpanel', { name: 'Partitions' }).getByRole('table', { name: 'Topic Partitions' }) @@ -155,7 +156,7 @@ test('Visit Kafka topic mokapi.shop.products', async ({ page, context }) => { }) await test.step('Go back to cluster view', async () => { - await page.getByRole('link', { name: 'cluster' }).click() + await breadcrumb.getByRole('link', { name: 'Kafka World' }).click() const region = page.getByRole('region', { name: 'Info' }); await expect(region.getByLabel('name')).toHaveText(cluster.name) }) diff --git a/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts b/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts index 3c0f2e2ec..0ba1303a4 100644 --- a/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts +++ b/webui/e2e/tests/dashboard/kafka/topic.userSignedUp.spec.ts @@ -6,6 +6,7 @@ import { useSourceView } from '../../components/source' test('Visit Kafka topic mokapi.shop.userSignedUp', async ({ page, context }) => { await context.grantPermissions(["clipboard-read", "clipboard-write"]) + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) const topic = cluster.topics[1] await test.step('Browse to topic "mokapi.shop.userSignedUp"', async () => { @@ -23,7 +24,6 @@ test('Visit Kafka topic mokapi.shop.userSignedUp', async ({ page, context }) => const info = page.getByRole('region', { name: "Info" }) await expect(info).toBeVisible() await expect(info.getByLabel('Topic')).toHaveText(topic.name) - await expect(info.getByLabel('Cluster')).toHaveText(cluster.name) await expect(info.getByLabel('Type of API')).toHaveText('Kafka') await expect(info.getByLabel('Summary')).toHaveText(topic.summary) }) @@ -142,7 +142,7 @@ test('Visit Kafka topic mokapi.shop.userSignedUp', async ({ page, context }) => }) await test.step('Go back to cluster view', async () => { - await page.getByRole('link', { name: 'cluster' }).click() + await breadcrumb.getByRole('link', { name: 'Kafka World' }).click() const region = page.getByRole('region', { name: 'Info' }); await expect(region.getByLabel('name')).toHaveText(cluster.name) }) diff --git a/webui/e2e/tests/dashboard/mail/testserver.spec.ts b/webui/e2e/tests/dashboard/mail/testserver.spec.ts index dfef572bb..172ee890a 100644 --- a/webui/e2e/tests/dashboard/mail/testserver.spec.ts +++ b/webui/e2e/tests/dashboard/mail/testserver.spec.ts @@ -4,6 +4,7 @@ import { test, expect } from '../../models/fixture-dashboard' import { formatDateTime } from "../../helpers/format" test('Visit Mail Testserver', async ({ page }) => { + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) await test.step('Browse to Mail Testserver', async () => { const { tabs, open } = useDashboard(page) @@ -78,7 +79,7 @@ test('Visit Mail Testserver', async ({ page }) => { row = mails.getRow(1) await expect(row.getCellByName('Subject')).toHaveText('A test mail') - await page.getByLabel('Service', { exact: true }).getByRole('link').click() + await breadcrumb.getByRole('link', { name: 'Mail Testserver' }).click() }) await test.step('Check rules', async () => { diff --git a/webui/e2e/tests/dashboard/mqtt/overview.spec.ts b/webui/e2e/tests/dashboard/mqtt/overview.spec.ts index e64b622ec..509e8a1a1 100644 --- a/webui/e2e/tests/dashboard/mqtt/overview.spec.ts +++ b/webui/e2e/tests/dashboard/mqtt/overview.spec.ts @@ -3,6 +3,7 @@ import { getCellByColumnName } from '../../helpers/table' test('Visit MQTT overview', async ({ page }) => { await page.goto('/dashboard') + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) await test.step('Verify Dashboard', async () => { @@ -36,7 +37,7 @@ test('Visit MQTT overview', async ({ page }) => { await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('1'); - await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('sensors/{sensorId}/data'); + await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('Template sensors/{sensorId}/data, 1 topic'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText(''); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(1))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(1))).toHaveText('1'); @@ -92,7 +93,7 @@ test('Visit MQTT overview', async ({ page }) => { }) await meta.getByLabel('Topic').click(); - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await breadcrumb.getByRole('link', { name: 'MQTT Temperature Sensor API' }).click() }) }) @@ -101,12 +102,18 @@ test('Visit MQTT overview', async ({ page }) => { const topics = page.getByRole('table', { name: 'Topics' }); await topics.getByText('sensors/{sensorId}/data').click(); - await expect(page.getByLabel('Topic', { exact: true })).toHaveText('sensors/{sensorId}/data'); + await expect(page.getByLabel('Topic Template', { exact: true })).toHaveText('sensors/{sensorId}/data'); await expect(page.getByLabel('Title')).not.toBeVisible(); await expect(page.getByLabel('Summary')).not.toBeVisible(); await expect(page.getByLabel('Description')).not.toBeVisible(); await expect(page.getByLabel('Type of API')).toHaveText('MQTT'); + const instances = page.getByRole('table', { name: 'Topics' }) + const instance = instances.locator('tbody tr').nth(0); + await expect(await getCellByColumnName(instances, 'Topic', instance)).toContainText('sensors/123/data') + await expect(await getCellByColumnName(instances, 'Last Message', instance)).not.toHaveText('-') + await expect(await getCellByColumnName(instances, 'Messages', instance)).toContainText('1') + const messages = page.getByRole('table', { name: 'Messages' }); const rows = messages.locator('tbody tr'); await expect(rows).toHaveCount(1); @@ -114,6 +121,16 @@ test('Visit MQTT overview', async ({ page }) => { await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toHaveText('{"temp":33,"timestamp":"2026-02-14T09:49:25.482366+01:00"}'); await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); + await test.step('Visit instance', async() => { + await instance.click() + + const meta = page.getByRole('region', { name: 'Info' }) + await expect(meta.getByLabel('Topic')).toHaveText('sensors/123/data'); + + await expect(breadcrumb).toContainText('sensors/{sensorId}/data') + await expect(breadcrumb).toContainText('sensors/123/data') + }) + await test.step('Visit message', async () => { await rows.nth(0).click(); @@ -151,7 +168,7 @@ test('Visit MQTT overview', async ({ page }) => { }) await meta.getByLabel('Topic').click(); - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await breadcrumb.getByRole('link', { name: 'MQTT Temperature Sensor API' }).click() }) }) diff --git a/webui/e2e/tests/dashboard/websocket/overview.spec.ts b/webui/e2e/tests/dashboard/websocket/overview.spec.ts index cc21392a9..8e9183237 100644 --- a/webui/e2e/tests/dashboard/websocket/overview.spec.ts +++ b/webui/e2e/tests/dashboard/websocket/overview.spec.ts @@ -3,6 +3,7 @@ import { getCellByColumnName, getCellByName } from '../../helpers/table' test('Visit Websocket overview', async ({ page }) => { await page.goto('/dashboard') + const breadcrumb = page.getByRole('navigation', { name: 'Breadcrumb' }) await test.step('Verify Dashboard', async () => { @@ -28,24 +29,24 @@ test('Visit Websocket overview', async ({ page }) => { await expect(region.getByLabel('Name')).toHaveText('WebSocket Chat API') await expect(region.getByLabel('Description')).toHaveText('A simple chat app mocked over WebSocket. Clients open a single connection to /chat and exchange text frames in real time.') - const table = page.getByRole('table', { name: 'Channels' }); + const table = page.getByRole('table', { name: 'Paths' }); const rows = table.locator('tbody tr'); await expect(rows).toHaveCount(2); - await expect(await getCellByColumnName(table, 'Name', rows.nth(0))).toHaveText('/chat'); + await expect(await getCellByColumnName(table, 'Path', rows.nth(0))).toHaveText('/chat'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(0))).toHaveText('Single WS endpoint carrying all chat messages'); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(0))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(0))).toHaveText('2'); - await expect(await getCellByColumnName(table, 'Name', rows.nth(1))).toHaveText('/chats/{chatId}'); + await expect(await getCellByColumnName(table, 'Path', rows.nth(1))).toHaveText('Template /chats/{chatId}, 1 path'); await expect(await getCellByColumnName(table, 'Summary', rows.nth(1))).toHaveText(''); await expect(await getCellByColumnName(table, 'Last Message', rows.nth(1))).not.toBeEmpty(); await expect(await getCellByColumnName(table, 'Messages', rows.nth(1))).toHaveText('2'); await test.step('Visit /chat', async () => { - const topics = page.getByRole('table', { name: 'Channels' }); + const topics = page.getByRole('table', { name: 'Paths' }); await topics.getByText('/chat', { exact: true }).click(); - await expect(page.getByLabel('Channel', { exact: true })).toHaveText('/chat'); + await expect(page.getByLabel('Path', { exact: true })).toHaveText('/chat'); await expect(page.getByLabel('Summary')).toHaveText('Single WS endpoint carrying all chat messages'); await expect(page.getByLabel('Type of API')).toHaveText('Websocket'); @@ -60,7 +61,7 @@ test('Visit Websocket overview', async ({ page }) => { await rows.nth(0).click(); const meta = page.getByRole('region', { name: 'Meta' }) - await expect(meta.getByLabel('Channel')).toHaveText('/chat'); + await expect(meta.getByLabel('Path')).toHaveText('/chat'); await expect(meta.getByLabel('Time')).not.toBeEmpty(); await expect(meta.getByLabel('Client')).toHaveText('127.0.0.1:53211'); await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); @@ -81,51 +82,76 @@ test('Visit Websocket overview', async ({ page }) => { await expect(info.getByLabel('Server')).toHaveText('localhost:8080'); await expect(info.getByLabel('Type of API')).toHaveText('Websocket'); - const messages = page.getByRole('table', { name: 'Messages' }); - const rows = messages.locator('tbody tr'); + const tblMessages = page.getByRole('table', { name: 'Messages' }); + const messages = tblMessages.locator('tbody tr'); + await expect(messages).toHaveCount(1); + + const tblRequests = page.getByRole('table', { name: 'Requests' }); + const requests = tblRequests.locator('tbody tr'); await expect(rows).toHaveCount(1); + await expect(await getCellByColumnName(tblRequests, 'Type', requests.nth(0))).toHaveText('Connect') + await expect(await getCellByColumnName(tblRequests, 'Channel', requests.nth(0))).toHaveText('/chats/general') + await expect(await getCellByColumnName(tblRequests, 'Reason', requests.nth(0))).toHaveText('') + await expect(await getCellByColumnName(tblRequests, 'Closed By', requests.nth(0))).toHaveText('') + await expect(await getCellByColumnName(tblRequests, 'Time', requests.nth(0))).not.toBeEmpty() - await rows.nth(0).click() + await messages.nth(0).click() }) - await meta.getByLabel('Channel').click(); - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await page.getByRole('region', { name: 'Meta' }).getByLabel('Path').click(); + await breadcrumb.getByRole('link', { name: 'WebSocket Chat API' }).click() }) }) await test.step('Visit chats/{chatId}', async () => { - const topics = page.getByRole('table', { name: 'Channel' }); + const topics = page.getByRole('table', { name: 'Paths' }); await topics.getByText('/chats/{chatId}').click(); - await expect(page.getByLabel('Channel', { exact: true })).toHaveText('/chats/{chatId}'); + await expect(page.getByLabel('Path Template', { exact: true })).toHaveText('/chats/{chatId}'); await expect(page.getByLabel('Title')).not.toBeVisible(); await expect(page.getByLabel('Summary')).not.toBeVisible(); await expect(page.getByLabel('Description')).not.toBeVisible(); await expect(page.getByLabel('Type of API')).toHaveText('Websocket'); + const instances = page.getByRole('table', { name: 'Paths' }) + const instance = instances.locator('tbody tr').nth(0); + await expect(await getCellByColumnName(instances, 'Path', instance)).toContainText('/chats/1234') + await expect(await getCellByColumnName(instances, 'Last Message', instance)).not.toHaveText('-') + await expect(await getCellByColumnName(instances, 'Messages', instance)).toContainText('2') + const messages = page.getByRole('table', { name: 'Messages' }); const rows = messages.locator('tbody tr'); await expect(rows).toHaveCount(2); - let channel = await getCellByColumnName(messages, 'Channel', rows.nth(0)) + let channel = await getCellByColumnName(messages, 'Path', rows.nth(0)) await expect(channel).toHaveText('/chats/1234'); await expect(channel.getByRole('img')).not.toBeVisible(); await expect(await getCellByColumnName(messages, 'Value', rows.nth(0))).toHaveText('{"userId":"carol","username":"Carol","text":"Hi Alice!","timestamp":"2026-02-13T09:49:26.100000+01:00"}'); await expect(await getCellByColumnName(messages, 'Time', rows.nth(0))).not.toBeEmpty(); - channel = await getCellByColumnName(messages, 'Channel', rows.nth(1)) + channel = await getCellByColumnName(messages, 'Path', rows.nth(1)) await expect(channel).toHaveText('/chats/1234'); await expect(channel.getByRole('img')).toBeVisible(); await expect(await getCellByColumnName(messages, 'Value', rows.nth(1))).toHaveText('{"bar": "not a number"}'); await expect(await getCellByColumnName(messages, 'Time', rows.nth(1))).not.toBeEmpty(); + await test.step('Visit instance', async() => { + await instance.click() + + const meta = page.getByRole('region', { name: 'Info' }) + await expect(meta.getByLabel('Path')).toHaveText('/chats/1234'); + + await expect(breadcrumb).toContainText('/chats/{chatId}') + await expect(breadcrumb).toContainText('/chats/1234') + }) + await test.step('Visit message', async () => { await rows.nth(0).click(); const meta = page.getByRole('region', { name: 'Meta' }) - await expect(meta.getByLabel('Channel')).toHaveText('/chats/1234'); + await expect(meta.getByLabel('Path')).toHaveText('/chats/1234'); await expect(meta.getByLabel('Time')).not.toBeEmpty(); await expect(meta.getByLabel('Client')).toHaveText('127.0.0.1:53298'); await expect(meta.getByLabel('Content Type')).toHaveText('application/json'); @@ -154,7 +180,7 @@ test('Visit Websocket overview', async ({ page }) => { }) - await meta.getByLabel('Channel').click(); + await meta.getByLabel('Path').click(); }) await test.step('Visit invalid message', async () => { @@ -180,14 +206,11 @@ test('Visit Websocket overview', async ({ page }) => { const dialog = page.getByRole('dialog') await expect(dialog.getByRole('region', { name: 'Content' })).toContainText('{"bar": "not a number"}') await dialog.getByRole('button', { name: 'Close' }).click() - - const ids = await page.locator('.modal.fade').evaluateAll(els => els.map(e => e.id)) -console.log(ids) }) const meta = page.getByRole('region', { name: 'Meta' }) - await expect(meta.getByLabel('Channel')).toHaveText('/chats/1234'); + await expect(meta.getByLabel('Path')).toHaveText('/chats/1234'); await expect(meta.getByLabel('Time')).not.toBeEmpty(); await expect(meta.getByLabel('Client')).toHaveText('127.0.0.1:53298'); await expect(meta.getByLabel('Content Type')).toHaveText('-'); @@ -199,11 +222,10 @@ console.log(ids) await expect(value.getByLabel('Size of Code')).toHaveText('23 B'); await expect(value.getByLabel('Content', { exact: true })).toContainText('{"bar": "not a number"}'); - await meta.getByLabel('Channel').click(); + await meta.getByLabel('Path').click(); }) - - await page.getByRole('region', { name: 'Info' }).getByLabel('Cluster').click(); + await breadcrumb.getByRole('link', { name: 'WebSocket Chat API' }).click() }) diff --git a/webui/lib/events.ts b/webui/lib/events.ts new file mode 100644 index 000000000..c1730f949 --- /dev/null +++ b/webui/lib/events.ts @@ -0,0 +1,164 @@ +interface Event { + id: string; + traits: Traits; + time: string; + data: any +} + +interface Label { + name: string + value: string +} + +export interface Query { + labels: Label[] + http?: HttpEventFilter +} + +export interface HttpEventFilter { + methods?: string[] + url?: string + requestHeaders?: { name: string, value: string }[] + statusCode?: string + responseHeaders?: { name: string, value: string }[] + clientIP?: string +} + +export function match(e: Event, q?: Query): boolean { + if (!q) { + return true + } + + if (!matchTraits(e, q.labels)) { + return false + } + if (q.http && e.traits.namespace === 'http') { + console.log(q.http.methods, e) + if (q.http.methods) { + if (!q.http.methods.includes(e.data.request.method)) { + return false + } + } + if (q.http.url) { + if (!e.data.request.url.match(q.http.url)) { + return false + } + } + if (q.http.requestHeaders) { + if (!e.data.request.parameters) { + return false + } + for (const h of q.http.requestHeaders) { + const found = e.data.request.parameters.find((x: any) => x.type === 'header' && x.name === h.name) + if (!found) { + return false + } + if (h.value !== '' && (!found.raw || !found.raw.match(h.value))) { + return false + } + } + } + if (q.http.responseHeaders) { + if (!e.data.response.headers) { + return false + } + for (const h of q.http.responseHeaders) { + const found = e.data.response.headers[h.name] + if (!found) { + return false + } + if (h.value !== '' && !found.match(h.value)) { + return false + } + } + } + if (q.http.statusCode) { + const values = q.http.statusCode.split(',').map((x: string) => x.trim()) as string[] + const statusCode = e.data.response.statusCode + for (let val of values) { + let not = false + let match = false + if (val[0] === '!') { + not = true + val = val.substring(1) + } + + if (val.length === 3 && val.includes('xx')) { + val = `${val[0]}00-${val[0]}99` + } + + if (val.includes('-')) { + const parts = val.split('-').map(x => x.trim()) + if (parts.length != 2) { + continue + } + const min = parseInt(parts[0]) + const max = parseInt(parts[1]) + match = statusCode >= min && statusCode <= max + } else { + match = statusCode.toString() === val + } + + if (not === match) { + return false + } + } + } + if (q.http.clientIP) { + const values = q.http.clientIP.split(',').map((x: string) => x.trim()) as string[] + for (let val of values) { + let not = false + let match = false + if (val[0] === '!') { + not = true + val = val.substring(1) + } + + if (val.includes('/')) { + match = cidrMatch(e.data.clientIP, val) + } else { + match = e.data.clientIP === val + } + + if (not === match) { + return false + } + } + } + } + return true +} + +export function matchTraits(e: ServiceEvent, labels: Label[]) { + for (let label of labels) { + if (e.traits[label.name] !== label.value) { + return false + } + } + return true +} + +function cidrMatch(ip: string, cidr: string): boolean { + const parts: string[] = cidr.split('/'); + if (parts.length !== 2) { + return false + } + + const [range, bitsStr] = parts; + const bits: number = parseInt(bitsStr!, 10); + + if (isNaN(bits) || bits < 0 || bits > 32) { + return false + } + + const mask: number = ~(2 ** (32 - bits) - 1); + + return (ipToInt(ip) & mask) === (ipToInt(range!) & mask); +} + +function ipToInt(ip: string): number { + return ip + .split('.') + .map((octet) => parseInt(octet, 10)) + .reduce((acc, octet) => (acc << 8) + octet, 0); +} \ No newline at end of file diff --git a/webui/package-lock.json b/webui/package-lock.json index a2d402c87..ea52f7dd9 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -27,7 +27,7 @@ "js-yaml": "^5.2.2", "kafkajs": "^2.2.4", "ldapts": "^9.0.0", - "markdown-it": "^14.2.0", + "markdown-it": "^15.0.0", "markdown-it-container": "^4.0.0", "mime-types": "^3.0.2", "mqtt": "^5.15.1", @@ -40,22 +40,22 @@ "xml-formatter": "^3.7.0" }, "devDependencies": { - "@playwright/test": "1.63.0", + "@playwright/test": "1.64.0", "@rushstack/eslint-patch": "1.16.1", "@types/js-yaml": "4.0.9", "@types/markdown-it-container": "4.0.1", - "@types/node": "26.6.4", - "@vitejs/plugin-vue": "6.0.9", + "@types/node": "26.6.5", + "@vitejs/plugin-vue": "6.0.10", "@vue/eslint-config-prettier": "10.2.0", "@vue/eslint-config-typescript": "14.9.0", "@vue/tsconfig": "0.9.1", "eslint": "10.12.0", "eslint-plugin-vue": "10.11.1", "npm-run-all2": "9.0.3", - "prettier": "3.9.9", + "prettier": "3.9.10", "typescript": "6.0.3", - "vite": "8.3.3", - "vue-tsc": "3.3.12", + "vite": "8.3.4", + "vue-tsc": "3.3.13", "xml2js": "0.6.2" } }, @@ -78,9 +78,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", - "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "version": "7.29.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.9.tgz", + "integrity": "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA==", "license": "MIT", "dependencies": { "@babel/types": "^7.29.8" @@ -328,9 +328,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { @@ -403,9 +403,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.151.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.151.0.tgz", - "integrity": "sha512-J1yXrIlNDZVzE3ada310xeAw7nH8yCAyLPuUIsjKatFPmfn5bS1oW+cM+QsGOtVWd5nhSpbwZWx/rue+r5Z+PA==", + "version": "0.153.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.153.0.tgz", + "integrity": "sha512-jSNrRIyvPVLL5E1z/tS5zLAprosGz/FuI1PWwEOz59j3E4TTTzSpqKb/RDGqJhhW0QHYISGXxq582wGx6T11Tw==", "devOptional": true, "license": "MIT", "funding": { @@ -426,13 +426,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.63.0", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", - "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.0.tgz", + "integrity": "sha512-oTN9FNYuN6bqQMwiho6jJKLCG2/QOYCixGXuCjRmzOq23eODeukuY2XiAVoVpAECbuezMt1MHQjAiQxK8F+Y+g==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.63.0" + "playwright": "1.64.0" }, "bin": { "playwright": "cli.js" @@ -452,9 +452,9 @@ } }, "node_modules/@rolldown/binding-android-arm-eabi": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.11.tgz", - "integrity": "sha512-A5kXfGKvKWWZE0TtPrfsvT+q4Y5d1QG8gGUzpYjGydM+fARM9MuX90PrXYXe0XbsDVgyxxNzHo6giCj90bsFNw==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.13.tgz", + "integrity": "sha512-Vu/NICrgdRvUOeaJZIyB2FP63s6fO/4ud5l8dXmxDx9yDyn+mN0y+gsvsVviSli+T361Aqo0h46EuMoobrwawA==", "cpu": [ "arm" ], @@ -468,9 +468,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.11.tgz", - "integrity": "sha512-z6cTycz+iJ4PVkuL4HHW4DfTfoeU/2nqYYuSOrTmH7yHK5Y0LCOnA03V4ZNxavyVaU1oOqUgIg2klN/s+USGOA==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.13.tgz", + "integrity": "sha512-6LEoIrGy5vvCd2boNnv2soPSUZj/PRvhp8R/t9KlX+mIZxThIE/ZqxrGXxmuNX0FuvQFvLdZr7S4HrYpYBVhEQ==", "cpu": [ "arm64" ], @@ -484,9 +484,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.11.tgz", - "integrity": "sha512-jShvqNtP6vDC6/A5JOAzbVV+DkgHqhl/ScVCJEbt+TUY6QYz7YnXcrg3sLtFBniro0f/Ld50ZwCWA6f7KYD1nQ==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.13.tgz", + "integrity": "sha512-CaJSRzlQ6Q+zkDMVA9OkVxBjVoPRCDEE3v0FSJ9kIQBgRjFApA3Si4ijhbNVUji7ZJt+PfO0e46qy4nPInHUPA==", "cpu": [ "arm64" ], @@ -500,9 +500,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.11.tgz", - "integrity": "sha512-f2i2xiNWq1Z1l2++q2fuhZRdLAT3aqxD6vRNm1RAxpUoBcdqNB3C0s1Bt+K+PbEx2F5F4gQp6hqKkphCY/xF9w==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.13.tgz", + "integrity": "sha512-e4r340I5hxSzJkD3YJBxgzq/rJ3wwxwdLLb2FPt1Zpyem20nrQPT11DDEEkazwxaHumH8wpglsp/Ozgj/wT6mw==", "cpu": [ "x64" ], @@ -516,9 +516,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.11.tgz", - "integrity": "sha512-4Ir5FSOKIAMr4r0kExpt1s3bMgzJU3rA45AYOHtQpls0oNeqcYBKrWMlckrYH4KCfGLfkfn1tN1dmZPMVsdXow==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.13.tgz", + "integrity": "sha512-hoBWDZ4/K/mJKVMbsgUd1jfpA2DiOrJ54mpegObU9TBHBBrtX4FLIXJJQRDPZ/2Dn91GvPob+Z2hwKJzf4E5SQ==", "cpu": [ "x64" ], @@ -532,9 +532,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.11.tgz", - "integrity": "sha512-/gnRDM+39BROzAN/k1OZjDPnDMcZxB/0EUxKjONO5yVkNEvlsoMDrxGNKgZi/ttFriS2gwlDNzB65pvNbFOXIQ==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.13.tgz", + "integrity": "sha512-oF9fZu3vTWnTiVAkDFxU9PJLt1nh20K9RRpuHsR7ZnE2L7XookBbDBVLojCVSLGbvtSlxeUNnjAj6DbNsKyzbg==", "cpu": [ "arm" ], @@ -548,9 +548,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.11.tgz", - "integrity": "sha512-PFaK8HwvAHbaKbBcDNQihjMKYvFnA5hiENx/l5tphTDz1E0WFp32l0A7aq7lyUwGsRw/xSrNIy/gIK4thrSCrw==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.13.tgz", + "integrity": "sha512-DTpb/+jMiqWMPgphn4ksW1zW+XLJ6dJJ9FMvnIk9I291LhCctMPhMYJrP5t9p6C9qa3KXadDdUEZHBaPh3kNxQ==", "cpu": [ "arm64" ], @@ -567,9 +567,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.11.tgz", - "integrity": "sha512-AskzJUIKRLPxkruR1wLKewGbOw+EYfU/9lOrBFj4AFrEA8hPpKFnODWNu2WLaNs0QNkEb9QIJufmVZZIL/bJlg==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.13.tgz", + "integrity": "sha512-xU+Kvv9N2A3SY35zFh1FWWNjxbadE/BKeqnkXe7vScU4GZsw757BlVmDLITZEq8DPpk6m7uoDYkK7P+x/uU5Ow==", "cpu": [ "arm64" ], @@ -586,9 +586,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.11.tgz", - "integrity": "sha512-qlUGAheh2yh8afH7QBgx0PrRHN85hKnNd78x8MeMhXivuevgd8vgf6/CstOzmNKY/lLTHvNTrPy98cLnAugzJw==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.13.tgz", + "integrity": "sha512-XMk/hnhh25oQ756Lclb1Q11h4ll0xC0Esa5w4Vzp+mAE2S3AGJgeM0SonrIgY8fHhjarb6hcoTuwoFG0VVP2ig==", "cpu": [ "ppc64" ], @@ -605,9 +605,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.11.tgz", - "integrity": "sha512-secpEad+0vCbSfn8upFySkDskv+bGPk3THSDS9Y89yc4rb4kzqHp8Dmyd9BkQW4SnhNXBZCl/6CrO//hZahNJQ==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.13.tgz", + "integrity": "sha512-U1BoTHYmlywE2n3HUWjzZ+8EQsKr+OwpxYuDWaubx42tUL8QSL7nOqRz/NVHanibbFP3M1Rxuprz/wNcbMKjqw==", "cpu": [ "s390x" ], @@ -624,9 +624,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.11.tgz", - "integrity": "sha512-mOVBT3dPpkWm8XBWPmU4bf+U6dYDLeMo/9ojUmis4N0L5uu10qra5vOyngZ7/PSdoE4G9KvRt4bloRxNjLas7A==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.13.tgz", + "integrity": "sha512-J4cm80Pc6wEwV5AZ42o62sIQgDk+Y7R2s646jIwuvO+4V46QB58N4vVuMPWCBaNgursXLpmUWJV//WQAkPvHBQ==", "cpu": [ "x64" ], @@ -643,9 +643,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.11.tgz", - "integrity": "sha512-Is78i9A8Ui4SqcxUwFJ9uMmjDn58IbVTjFWYdQestFEgeuEmHMLGNriXnVJKkwG2YiZjw8cP0zCTyDMdDGtOOg==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.13.tgz", + "integrity": "sha512-W0euZS40U4V6JkmEnVCIX9mcsL6cbXQ4X8c2w+MMOIjFg68TXUW+u3RT+2e1t1pv9mg/EP3pDtK3POXdQ7Rxyw==", "cpu": [ "x64" ], @@ -662,9 +662,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.11.tgz", - "integrity": "sha512-dUCXneZ87INUMyQ0D+C0HrEBNUPNXHaPmU5GTjyKTJEiussw9Kaj5Ln8UztPe4epV/ffvgNBEadksdYhmW6xJA==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.13.tgz", + "integrity": "sha512-do/AF/4n9MRshia3a5aXoy8xM2c6mgfT6D7H/dY6my5z+hfCIR4ab7mk4IRj++nsEQpKy39kd0DQnDq+r692tw==", "cpu": [ "arm64" ], @@ -678,9 +678,9 @@ } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.11.tgz", - "integrity": "sha512-jByxb6qfd+bH1xUd0qnfFnb17i9sWBPY2tOavJ0l3tdr3OTu+Kvtm8cd/JV5nFt657b1VqGltxg9olOEfofXWw==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.13.tgz", + "integrity": "sha512-6FtUf2cJZobutJdxnGYX8z0vtbgqU0b2v8vfBhkXb9wp2MppMFtkunE+IaOntyi01rkIQoPW2GHABtfzsC0BRQ==", "cpu": [ "arm64" ], @@ -694,9 +694,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.11.tgz", - "integrity": "sha512-/PzKqzAJ03i19oy2ItPvyvaVjOjBCNnfaJs8yvUdGBKmiESgnrJSQ2awd81QzFbbnAmu7YO9ZnJrDCb9VSJPRA==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.13.tgz", + "integrity": "sha512-UKhFaZTeTcbKrb3iU82m17jVcPEAoqDkCN5MBqgPyZSwp/cQmib21crTS0DdH7T1B7wZRWEI89YQAOJMet5yhg==", "cpu": [ "x64" ], @@ -772,7 +772,6 @@ "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "dev": true, "license": "MIT" }, "node_modules/@types/js-yaml": { @@ -831,9 +830,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "26.6.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.4.tgz", - "integrity": "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg==", + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", "license": "MIT", "dependencies": { "undici-types": "~8.9.0" @@ -1116,9 +1115,9 @@ } }, "node_modules/@vitejs/plugin-vue": { - "version": "6.0.9", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.9.tgz", - "integrity": "sha512-rD/MORlhaZMlXWW0rEn4FB4wMinWC0z/D7Ye160S5+Rs1mC8ZDAdcDp4SjKfUhG5t8q1ktcPVw4xuTkPRlzrSA==", + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-6.0.10.tgz", + "integrity": "sha512-b80aUQh3n+qEzzZm5oQp3rT2QImeCx58K6eFl/RkJ3d65KW0kTuv5lNsLRVAjbW9YRUpo/wzmvzMgV34bTAItA==", "dev": true, "license": "MIT", "dependencies": { @@ -1129,7 +1128,7 @@ }, "peerDependencies": { "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", - "vue": "^3.2.25" + "vue": "^3.2.25 || ^3.6.0-rc.10" } }, "node_modules/@volar/language-core": { @@ -1189,16 +1188,16 @@ } }, "node_modules/@vue/compiler-core": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.43.tgz", - "integrity": "sha512-zdiLhnbe1QQqgDT8xZMpNmyqZ3qlI+/Q/FHQco57Kwl/b05HhCzN6eVGN9QU9rbga4CrS0H5SYY8VGHZCt/1Hg==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.44.tgz", + "integrity": "sha512-k34nCytbK8PQ1gR2mGROYOw6Pnv/mumqIdrZ3x2tbyXZDWm7ViKGLTfdB+DRrWqEQsLgGJvn0WN9jBif5zCTLQ==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.8", - "@vue/shared": "3.5.43", + "@babel/parser": "^7.29.9", + "@vue/shared": "3.5.44", "entities": "^7.0.1", "estree-walker": "^2.0.2", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" } }, "node_modules/@vue/compiler-core/node_modules/entities": { @@ -1214,40 +1213,40 @@ } }, "node_modules/@vue/compiler-dom": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.43.tgz", - "integrity": "sha512-PEZoAk3NQmsn/ejMzSOCyTYqwGqczrWm70PuhBKjjv1+TCoQAaO/zOqNwjV+honlNstT5ILxtc+8r8UUfj+iEQ==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.44.tgz", + "integrity": "sha512-W5AoBFWdw1lQAKiHowkfgFKFab82oSRT6sGsPfw/q1GRjsy/ZJFLHGhJyBRIFN9APLTUduDH9U2JAiPOIyKuYg==", "license": "MIT", "dependencies": { - "@vue/compiler-core": "3.5.43", - "@vue/shared": "3.5.43" + "@vue/compiler-core": "3.5.44", + "@vue/shared": "3.5.44" } }, "node_modules/@vue/compiler-sfc": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.43.tgz", - "integrity": "sha512-FCbrG3XNCRl+js3huuKx4IVHBLTvMkJhVepjbxSPu1gn4yWLaYtGNQdjJGZaMytXB6qb76qQDDmDSLy/vkmleQ==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.44.tgz", + "integrity": "sha512-dzHmQ4jXTpdr5FjIQUeykVakN/uzqFbUo9/C5ikBFg3a3PeM6uRxeczx1ZbXBML0/zJstEh3ygYipzR5Z7b2Pw==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.8", - "@vue/compiler-core": "3.5.43", - "@vue/compiler-dom": "3.5.43", - "@vue/compiler-ssr": "3.5.43", - "@vue/shared": "3.5.43", + "@babel/parser": "^7.29.9", + "@vue/compiler-core": "3.5.44", + "@vue/compiler-dom": "3.5.44", + "@vue/compiler-ssr": "3.5.44", + "@vue/shared": "3.5.44", "estree-walker": "^2.0.2", "magic-string": "^0.30.21", - "postcss": "^8.5.28", - "source-map-js": "^1.2.1" + "postcss": "^8.5.29", + "source-map-js": "^1.2.2" } }, "node_modules/@vue/compiler-ssr": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.43.tgz", - "integrity": "sha512-GF62orf7KiJX9RqrHNGrYBudsQGD0OhJ5nDs90O8UiDuS40+YMYomiXu6w6EuvtXuRDc3MSNis3EaaSKAVSWpg==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.44.tgz", + "integrity": "sha512-xr5gb1JVHlfXJwSlRz1FeP6uxoazdzHuMTVLE7up+4Vrqxk219CvxLPxD9NXBt47yvZW7+p4DXfyp4O1fkByIA==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.43", - "@vue/shared": "3.5.43" + "@vue/compiler-dom": "3.5.44", + "@vue/shared": "3.5.44" } }, "node_modules/@vue/devtools-api": { @@ -1322,9 +1321,9 @@ } }, "node_modules/@vue/language-core": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-3.3.12.tgz", - "integrity": "sha512-k7uIKR0n4CBidk233y7cac9osnDQ5WRidTl6nsrNvpZ3nf8cx9iJxRVsRzVsoPRHbowT/iKgGMTS9RGov3jeIA==", + "version": "3.3.13", + "resolved": "https://registry.npmjs.org/@vue/language-core/-/language-core-3.3.13.tgz", + "integrity": "sha512-axA0Xemn5LuUKuKWob0/kVq7xQ2mJJM6VqA9Exvt35nM9yRC31goWVve/G/Zb2ZCLsXC8FHYjjfXlnshknSCew==", "dev": true, "license": "MIT", "dependencies": { @@ -1351,51 +1350,51 @@ } }, "node_modules/@vue/reactivity": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.43.tgz", - "integrity": "sha512-G/c9GyOZNI2jVaaS6OX1EF1SSFSv7H0ERqNTl4+DTFMlZmB5eVAB53aLQNam/7NL2NPtaDD7RdVrzf8uJzMuOA==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.44.tgz", + "integrity": "sha512-u+QJdKeBBEADkPdevsL+6E12QlYVfBD8Zcw5qjDQKtamI6l14KyqazJMobhCKelhWN5zpyKUiEDoTyHE93cqWw==", "license": "MIT", "dependencies": { - "@vue/shared": "3.5.43" + "@vue/shared": "3.5.44" } }, "node_modules/@vue/runtime-core": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.43.tgz", - "integrity": "sha512-hU6U6VnVhBGQDpvlnnDlIB8ZGJBiOcgk2lh/0InltHiz3D8oSkluvuvY+do1G2H3+udeKFsmaBlgVYP7gXQzEw==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.44.tgz", + "integrity": "sha512-uEa/ndwjhrrhnOfb2p+sZy6zeoamJOISqaLSJGk/8R6nABT1ZldpzcJoPzKj+zHVomVBPcmQYyC2r4LtvDV6CQ==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.43", - "@vue/shared": "3.5.43" + "@vue/reactivity": "3.5.44", + "@vue/shared": "3.5.44" } }, "node_modules/@vue/runtime-dom": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.43.tgz", - "integrity": "sha512-Bb2Jc0YjjJdMt1SJmb9b2L/IWd3I8lIT9x9eS/xvvP9CiVgna0ffua74xKRmt4/uSJ+0r4iN8ex1jrqkhQGWQw==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.44.tgz", + "integrity": "sha512-3fc4ARcXVhVBaJ3ujWiNZJARD6pkyRcwG+ZYhMzjKtKUp7MAY0HLwr7r3Smep3Li6ZaWT6ewdmx8XqdaD5oNZg==", "license": "MIT", "dependencies": { - "@vue/reactivity": "3.5.43", - "@vue/runtime-core": "3.5.43", - "@vue/shared": "3.5.43", + "@vue/reactivity": "3.5.44", + "@vue/runtime-core": "3.5.44", + "@vue/shared": "3.5.44", "csstype": "^3.2.3" } }, "node_modules/@vue/server-renderer": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.43.tgz", - "integrity": "sha512-l2Ygjv9NehV94PSBxNWsAHC0j/eIIKbn92mBuWXAPGLnn6HfJ6MH5ubsd+Nk0YoZ5FRuxWI1P2VSoh+dbPQhCQ==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.44.tgz", + "integrity": "sha512-R4S1UoH4h5Frrauv8Z5XE5TxV9oBiUSgWIC4TyVK1FK+RocRnOLwFA5Xcmo2011RlgJp+AbZCnw6afrhlgRuUQ==", "license": "MIT", "dependencies": { - "@vue/compiler-ssr": "3.5.43", - "@vue/runtime-dom": "3.5.43", - "@vue/shared": "3.5.43" + "@vue/compiler-ssr": "3.5.44", + "@vue/runtime-dom": "3.5.44", + "@vue/shared": "3.5.44" } }, "node_modules/@vue/shared": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.43.tgz", - "integrity": "sha512-uksS7YGMR5NZyr4JNq0Rp+QyLns0ueaz20KwzIPW9R0LH1Vnt4E+XUM29PNseEbf1www2gOuhuDi5AKOIXag9Q==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.44.tgz", + "integrity": "sha512-H82XLH0DjmSOdbEwO15x3QTo1f5PWEtv237eLpYKc0mqLu+B8hnq16lFWmuBmNurkeiG3wr+oM1FhOll/YVFRg==", "license": "MIT" }, "node_modules/@vue/tsconfig": { @@ -1530,22 +1529,94 @@ } }, "node_modules/ast-walker-scope": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/ast-walker-scope/-/ast-walker-scope-0.9.0.tgz", - "integrity": "sha512-IJdzo2vLiElBxKzwS36VsCue/62d6IdWjnPB2v3nuPKeWGynp6FF/CYoLa5i/3jXH/z97ZDdsXz6abpgM6w07A==", + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/ast-walker-scope/-/ast-walker-scope-0.10.0.tgz", + "integrity": "sha512-KzvoxDfXNz2Gm8KYLh47K3qcY3Zza3WKOuh2xoqa7LBu6MDKWAZLovJtU9DuAd/Er+rhQHCRn619lHS5U+qmhw==", "license": "MIT", "dependencies": { - "@babel/parser": "^7.29.2", - "@babel/types": "^7.29.0", - "ast-kit": "^2.2.0" + "@babel/parser": "^8.0.0", + "@babel/types": "^8.0.0", + "ast-kit": "^3.0.0" }, "engines": { - "node": ">=20.19.0" + "node": "^22.18.0 || >=24.11.0" + }, + "funding": { + "url": "https://github.com/sponsors/sxzz" + } + }, + "node_modules/ast-walker-scope/node_modules/@babel/helper-string-parser": { + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-8.0.6.tgz", + "integrity": "sha512-+6HYEBnsDRZHJtUTAEfMQd0zMDqdFHS7C+5vAsEZqBUr3vEMWOWlHJ0Jz1gS8jJIlCvLsuA6ivmeoqzH936YcA==", + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/ast-walker-scope/node_modules/@babel/helper-validator-identifier": { + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-8.0.6.tgz", + "integrity": "sha512-HYx0YmuchRxRSpnjvcNffnPS8DCauKNmnumIuHG3jnQAVewRA7kfJGwF13ayEpSZs7s4R/oBH4eppuXcAQd5JQ==", + "license": "MIT", + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/ast-walker-scope/node_modules/@babel/parser": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.7.tgz", + "integrity": "sha512-UkDwEbPO1px2JQSrdh4qIgaH1JgG7GO/pFeJkpxhDxIETkT/QvGpnigXIw75ViHYPtIgKITmVUJqLjiO+JuWjA==", + "license": "MIT", + "dependencies": { + "@babel/types": "^8.0.6" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/ast-walker-scope/node_modules/@babel/types": { + "version": "8.0.6", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.6.tgz", + "integrity": "sha512-c+xgSWboV2pdwXP67BUWDVRHi2BO5Q3KwxyFVs7taVTY+fsK47/L51ZXQoV9rozHJDNHvJ4v4WEsI/IlxV2l2A==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^8.0.6", + "@babel/helper-validator-identifier": "^8.0.6" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" + } + }, + "node_modules/ast-walker-scope/node_modules/ast-kit": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ast-kit/-/ast-kit-3.0.0.tgz", + "integrity": "sha512-8OG92q3R35qjC/4i6BLBMg8IB+fClWu/1PEwg2Z9Rn+BuNaiEgJzpzn+pxWOdHJWDCAwu2JP0wCDTozAM4QirQ==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^8.0.0", + "estree-walker": "^3.0.3", + "pathe": "^2.0.3" + }, + "engines": { + "node": "^22.18.0 || >=24.11.0" }, "funding": { "url": "https://github.com/sponsors/sxzz" } }, + "node_modules/ast-walker-scope/node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -1621,19 +1692,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/body-parser/node_modules/content-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", - "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/body-parser/node_modules/iconv-lite": { "version": "0.7.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", @@ -1677,9 +1735,9 @@ } }, "node_modules/bootstrap-icons": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/bootstrap-icons/-/bootstrap-icons-1.13.1.tgz", - "integrity": "sha512-ijombt4v6bv5CLeXvRWKy7CuM3TRTuPEuGaGKvTV5cz65rQSY8RQ2JcHt6b90cBBAC7s8fsf2EkQDldzCoXUjw==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/bootstrap-icons/-/bootstrap-icons-1.13.2.tgz", + "integrity": "sha512-fAU1Ly3dH7hwRg+CB8EZ0Vl0fF80zgXHeUVJyfUOKzoURr+TbwISbCW5tKUoQLERaWYG2emTCybcPbD96hdD1A==", "funding": [ { "type": "github", @@ -1877,9 +1935,9 @@ "license": "MIT" }, "node_modules/content-disposition": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", - "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-2.0.1.tgz", + "integrity": "sha512-e+H0ZXHSWYrENhQzw1LPuP4oF5MzVKmDU6d3hxlvaPEYLLg62MxtQNPRx4SYSuYJSBUgnQIG4HIN2tEtNv7Dog==", "license": "MIT", "engines": { "node": ">=18" @@ -1890,12 +1948,16 @@ } }, "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", + "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", "license": "MIT", "engines": { - "node": ">= 0.6" + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/cookie": { @@ -2498,12 +2560,12 @@ } }, "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", "license": "BSD-2-Clause", "engines": { - "node": ">=0.12" + "node": ">=20.19.0" }, "funding": { "url": "https://github.com/fb55/entities?sponsor=1" @@ -2852,15 +2914,15 @@ } }, "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/express/-/express-5.3.0.tgz", + "integrity": "sha512-qCqy1xSoaugxOD4IVHb1ZWdkvCF2W3aE2okKouWj8VwJ+ztskSiXoBnCSJqfKT+cVPJhuJqe9YLbY425+Zh00g==", "license": "MIT", "dependencies": { "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", + "body-parser": "^2.3.0", + "content-disposition": "^2.0.1", + "content-type": "^2.0.0", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", @@ -2876,14 +2938,14 @@ "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", + "proxy-addr": "^2.0.8", + "qs": "^6.16.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", - "type-is": "^2.0.1", + "type-is": "^2.1.0", "vary": "^1.1.2" }, "engines": { @@ -3799,9 +3861,9 @@ } }, "node_modules/linkify-it": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.2.tgz", - "integrity": "sha512-ONTm2jCMAVZjgQa/Fy1kScXsuOoF5NPTsoFBdE1KVIZ2vAh/r9+Bqo+0jINCBYnavTPQZz38QzFTme79ENoN3Q==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-6.1.0.tgz", + "integrity": "sha512-wJ/TwpSDTLepCrQoYWYIExIKg5Zchex2Nn5yk2mFnB+6PtdkHtyLx742md9csRjjOnGkKIS/RrbY7l8D6gT9Vw==", "funding": [ { "type": "github", @@ -3814,7 +3876,7 @@ ], "license": "MIT", "dependencies": { - "uc.micro": "^2.0.0" + "uc.micro": "^3.0.0" } }, "node_modules/local-pkg": { @@ -3881,9 +3943,9 @@ } }, "node_modules/markdown-it": { - "version": "14.3.2", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.3.2.tgz", - "integrity": "sha512-sHHjZ5fJKlgrG4qns2YwVcdNep35h5fERrfkD2YNsb9UFk0UIHarbiTaHKVMlPuWAoiilyK8Fv/jAm11slsY7Q==", + "version": "15.0.2", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-15.0.2.tgz", + "integrity": "sha512-q4IGxMv56jCqT4OCRCADBoDP3LO4MhmTXjFbphHPXs4g3j9Xg5RDnxqN8IF/3vIWEU+VCnUq+7JUg/cfy2E6Qw==", "funding": [ { "type": "github", @@ -3896,12 +3958,12 @@ ], "license": "MIT", "dependencies": { - "argparse": "^2.0.1", - "entities": "^4.5.0", - "linkify-it": "^5.0.2", - "mdurl": "^2.0.0", + "argparse": "^3.0.0", + "entities": "^8.0.0", + "linkify-it": "^6.0.0", + "mdurl": "^2.1.0", "punycode.js": "^2.3.1", - "uc.micro": "^2.1.0" + "uc.micro": "^3.0.0" }, "bin": { "markdown-it": "bin/markdown-it.mjs" @@ -3913,6 +3975,22 @@ "integrity": "sha512-HaNccxUH0l7BNGYbFbjmGpf5aLHAMTinqRZQAEQbMr2cdD3z91Q6kIo1oUn1CQndkT03jat6ckrdRYuwwqLlQw==", "license": "MIT" }, + "node_modules/markdown-it/node_modules/argparse": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-3.0.2.tgz", + "integrity": "sha512-mFdDM6WqWKraGLsVb+C9CahPnzTXOefAOLq3jYcca2YZ8bEWpr++Tzj+zSaKW9+X9L5uSxcm1AZ3Y6aZJ09OhQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "PSF-2.0" + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -3923,9 +4001,9 @@ } }, "node_modules/mdurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.1.0.tgz", + "integrity": "sha512-1+HBaOx0zi/dQWht8rNv9MYf9qqpqL/kxI0hXImU6Y547zM6Sni8BQibt7ifgMcYtQg41ao3Ivd6cnSM86inpg==", "license": "MIT" }, "node_modules/media-typer": { @@ -4126,9 +4204,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "funding": [ { "type": "github", @@ -4169,18 +4247,18 @@ } }, "node_modules/nodemailer": { - "version": "10.0.15", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.0.15.tgz", - "integrity": "sha512-EUqp5PhtcsYXs9Fq/lS7s/8zlTrnBqmzZWzFFROrNikMiz+om/YRKMwqN907t+0A1KKyT8jd39CBUbFZmaZmcA==", + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-10.1.0.tgz", + "integrity": "sha512-fXW8AulB25aOTuwfRYV6lqHyfz2VM4rr4SGXH03mE7x5znEUpzL5mu6anARUpZN7744Ziwkmr4AxSLPJsBQspg==", "license": "MIT-0", "engines": { "node": ">=20.0.0" } }, "node_modules/nostics": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/nostics/-/nostics-1.2.0.tgz", - "integrity": "sha512-FGqEfhQjrvo1lL8KFifdTQiNwwQHJxC1jtYE1Rc54qF/jxONUNL+kC9gS1krX8Q65PgrQ5fCqH/I4NhWBvdSqg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/nostics/-/nostics-1.3.0.tgz", + "integrity": "sha512-wzRsrJwNlFbqIRQt8aCeL2WE8USip/uuvGSOZV1D02CxX8ABTMbw8IywKjH7yWHETMXbWJI4OBpihg9rmz2Vyg==", "license": "MIT" }, "node_modules/npm-normalize-package-bin": { @@ -4489,13 +4567,13 @@ } }, "node_modules/playwright": { - "version": "1.63.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", - "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.0.tgz", + "integrity": "sha512-kVzTnYFYEyQ9RGzBjNCd2+yEqWmVbxl3AHPynv3qQ6JTjv4a+kwRcnqQwIpuP3NnrnIrE/Q6wWAIAvMfIE2/Yg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.63.0" + "playwright-core": "1.64.0" }, "bin": { "playwright": "cli.js" @@ -4505,9 +4583,9 @@ } }, "node_modules/playwright-core": { - "version": "1.63.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", - "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "version": "1.64.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.0.tgz", + "integrity": "sha512-T9r+MZkTECl2+oUcZ26YgZzTrYidTsIhYYkJix+6iDOymOb1FB2RLLSOxRGnZGv5DStt+aduIVbt+E8WygXYpg==", "dev": true, "license": "Apache-2.0", "bin": { @@ -4518,9 +4596,9 @@ } }, "node_modules/postcss": { - "version": "8.5.28", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", - "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "version": "8.5.30", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz", + "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==", "funding": [ { "type": "opencollective", @@ -4537,9 +4615,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.18", + "nanoid": "^3.3.20", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -4582,9 +4660,9 @@ } }, "node_modules/prettier": { - "version": "3.9.9", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", - "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", + "version": "3.9.10", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.10.tgz", + "integrity": "sha512-skGHTH1Yb9QvaTC4hWnRTzEYtCpApF0UMBPz+t/0ZPzP8ihefo/btKd+QA7pUE0fEwA1SbS/NmhyEcstdXOIdA==", "dev": true, "license": "MIT", "bin": { @@ -4849,13 +4927,13 @@ "license": "Unlicense" }, "node_modules/rolldown": { - "version": "1.2.11", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.11.tgz", - "integrity": "sha512-qpSwIyz0jHQq5qXBTNxFmE6664rJ7O+4TvPFOiOaBSrz8IOHc1koKKSqTM2H6u1UG1+TveuC6vaDHKXFOvb1Kw==", + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.13.tgz", + "integrity": "sha512-/5apOkwWxtACcQtZ5WlrAsKS4JwXVThQPJZ3RqIruRY9UG9M8e0VlXN068xum/JRKA4aIUoJ13ghe/kGCvidiQ==", "devOptional": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.151.0", + "@oxc-project/types": "=0.153.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -4865,21 +4943,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm-eabi": "1.2.11", - "@rolldown/binding-android-arm64": "1.2.11", - "@rolldown/binding-darwin-arm64": "1.2.11", - "@rolldown/binding-darwin-x64": "1.2.11", - "@rolldown/binding-freebsd-x64": "1.2.11", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.11", - "@rolldown/binding-linux-arm64-gnu": "1.2.11", - "@rolldown/binding-linux-arm64-musl": "1.2.11", - "@rolldown/binding-linux-ppc64-gnu": "1.2.11", - "@rolldown/binding-linux-s390x-gnu": "1.2.11", - "@rolldown/binding-linux-x64-gnu": "1.2.11", - "@rolldown/binding-linux-x64-musl": "1.2.11", - "@rolldown/binding-openharmony-arm64": "1.2.11", - "@rolldown/binding-win32-arm64-msvc": "1.2.11", - "@rolldown/binding-win32-x64-msvc": "1.2.11" + "@rolldown/binding-android-arm-eabi": "1.2.13", + "@rolldown/binding-android-arm64": "1.2.13", + "@rolldown/binding-darwin-arm64": "1.2.13", + "@rolldown/binding-darwin-x64": "1.2.13", + "@rolldown/binding-freebsd-x64": "1.2.13", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.13", + "@rolldown/binding-linux-arm64-gnu": "1.2.13", + "@rolldown/binding-linux-arm64-musl": "1.2.13", + "@rolldown/binding-linux-ppc64-gnu": "1.2.13", + "@rolldown/binding-linux-s390x-gnu": "1.2.13", + "@rolldown/binding-linux-x64-gnu": "1.2.13", + "@rolldown/binding-linux-x64-musl": "1.2.13", + "@rolldown/binding-openharmony-arm64": "1.2.13", + "@rolldown/binding-win32-arm64-msvc": "1.2.13", + "@rolldown/binding-win32-x64-msvc": "1.2.13" } }, "node_modules/router": { @@ -5351,19 +5429,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/type-is/node_modules/content-type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", - "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -5409,9 +5474,9 @@ } }, "node_modules/uc.micro": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-3.0.0.tgz", + "integrity": "sha512-U3PppEkleoTnIfi8BozMx3yju3qc/L6SwqWo2Sw+54PX+PX0q9I+r1Um5HCmqD7n9VDX5/v3vQH/AjA6deDdtw==", "license": "MIT" }, "node_modules/ufo": { @@ -5448,13 +5513,13 @@ } }, "node_modules/unplugin": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.3.0.tgz", - "integrity": "sha512-qa66K+crbfyE6JK10GjvbJeRrOsuC/JpbnHctfyp/i4oBTxWOzJfRZyDiOk1PtErMFRu8JhsU/wPvOdBNWe5Rg==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-3.4.0.tgz", + "integrity": "sha512-9skdIFlCsPdFV7wUfZxNsFInlW+7nJmGu2gkTu0OUhF56aXGsHab9x52/QhdJ4lC7ZDPWTxbiC4ANqVlsuaW3w==", "license": "MIT", "dependencies": { "@jridgewell/remapping": "^2.3.5", - "picomatch": "^4.0.4", + "picomatch": "^4.0.7", "webpack-virtual-modules": "^0.6.2" }, "engines": { @@ -5462,6 +5527,7 @@ }, "peerDependencies": { "@farmfe/core": "*", + "@rsbuild/core": "*", "@rspack/core": "*", "bun-types-no-globals": "*", "esbuild": "*", @@ -5475,6 +5541,9 @@ "@farmfe/core": { "optional": true }, + "@rsbuild/core": { + "optional": true + }, "@rspack/core": { "optional": true }, @@ -5530,9 +5599,9 @@ } }, "node_modules/unplugin/node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "license": "MIT", "engines": { "node": ">=12" @@ -5567,16 +5636,16 @@ } }, "node_modules/vite": { - "version": "8.3.3", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.3.tgz", - "integrity": "sha512-cTAldKPImjg6c+gk48U19POPn3GCBzZwpdsN8ZMEEcbpes+6/wvfqUd0C2y3qYY4wsj8PwgFwrZ/1jBPVttMSg==", + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.4.tgz", + "integrity": "sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ==", "devOptional": true, "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", - "postcss": "^8.5.28", - "rolldown": "~1.2.11", + "postcss": "^8.5.29", + "rolldown": "~1.2.12", "tinyglobby": "^0.2.17" }, "bin": { @@ -5665,16 +5734,16 @@ "license": "MIT" }, "node_modules/vue": { - "version": "3.5.43", - "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.43.tgz", - "integrity": "sha512-o5qZoksdnjIKvW1srZ3ab7pcDNYAerBjRe54D0LBLfRdCYFrSgBHVXokMas35czQc0//lmx4/tuY4ZNQ+Rf2Ng==", + "version": "3.5.44", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.44.tgz", + "integrity": "sha512-dvvmaOCG7lAMkJTPqPwWeGMsPS6DIsFsfZDUgBp9V6mHf9b3+/wyVwYR4Es+WV6uJC0FKRxp81Obcq0ZwUcGFQ==", "license": "MIT", "dependencies": { - "@vue/compiler-dom": "3.5.43", - "@vue/compiler-sfc": "3.5.43", - "@vue/runtime-dom": "3.5.43", - "@vue/server-renderer": "3.5.43", - "@vue/shared": "3.5.43" + "@vue/compiler-dom": "3.5.44", + "@vue/compiler-sfc": "3.5.44", + "@vue/runtime-dom": "3.5.44", + "@vue/server-renderer": "3.5.44", + "@vue/shared": "3.5.44" }, "peerDependencies": { "typescript": "*" @@ -5723,26 +5792,26 @@ } }, "node_modules/vue-router": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-5.3.1.tgz", - "integrity": "sha512-GDBZzgmILxA/kFnkFbJjQZdZ2QQbngnIMMuoUcjhZIfH1RGMaPjPwX5ASnV38qamuA9uhO0RDjSBHTDNG2uXyQ==", + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/vue-router/-/vue-router-5.4.0.tgz", + "integrity": "sha512-pFvcYcxinDHjZUlf7AzFBrT8r8Xqvv4uMHoJnjc+EqPawXQvHTkF8S4tfSmQLpriVHUGfkaKlI7Et0KRjTMTdw==", "license": "MIT", "dependencies": { "@vue-macros/common": "^3.1.3", "@vue/devtools-api": "^8.1.5", - "ast-walker-scope": "^0.9.0", + "ast-walker-scope": "^0.10.0", "chokidar": "^5.0.0", - "confbox": "^0.2.4", + "confbox": "^0.3.1", "local-pkg": "^1.2.1", - "magic-string": "^0.30.21", + "magic-string": "^1.4.3", "mlly": "^1.8.2", "muggle-string": "^0.4.1", - "nostics": "^1.1.4", + "nostics": "^1.3.0", "pathe": "^2.0.3", - "picomatch": "^4.0.5", + "picomatch": "^4.0.7", "scule": "^1.3.0", "tinyglobby": "^0.2.17", - "unplugin": "^3.3.0", + "unplugin": "^3.4.0", "unplugin-utils": "^0.3.2" }, "funding": { @@ -5770,6 +5839,21 @@ } } }, + "node_modules/vue-router/node_modules/confbox": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.3.1.tgz", + "integrity": "sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA==", + "license": "MIT" + }, + "node_modules/vue-router/node_modules/magic-string": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.3.tgz", + "integrity": "sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.6.0" + } + }, "node_modules/vue-router/node_modules/picomatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", @@ -5783,14 +5867,14 @@ } }, "node_modules/vue-tsc": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-3.3.12.tgz", - "integrity": "sha512-kbpaz18O9jPVkXVHD1BEjpUXM6WhXSu+62L1sGGr6Wbzd8GbvkBQWjoj3KZ/C0aq2MZllce7UdtWzwCIYsBMGA==", + "version": "3.3.13", + "resolved": "https://registry.npmjs.org/vue-tsc/-/vue-tsc-3.3.13.tgz", + "integrity": "sha512-jE4/bXdOMSHVxEGgQCqNllJy+9N3Gx6tfSAsg0yhFK/MAN6eXqISihupnDEKSCt12VVnjN2gI3AqLYcqoh5x/A==", "dev": true, "license": "MIT", "dependencies": { "@volar/typescript": "2.4.28", - "@vue/language-core": "3.3.12" + "@vue/language-core": "3.3.13" }, "bin": { "vue-tsc": "bin/vue-tsc.js" diff --git a/webui/package.json b/webui/package.json index ab3e61fa4..00289951d 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,5 +1,6 @@ { "name": "mokapi", + "type": "module", "version": "0.10.0", "private": true, "scripts": { @@ -36,7 +37,7 @@ "js-yaml": "^5.2.2", "kafkajs": "^2.2.4", "ldapts": "^9.0.0", - "markdown-it": "^14.2.0", + "markdown-it": "^15.0.0", "markdown-it-container": "^4.0.0", "mime-types": "^3.0.2", "mqtt": "^5.15.1", @@ -49,22 +50,22 @@ "xml-formatter": "^3.7.0" }, "devDependencies": { - "@playwright/test": "1.63.0", + "@playwright/test": "1.64.0", "@rushstack/eslint-patch": "1.16.1", "@types/js-yaml": "4.0.9", "@types/markdown-it-container": "4.0.1", - "@types/node": "26.6.4", - "@vitejs/plugin-vue": "6.0.9", + "@types/node": "26.6.5", + "@vitejs/plugin-vue": "6.0.10", "@vue/eslint-config-prettier": "10.2.0", "@vue/eslint-config-typescript": "14.9.0", "@vue/tsconfig": "0.9.1", "eslint": "10.12.0", "eslint-plugin-vue": "10.11.1", "npm-run-all2": "9.0.3", - "prettier": "3.9.9", + "prettier": "3.9.10", "typescript": "6.0.3", - "vite": "8.3.3", - "vue-tsc": "3.3.12", + "vite": "8.3.4", + "vue-tsc": "3.3.13", "xml2js": "0.6.2" }, "allowScripts": { diff --git a/webui/scripts/dashboard-demo/collect-dashboard.ts b/webui/scripts/dashboard-demo/collect-dashboard.ts index 09767dc95..2f922f7b4 100644 --- a/webui/scripts/dashboard-demo/collect-dashboard.ts +++ b/webui/scripts/dashboard-demo/collect-dashboard.ts @@ -15,17 +15,27 @@ export async function collectDashboard() { 'service_Swagger Petstore': { path: '/api/services/http/Swagger%20Petstore', loader: loadJson }, 'service_Swagger Petstore_operations': { path: '/api/services/http/Swagger%20Petstore/operations', loader: loadJson }, 'service_Swagger Petstore_bruno': { path: '/api/services/http/Swagger%20Petstore/bruno.yaml', loader: load }, + 'service_Kafka Order Service API': { path: '/api/services/kafka/Kafka%20Order%20Service%20API', loader: loadJson }, + 'service_Kafka Order Service API_channel_order-topic': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/channels/order-topic', loader: loadJson }, 'service_Kafka Order Service API_topic_order-topic': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/topics/order-topic', loader: loadJson }, + 'service_Kafka Order Service API_channel_user-events': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/channels/user-events', loader: loadJson }, 'service_Kafka Order Service API_topic_user-events': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/topics/user-events', loader: loadJson }, + 'service_Kafka Order Service API_channel_shipments.{region}.updates': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/channels/shipments.{region}.updates', loader: loadJson }, + 'service_Kafka Order Service API_topic_shipments.eu.updates': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/topics/shipments.eu.updates', loader: loadJson }, + 'service_Kafka Order Service API_group_order-status-group-100': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/groups/order-status-group-100', loader: loadJson }, 'service_Kafka Order Service API_client_consumer-1': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/clients/consumer-1', loader: loadJson }, 'service_Kafka Order Service API_client_producer-1': { path: '/api/services/kafka/Kafka%20Order%20Service%20API/clients/producer-1', loader: loadJson }, + 'service_Mail Server': { path: '/api/services/mail/Mail%20Server', loader: loadJson }, + 'service_HR Employee Directory': { path: '/api/services/ldap/HR%20Employee%20Directory', loader: loadJson }, + 'service_Smart Home MQTT API': { path: '/api/services/mqtt/Smart%20Home%20MQTT%20API', loader: loadJson }, + 'service_WebSocket Chat API': { path: '/api/services/websocket/WebSocket%20Chat%20API', loader: loadJson }, - events: { path: '/api/events', loader: fetchEvents }, + events: { path: '/api/events?size=1000', loader: fetchEvents }, 'mailbox_alice.johnson@example.com': { path: '/api/services/mail/Mail%20Server/mailboxes/alice.johnson@example.com', loader: loadJson }, 'mailbox_bob.miller@example.com': { path: '/api/services/mail/Mail%20Server/mailboxes/bob.miller@example.com', loader: loadJson }, configs: { path: '/api/configs', loader: loadConfigs }, @@ -73,11 +83,14 @@ async function directoryExists(path: string) { } async function fetchEvents(url: string, key: string, snapshot: Record) { - const events = await fetchJson(url); - snapshot[key] = events + const page = await fetchJson(url); + snapshot[key] = page.items const mails = []; - for (const event of events) { + for (const event of page.items) { + const detail = await fetchJson(`${baseUrl}/api/events/${event.id}`); + snapshot['event_'+event.id] = detail + if (event.traits.namespace !== 'mail') { continue; } diff --git a/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml b/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml index 347c24e0c..46e8be695 100644 --- a/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml +++ b/webui/scripts/dashboard-demo/demo-configs/asyncapi.yaml @@ -23,9 +23,30 @@ channels: summary: The Kafka topic for user events. tags: - name: user + shipment_updates: + address: shipments.{region}.updates + title: Shipment Updates per Region + summary: Regional topics with shipment tracking events. + description: | + Each fulfilment region publishes tracking events to its own topic, + for example `shipments.eu.updates` or `shipments.us.updates`. + Topics are created on demand as soon as a region starts shipping, + so a single channel definition covers all regional topics. + parameters: + region: + description: Fulfilment region that handles the shipment. + enum: [eu, us, apac, latam] + examples: [eu, us] + messages: + ShipmentUpdated: + $ref: '#/components/messages/ShipmentUpdated' + bindings: + kafka: + partitions: 2 + tags: + - name: shipping operations: - # Operation for publishing new orders publishNewOrder: action: send channel: @@ -38,9 +59,7 @@ operations: type: string description: The producer client ID, must be prefixed with 'producer-'. pattern: '^producer-[a-zA-Z0-9-]+$' - bindingVersion: 0.4.0 - # Operation for subscribing to status updates (presumably from another service) receiveOrderStatus: action: receive channel: @@ -53,7 +72,27 @@ operations: type: string description: The consumer group ID, must adhere to internal naming conventions. pattern: '^order-status-group-[0-9]{3}$' - bindingVersion: 0.4.0 + + publishShipmentUpdate: + action: send + channel: + $ref: '#/channels/shipment_updates' + summary: A regional fulfilment center reports a shipment status change. + messages: + - $ref: '#/channels/shipment_updates/messages/ShipmentUpdated' + + receiveShipmentUpdates: + action: receive + channel: + $ref: '#/channels/shipment_updates' + summary: The order service tracks shipments across all regions. + messages: + - $ref: '#/channels/shipment_updates/messages/ShipmentUpdated' + bindings: + kafka: + groupId: + type: string + pattern: '^shipment-tracking-[a-z]+$' components: messages: @@ -66,7 +105,7 @@ components: examples: - name: new-order-example payload: - orderId: "12345" + orderId: "7e2b4d0f-1aa1-49e3-9f11-54c266d480c3" productId: "PROD-ABC" quantity: 2 customerEmail: "customer@example.com" @@ -81,11 +120,47 @@ components: examples: - name: status-update-example payload: - orderId: "12345" + orderId: "7e2b4d0f-1aa1-49e3-9f11-54c266d480c3" productId: "PROD-ABC" quantity: 2 customerEmail: "customer@example.com" status: "SHIPPED" + + ShipmentUpdated: + name: ShipmentUpdatedEvent + title: Shipment Updated Event + summary: A shipment moved to a new tracking status. + payload: + $ref: '#/components/schemas/ShipmentPayload' + bindings: + kafka: + key: + type: string + format: uuid + description: The order ID, so all events of an order stay in order on one partition. + bindingVersion: 0.4.0 + examples: + - name: shipment-in-transit + summary: Parcel left the EU hub. + payload: + shipmentId: "8f14e45f-ceea-467f-a8f2-3c6e5b1d2a90" + orderId: "c9bf9e57-1685-4c89-bafb-ff5af830be8a" + region: eu + carrier: DHL + status: IN_TRANSIT + location: "Frankfurt Hub" + timestamp: "2026-10-20T09:15:00Z" + estimatedDelivery: "2026-10-22" + - name: shipment-delivered + summary: Parcel delivered in the US. + payload: + shipmentId: "1b4e28ba-2fa1-41d2-883f-0016d3cca427" + orderId: "a3bb189e-8bf9-4888-9912-ace4e6543002" + region: us + carrier: UPS + status: DELIVERED + location: "Austin, TX" + timestamp: "2026-10-20T16:42:00Z" schemas: OrderPayload: @@ -117,3 +192,37 @@ components: - quantity - customerEmail - status + + ShipmentPayload: + type: object + properties: + shipmentId: + type: string + format: uuid + description: Unique ID of the shipment. + orderId: + type: string + format: uuid + description: The order this shipment belongs to. + region: + type: string + enum: [eu, us, apac, latam] + description: Fulfilment region; matches the topic parameter. + carrier: + type: string + enum: [DHL, UPS, FedEx, DPD, Post] + status: + type: string + enum: [LABEL_CREATED, PICKED_UP, IN_TRANSIT, OUT_FOR_DELIVERY, DELIVERED, EXCEPTION] + description: Current tracking status. + location: + type: string + description: Last known location of the parcel. + timestamp: + type: string + format: date-time + description: When the status change happened. + estimatedDelivery: + type: string + format: date + required: [shipmentId, orderId, region, carrier, status, timestamp] \ No newline at end of file diff --git a/webui/scripts/dashboard-demo/demo-configs/tsconfig.json b/webui/scripts/dashboard-demo/demo-configs/tsconfig.json new file mode 100644 index 000000000..c2ff43631 --- /dev/null +++ b/webui/scripts/dashboard-demo/demo-configs/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "lib": ["es2020", "dom"], + "paths": { + "mokapi": ["../../../../npm/go-mokapi/types/index"], + "mokapi/*": ["../../../../npm/go-mokapi/types/*"] + }, + "types": ["../../../../npm/go-mokapi/types/global"] + } +} \ No newline at end of file diff --git a/webui/scripts/dashboard-demo/demo-configs/websocket.ts b/webui/scripts/dashboard-demo/demo-configs/websocket.ts index 11e544199..4e355865c 100644 --- a/webui/scripts/dashboard-demo/demo-configs/websocket.ts +++ b/webui/scripts/dashboard-demo/demo-configs/websocket.ts @@ -2,9 +2,11 @@ import { on } from 'mokapi' export default function() { on('websocket', (event) => { - event.broadcast({ - from: event.message.username, - text: event.message.text - }) + if (event.type === 'message') { + event.broadcast({ + from: event.message.username, + text: event.message.text + }) + } }, { track: true }) } \ No newline at end of file diff --git a/webui/scripts/dashboard-demo/drive-kafka.ts b/webui/scripts/dashboard-demo/drive-kafka.ts index ffb9777a0..e8c22eec5 100644 --- a/webui/scripts/dashboard-demo/drive-kafka.ts +++ b/webui/scripts/dashboard-demo/drive-kafka.ts @@ -2,7 +2,7 @@ import { Kafka } from 'kafkajs'; const producerClient = new Kafka({ clientId: 'producer-1', - brokers: ['localhost:9092'] + brokers: ['localhost:9092'], }); const consumerClient = new Kafka({ @@ -12,7 +12,8 @@ const consumerClient = new Kafka({ const consumer = consumerClient.consumer({ groupId: 'order-status-group-100' }); const producer = producerClient.producer({ - idempotent: true + idempotent: true, + allowAutoTopicCreation: true }); export async function driveKafka() { @@ -22,7 +23,7 @@ export async function driveKafka() { await consumer.subscribe({ topic: 'order-topic', fromBeginning: true }); await consumer.run({ - eachMessage: async ({ topic, partition, message }) => { + eachMessage: async ({ message }) => { const value = JSON.parse(message.value!.toString()); console.log('Received command:', value); } @@ -42,6 +43,23 @@ export async function driveKafka() { messages: [{ key: order.orderId, value: JSON.stringify(order) }] }); + await producer.send({ + topic: 'shipments.eu.updates', + messages: [{ + key: '7e2b4d0f-1aa1-49e3-9f11-54c266d480c3', + value: JSON.stringify({ + shipmentId: '8f14e45f-ceea-467f-a8f2-3c6e5b1d2a90', + orderId: '7e2b4d0f-1aa1-49e3-9f11-54c266d480c3', + region: 'eu', + carrier: 'DHL', + status: 'IN_TRANSIT', + location: 'Frankfurt Hub', + timestamp: '2026-10-20T09:15:00Z', + estimatedDelivery: '2026-10-22' + }) + }], + }); + console.log('Published order:', order); }; diff --git a/webui/src/assets/dashboard.css b/webui/src/assets/dashboard.css index f80b7bd7b..a0449a504 100644 --- a/webui/src/assets/dashboard.css +++ b/webui/src/assets/dashboard.css @@ -234,4 +234,6 @@ .status { text-transform: capitalize; color: var(--color-red); -} \ No newline at end of file +} + +.dashboard .channel-name .param { color: var(--bs-primary); font-weight: 500; } \ No newline at end of file diff --git a/webui/src/assets/datatable.css b/webui/src/assets/datatable.css index 8c4ed6bd6..3312daa80 100644 --- a/webui/src/assets/datatable.css +++ b/webui/src/assets/datatable.css @@ -92,4 +92,12 @@ table.dataTable button.row-link { .cell-link:hover::after { opacity: 0.8; +} + +.pagination .page-link { + cursor: pointer; + font-size: 12px; +} +.pagination .page-item:not(.active) .page-link { + color: var(--link-color) } \ No newline at end of file diff --git a/webui/src/components/dashboard/Actions.vue b/webui/src/components/dashboard/Actions.vue index 5f23ceb48..083bf9aff 100644 --- a/webui/src/components/dashboard/Actions.vue +++ b/webui/src/components/dashboard/Actions.vue @@ -5,6 +5,7 @@ import SourceView from './SourceView.vue' import { usePrettyLanguage } from '@/composables/usePrettyLanguage' import { usePrettyText } from '@/composables/usePrettyText' import { getRouteName } from '@/composables/dashboard' +import { usePrettyBytes } from '@/composables/usePrettyBytes.ts' defineProps({ actions: { type: Object as PropType, required: true }, @@ -14,6 +15,7 @@ const { formatLanguage } = usePrettyLanguage() const { duration } = usePrettyDates() const { parseUrls } = usePrettyText() +const { format: formatBytes } = usePrettyBytes() let status: string | null function getName(action: Action){ @@ -30,27 +32,45 @@ function formatParameters(action: Action): {name?: string, value: string}[] { return [ { name: 'request', - value: formatLanguage(action.parameters[0], 'application/json') + value: formatParam(action.parameters[0]).text }, { name: 'response', - value: formatLanguage(action.parameters[1], 'application/json') + value: formatParam(action.parameters[1]).text } ] case 'kafka': return [ { name: 'message', - value: formatLanguage(action.parameters[0], 'application/json') + value: formatParam(action.parameters[0]).text } ] case 'mqtt': return [ - { name: 'message', value: formatLanguage(action.parameters[0], 'application/json') } + { name: 'message', value: formatParam(action.parameters[0]).text } ] case 'websocket': return [ - { name: 'event', value: formatLanguage(action.parameters[0], 'application/json') } + { name: 'event', value: formatParam(action.parameters[0]).text } + ] + case 'ldap': + return [ + { + name: 'request', + value: formatParam(action.parameters[0]).text + }, + { + name: 'response', + value: formatParam(action.parameters[1]).text + } + ] + case 'smtp': + return [ + { + name: 'message', + value: formatParam(action.parameters[0]).text + } ] } @@ -87,6 +107,33 @@ function getStatus(action: Action) { } return 'success' } + +const isTruncated = function(v: any): v is Truncated { + return v !== null && typeof v === 'object' && v.truncated === true +} + +function formatParam(raw: any) { + const truncated: Truncated[] = [] + const token = `__TRUNCATED_${crypto.randomUUID()}_` + const obj = JSON.parse(raw, (_key, value) => { + if (isTruncated(value)) { + truncated.push(value) + return `${token}${truncated.length - 1}__` + } + return value + }) + + const text = JSON.stringify(obj, null, 2).replace( + new RegExp(`"${token}(\\d+)__"`, 'g'), + (_m, i) => { + const t = truncated[Number(i)] + return `"${t.preview}…" ⟨${formatBytes(t.size - t.preview.length)} more, truncated⟩` + } + ) + + return { text, truncated } +} +